From 8fb29dac7045aa87643794b665a00c4036807664 Mon Sep 17 00:00:00 2001 From: civ Date: Sun, 5 Jul 2026 18:11:23 +0700 Subject: [PATCH] initial release --- CMakeLists.txt | 200 + Dockerfile | 43 + LICENSE | 21 + README.md | 128 + buildall.cmd | 12 + buildall.sh | 10 + buildcpu.sh | 8 + buildcuda.cmd | 12 + buildcuda.sh | 8 + buildsycl.sh | 8 + buildtermux.sh | 8 + buildvulkan.cmd | 12 + buildvulkan.sh | 8 + checkpoints.sh | 26 + convert.py | 329 + docker-compose.yml | 31 + docs/ARCHITECTURE.md | 900 + examples/client.sh | 20 + examples/clone.cmd | 13 + examples/clone.sh | 11 + examples/freeman.rvq | Bin 0 -> 4741 bytes examples/freeman.txt | 1 + examples/prompt.txt | 1 + examples/server.sh | 7 + examples/tts.cmd | 12 + examples/tts.sh | 10 + format.sh | 3 + ggml/.editorconfig | 22 + ggml/.github/pull_request_template.md | 1 + ggml/.github/workflows/ci.yml | 272 + ggml/.github/workflows/release.yml | 27 + ggml/.gitignore | 39 + ggml/.gitmodules | 0 ggml/AUTHORS | 335 + ggml/CMakeLists.txt | 505 + ggml/CONTRIBUTING.md | 3 + ggml/LICENSE | 21 + ggml/README.md | 50 + ggml/ci/run.sh | 395 + ggml/cmake/FindNCCL.cmake | 36 + ggml/cmake/GitVars.cmake | 22 + ggml/cmake/common.cmake | 50 + ggml/cmake/ggml-config.cmake.in | 201 + ggml/docs/gguf.md | 828 + ggml/examples/CMakeLists.txt | 34 + ggml/examples/common-ggml.cpp | 244 + ggml/examples/common-ggml.h | 18 + ggml/examples/common.cpp | 675 + ggml/examples/common.h | 322 + ggml/examples/gpt-2/CMakeLists.txt | 32 + ggml/examples/gpt-2/README.md | 225 + .../gpt-2/convert-cerebras-to-ggml.py | 183 + ggml/examples/gpt-2/convert-ckpt-to-ggml.py | 159 + ggml/examples/gpt-2/convert-h5-to-ggml.py | 195 + ggml/examples/gpt-2/download-ggml-model.sh | 69 + ggml/examples/gpt-2/download-model.sh | 48 + ggml/examples/gpt-2/main-alloc.cpp | 880 + ggml/examples/gpt-2/main-backend.cpp | 946 + ggml/examples/gpt-2/main-batched.cpp | 1210 + ggml/examples/gpt-2/main-ctx.cpp | 840 + ggml/examples/gpt-2/main-sched.cpp | 1079 + ggml/examples/gpt-2/quantize.cpp | 184 + ggml/examples/gpt-j/CMakeLists.txt | 13 + ggml/examples/gpt-j/README.md | 239 + ggml/examples/gpt-j/convert-h5-to-ggml.py | 173 + ggml/examples/gpt-j/download-ggml-model.sh | 69 + ggml/examples/gpt-j/download-model.sh | 11 + ggml/examples/gpt-j/main.cpp | 755 + ggml/examples/gpt-j/quantize.cpp | 182 + ggml/examples/magika/CMakeLists.txt | 17 + ggml/examples/magika/README.md | 23 + ggml/examples/magika/convert.py | 32 + ggml/examples/magika/main.cpp | 374 + ggml/examples/mnist/.gitignore | 3 + ggml/examples/mnist/CMakeLists.txt | 58 + ggml/examples/mnist/README.md | 206 + ggml/examples/mnist/mnist-common.cpp | 496 + ggml/examples/mnist/mnist-common.h | 166 + ggml/examples/mnist/mnist-eval.cpp | 67 + ggml/examples/mnist/mnist-train-cnn.py | 91 + ggml/examples/mnist/mnist-train-fc.py | 131 + ggml/examples/mnist/mnist-train.cpp | 39 + ggml/examples/mnist/server.py | 36 + ggml/examples/perf-metal/CMakeLists.txt | 7 + ggml/examples/perf-metal/perf-metal.cpp | 152 + ggml/examples/prompts/dolly-v2.txt | 100 + ggml/examples/prompts/gpt-2-chinese.txt | 1 + ggml/examples/prompts/gpt-2.txt | 100 + ggml/examples/prompts/gpt-j.txt | 100 + ggml/examples/prompts/gpt-neox-japanese.txt | 1 + ggml/examples/prompts/gpt-neox.txt | 100 + ggml/examples/prompts/polyglot-ko.txt | 3 + ggml/examples/prompts/replit.txt | 100 + ggml/examples/prompts/starcoder.txt | 100 + ggml/examples/prompts/test-cases.txt | 110 + ggml/examples/prompts/tokenize_huggingface.py | 65 + ggml/examples/prompts/whisper.txt | 100 + ggml/examples/python/README.md | 115 + ggml/examples/python/api.h | 14 + ggml/examples/python/example_add_quant.py | 25 + .../python/example_test_all_quants.py | 68 + ggml/examples/python/ggml/__init__.py | 58 + ggml/examples/python/ggml/__init__.pyi | 2406 ++ ggml/examples/python/ggml/cffi.py | 11 + ggml/examples/python/ggml/ffi/__init__.pyi | 7 + ggml/examples/python/ggml/utils.py | 182 + ggml/examples/python/regenerate.py | 42 + ggml/examples/python/stubs.py | 128 + ggml/examples/python/test_tensor.py | 258 + ggml/examples/sam/CMakeLists.txt | 13 + ggml/examples/sam/README.md | 95 + ggml/examples/sam/convert-pth-to-ggml.py | 147 + ggml/examples/sam/example.jpg | Bin 0 -> 79236 bytes ggml/examples/sam/sam.cpp | 2370 ++ ggml/examples/simple/CMakeLists.txt | 21 + ggml/examples/simple/README.md | 61 + ggml/examples/simple/simple-backend.cpp | 153 + ggml/examples/simple/simple-ctx.cpp | 127 + ggml/examples/stb_image.h | 7987 +++++++ ggml/examples/stb_image_write.h | 1724 ++ ggml/examples/test-cmake/CMakeLists.txt | 10 + ggml/examples/test-cmake/README.md | 3 + ggml/examples/test-cmake/test-cmake.cpp | 6 + ggml/examples/yolo/CMakeLists.txt | 6 + ggml/examples/yolo/README.md | 59 + ggml/examples/yolo/convert-yolov3-tiny.py | 53 + ggml/examples/yolo/data/coco.names | 80 + ggml/examples/yolo/data/labels/100_0.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/100_1.png | Bin 0 -> 377 bytes ggml/examples/yolo/data/labels/100_2.png | Bin 0 -> 451 bytes ggml/examples/yolo/data/labels/100_3.png | Bin 0 -> 508 bytes ggml/examples/yolo/data/labels/100_4.png | Bin 0 -> 577 bytes ggml/examples/yolo/data/labels/100_5.png | Bin 0 -> 631 bytes ggml/examples/yolo/data/labels/100_6.png | Bin 0 -> 697 bytes ggml/examples/yolo/data/labels/100_7.png | Bin 0 -> 753 bytes ggml/examples/yolo/data/labels/101_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/101_1.png | Bin 0 -> 388 bytes ggml/examples/yolo/data/labels/101_2.png | Bin 0 -> 458 bytes ggml/examples/yolo/data/labels/101_3.png | Bin 0 -> 514 bytes ggml/examples/yolo/data/labels/101_4.png | Bin 0 -> 581 bytes ggml/examples/yolo/data/labels/101_5.png | Bin 0 -> 654 bytes ggml/examples/yolo/data/labels/101_6.png | Bin 0 -> 726 bytes ggml/examples/yolo/data/labels/101_7.png | Bin 0 -> 804 bytes ggml/examples/yolo/data/labels/102_0.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/102_1.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/102_2.png | Bin 0 -> 354 bytes ggml/examples/yolo/data/labels/102_3.png | Bin 0 -> 371 bytes ggml/examples/yolo/data/labels/102_4.png | Bin 0 -> 398 bytes ggml/examples/yolo/data/labels/102_5.png | Bin 0 -> 411 bytes ggml/examples/yolo/data/labels/102_6.png | Bin 0 -> 422 bytes ggml/examples/yolo/data/labels/102_7.png | Bin 0 -> 442 bytes ggml/examples/yolo/data/labels/103_0.png | Bin 0 -> 333 bytes ggml/examples/yolo/data/labels/103_1.png | Bin 0 -> 415 bytes ggml/examples/yolo/data/labels/103_2.png | Bin 0 -> 521 bytes ggml/examples/yolo/data/labels/103_3.png | Bin 0 -> 586 bytes ggml/examples/yolo/data/labels/103_4.png | Bin 0 -> 687 bytes ggml/examples/yolo/data/labels/103_5.png | Bin 0 -> 781 bytes ggml/examples/yolo/data/labels/103_6.png | Bin 0 -> 858 bytes ggml/examples/yolo/data/labels/103_7.png | Bin 0 -> 971 bytes ggml/examples/yolo/data/labels/104_0.png | Bin 0 -> 315 bytes ggml/examples/yolo/data/labels/104_1.png | Bin 0 -> 345 bytes ggml/examples/yolo/data/labels/104_2.png | Bin 0 -> 382 bytes ggml/examples/yolo/data/labels/104_3.png | Bin 0 -> 412 bytes ggml/examples/yolo/data/labels/104_4.png | Bin 0 -> 439 bytes ggml/examples/yolo/data/labels/104_5.png | Bin 0 -> 476 bytes ggml/examples/yolo/data/labels/104_6.png | Bin 0 -> 511 bytes ggml/examples/yolo/data/labels/104_7.png | Bin 0 -> 542 bytes ggml/examples/yolo/data/labels/105_0.png | Bin 0 -> 296 bytes ggml/examples/yolo/data/labels/105_1.png | Bin 0 -> 306 bytes ggml/examples/yolo/data/labels/105_2.png | Bin 0 -> 318 bytes ggml/examples/yolo/data/labels/105_3.png | Bin 0 -> 336 bytes ggml/examples/yolo/data/labels/105_4.png | Bin 0 -> 352 bytes ggml/examples/yolo/data/labels/105_5.png | Bin 0 -> 360 bytes ggml/examples/yolo/data/labels/105_6.png | Bin 0 -> 379 bytes ggml/examples/yolo/data/labels/105_7.png | Bin 0 -> 391 bytes ggml/examples/yolo/data/labels/106_0.png | Bin 0 -> 293 bytes ggml/examples/yolo/data/labels/106_1.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/106_2.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/106_3.png | Bin 0 -> 335 bytes ggml/examples/yolo/data/labels/106_4.png | Bin 0 -> 348 bytes ggml/examples/yolo/data/labels/106_5.png | Bin 0 -> 363 bytes ggml/examples/yolo/data/labels/106_6.png | Bin 0 -> 374 bytes ggml/examples/yolo/data/labels/106_7.png | Bin 0 -> 390 bytes ggml/examples/yolo/data/labels/107_0.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/107_1.png | Bin 0 -> 358 bytes ggml/examples/yolo/data/labels/107_2.png | Bin 0 -> 410 bytes ggml/examples/yolo/data/labels/107_3.png | Bin 0 -> 446 bytes ggml/examples/yolo/data/labels/107_4.png | Bin 0 -> 490 bytes ggml/examples/yolo/data/labels/107_5.png | Bin 0 -> 526 bytes ggml/examples/yolo/data/labels/107_6.png | Bin 0 -> 581 bytes ggml/examples/yolo/data/labels/107_7.png | Bin 0 -> 609 bytes ggml/examples/yolo/data/labels/108_0.png | Bin 0 -> 285 bytes ggml/examples/yolo/data/labels/108_1.png | Bin 0 -> 288 bytes ggml/examples/yolo/data/labels/108_2.png | Bin 0 -> 296 bytes ggml/examples/yolo/data/labels/108_3.png | Bin 0 -> 298 bytes ggml/examples/yolo/data/labels/108_4.png | Bin 0 -> 298 bytes ggml/examples/yolo/data/labels/108_5.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/108_6.png | Bin 0 -> 302 bytes ggml/examples/yolo/data/labels/108_7.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/109_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/109_1.png | Bin 0 -> 371 bytes ggml/examples/yolo/data/labels/109_2.png | Bin 0 -> 426 bytes ggml/examples/yolo/data/labels/109_3.png | Bin 0 -> 475 bytes ggml/examples/yolo/data/labels/109_4.png | Bin 0 -> 528 bytes ggml/examples/yolo/data/labels/109_5.png | Bin 0 -> 587 bytes ggml/examples/yolo/data/labels/109_6.png | Bin 0 -> 628 bytes ggml/examples/yolo/data/labels/109_7.png | Bin 0 -> 694 bytes ggml/examples/yolo/data/labels/110_0.png | Bin 0 -> 312 bytes ggml/examples/yolo/data/labels/110_1.png | Bin 0 -> 341 bytes ggml/examples/yolo/data/labels/110_2.png | Bin 0 -> 378 bytes ggml/examples/yolo/data/labels/110_3.png | Bin 0 -> 414 bytes ggml/examples/yolo/data/labels/110_4.png | Bin 0 -> 444 bytes ggml/examples/yolo/data/labels/110_5.png | Bin 0 -> 479 bytes ggml/examples/yolo/data/labels/110_6.png | Bin 0 -> 509 bytes ggml/examples/yolo/data/labels/110_7.png | Bin 0 -> 544 bytes ggml/examples/yolo/data/labels/111_0.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/111_1.png | Bin 0 -> 396 bytes ggml/examples/yolo/data/labels/111_2.png | Bin 0 -> 471 bytes ggml/examples/yolo/data/labels/111_3.png | Bin 0 -> 534 bytes ggml/examples/yolo/data/labels/111_4.png | Bin 0 -> 615 bytes ggml/examples/yolo/data/labels/111_5.png | Bin 0 -> 686 bytes ggml/examples/yolo/data/labels/111_6.png | Bin 0 -> 781 bytes ggml/examples/yolo/data/labels/111_7.png | Bin 0 -> 855 bytes ggml/examples/yolo/data/labels/112_0.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/112_1.png | Bin 0 -> 382 bytes ggml/examples/yolo/data/labels/112_2.png | Bin 0 -> 450 bytes ggml/examples/yolo/data/labels/112_3.png | Bin 0 -> 507 bytes ggml/examples/yolo/data/labels/112_4.png | Bin 0 -> 568 bytes ggml/examples/yolo/data/labels/112_5.png | Bin 0 -> 641 bytes ggml/examples/yolo/data/labels/112_6.png | Bin 0 -> 710 bytes ggml/examples/yolo/data/labels/112_7.png | Bin 0 -> 766 bytes ggml/examples/yolo/data/labels/113_0.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/113_1.png | Bin 0 -> 386 bytes ggml/examples/yolo/data/labels/113_2.png | Bin 0 -> 450 bytes ggml/examples/yolo/data/labels/113_3.png | Bin 0 -> 506 bytes ggml/examples/yolo/data/labels/113_4.png | Bin 0 -> 572 bytes ggml/examples/yolo/data/labels/113_5.png | Bin 0 -> 631 bytes ggml/examples/yolo/data/labels/113_6.png | Bin 0 -> 702 bytes ggml/examples/yolo/data/labels/113_7.png | Bin 0 -> 760 bytes ggml/examples/yolo/data/labels/114_0.png | Bin 0 -> 299 bytes ggml/examples/yolo/data/labels/114_1.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/114_2.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/114_3.png | Bin 0 -> 363 bytes ggml/examples/yolo/data/labels/114_4.png | Bin 0 -> 391 bytes ggml/examples/yolo/data/labels/114_5.png | Bin 0 -> 413 bytes ggml/examples/yolo/data/labels/114_6.png | Bin 0 -> 441 bytes ggml/examples/yolo/data/labels/114_7.png | Bin 0 -> 460 bytes ggml/examples/yolo/data/labels/115_0.png | Bin 0 -> 315 bytes ggml/examples/yolo/data/labels/115_1.png | Bin 0 -> 373 bytes ggml/examples/yolo/data/labels/115_2.png | Bin 0 -> 436 bytes ggml/examples/yolo/data/labels/115_3.png | Bin 0 -> 497 bytes ggml/examples/yolo/data/labels/115_4.png | Bin 0 -> 552 bytes ggml/examples/yolo/data/labels/115_5.png | Bin 0 -> 618 bytes ggml/examples/yolo/data/labels/115_6.png | Bin 0 -> 687 bytes ggml/examples/yolo/data/labels/115_7.png | Bin 0 -> 766 bytes ggml/examples/yolo/data/labels/116_0.png | Bin 0 -> 299 bytes ggml/examples/yolo/data/labels/116_1.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/116_2.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/116_3.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/116_4.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/116_5.png | Bin 0 -> 328 bytes ggml/examples/yolo/data/labels/116_6.png | Bin 0 -> 332 bytes ggml/examples/yolo/data/labels/116_7.png | Bin 0 -> 337 bytes ggml/examples/yolo/data/labels/117_0.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/117_1.png | Bin 0 -> 350 bytes ggml/examples/yolo/data/labels/117_2.png | Bin 0 -> 386 bytes ggml/examples/yolo/data/labels/117_3.png | Bin 0 -> 421 bytes ggml/examples/yolo/data/labels/117_4.png | Bin 0 -> 470 bytes ggml/examples/yolo/data/labels/117_5.png | Bin 0 -> 515 bytes ggml/examples/yolo/data/labels/117_6.png | Bin 0 -> 549 bytes ggml/examples/yolo/data/labels/117_7.png | Bin 0 -> 584 bytes ggml/examples/yolo/data/labels/118_0.png | Bin 0 -> 315 bytes ggml/examples/yolo/data/labels/118_1.png | Bin 0 -> 369 bytes ggml/examples/yolo/data/labels/118_2.png | Bin 0 -> 442 bytes ggml/examples/yolo/data/labels/118_3.png | Bin 0 -> 502 bytes ggml/examples/yolo/data/labels/118_4.png | Bin 0 -> 576 bytes ggml/examples/yolo/data/labels/118_5.png | Bin 0 -> 653 bytes ggml/examples/yolo/data/labels/118_6.png | Bin 0 -> 682 bytes ggml/examples/yolo/data/labels/118_7.png | Bin 0 -> 750 bytes ggml/examples/yolo/data/labels/119_0.png | Bin 0 -> 334 bytes ggml/examples/yolo/data/labels/119_1.png | Bin 0 -> 432 bytes ggml/examples/yolo/data/labels/119_2.png | Bin 0 -> 550 bytes ggml/examples/yolo/data/labels/119_3.png | Bin 0 -> 657 bytes ggml/examples/yolo/data/labels/119_4.png | Bin 0 -> 759 bytes ggml/examples/yolo/data/labels/119_5.png | Bin 0 -> 886 bytes ggml/examples/yolo/data/labels/119_6.png | Bin 0 -> 977 bytes ggml/examples/yolo/data/labels/119_7.png | Bin 0 -> 1086 bytes ggml/examples/yolo/data/labels/120_0.png | Bin 0 -> 316 bytes ggml/examples/yolo/data/labels/120_1.png | Bin 0 -> 380 bytes ggml/examples/yolo/data/labels/120_2.png | Bin 0 -> 452 bytes ggml/examples/yolo/data/labels/120_3.png | Bin 0 -> 509 bytes ggml/examples/yolo/data/labels/120_4.png | Bin 0 -> 580 bytes ggml/examples/yolo/data/labels/120_5.png | Bin 0 -> 634 bytes ggml/examples/yolo/data/labels/120_6.png | Bin 0 -> 706 bytes ggml/examples/yolo/data/labels/120_7.png | Bin 0 -> 773 bytes ggml/examples/yolo/data/labels/121_0.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/121_1.png | Bin 0 -> 395 bytes ggml/examples/yolo/data/labels/121_2.png | Bin 0 -> 477 bytes ggml/examples/yolo/data/labels/121_3.png | Bin 0 -> 563 bytes ggml/examples/yolo/data/labels/121_4.png | Bin 0 -> 645 bytes ggml/examples/yolo/data/labels/121_5.png | Bin 0 -> 726 bytes ggml/examples/yolo/data/labels/121_6.png | Bin 0 -> 798 bytes ggml/examples/yolo/data/labels/121_7.png | Bin 0 -> 866 bytes ggml/examples/yolo/data/labels/122_0.png | Bin 0 -> 311 bytes ggml/examples/yolo/data/labels/122_1.png | Bin 0 -> 352 bytes ggml/examples/yolo/data/labels/122_2.png | Bin 0 -> 388 bytes ggml/examples/yolo/data/labels/122_3.png | Bin 0 -> 421 bytes ggml/examples/yolo/data/labels/122_4.png | Bin 0 -> 458 bytes ggml/examples/yolo/data/labels/122_5.png | Bin 0 -> 500 bytes ggml/examples/yolo/data/labels/122_6.png | Bin 0 -> 541 bytes ggml/examples/yolo/data/labels/122_7.png | Bin 0 -> 556 bytes ggml/examples/yolo/data/labels/123_0.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/123_1.png | Bin 0 -> 384 bytes ggml/examples/yolo/data/labels/123_2.png | Bin 0 -> 446 bytes ggml/examples/yolo/data/labels/123_3.png | Bin 0 -> 503 bytes ggml/examples/yolo/data/labels/123_4.png | Bin 0 -> 551 bytes ggml/examples/yolo/data/labels/123_5.png | Bin 0 -> 601 bytes ggml/examples/yolo/data/labels/123_6.png | Bin 0 -> 661 bytes ggml/examples/yolo/data/labels/123_7.png | Bin 0 -> 712 bytes ggml/examples/yolo/data/labels/124_0.png | Bin 0 -> 286 bytes ggml/examples/yolo/data/labels/124_1.png | Bin 0 -> 288 bytes ggml/examples/yolo/data/labels/124_2.png | Bin 0 -> 287 bytes ggml/examples/yolo/data/labels/124_3.png | Bin 0 -> 294 bytes ggml/examples/yolo/data/labels/124_4.png | Bin 0 -> 293 bytes ggml/examples/yolo/data/labels/124_5.png | Bin 0 -> 299 bytes ggml/examples/yolo/data/labels/124_6.png | Bin 0 -> 296 bytes ggml/examples/yolo/data/labels/124_7.png | Bin 0 -> 301 bytes ggml/examples/yolo/data/labels/125_0.png | Bin 0 -> 318 bytes ggml/examples/yolo/data/labels/125_1.png | Bin 0 -> 385 bytes ggml/examples/yolo/data/labels/125_2.png | Bin 0 -> 437 bytes ggml/examples/yolo/data/labels/125_3.png | Bin 0 -> 496 bytes ggml/examples/yolo/data/labels/125_4.png | Bin 0 -> 547 bytes ggml/examples/yolo/data/labels/125_5.png | Bin 0 -> 602 bytes ggml/examples/yolo/data/labels/125_6.png | Bin 0 -> 655 bytes ggml/examples/yolo/data/labels/125_7.png | Bin 0 -> 689 bytes ggml/examples/yolo/data/labels/126_0.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/126_1.png | Bin 0 -> 343 bytes ggml/examples/yolo/data/labels/126_2.png | Bin 0 -> 368 bytes ggml/examples/yolo/data/labels/126_3.png | Bin 0 -> 406 bytes ggml/examples/yolo/data/labels/126_4.png | Bin 0 -> 437 bytes ggml/examples/yolo/data/labels/126_5.png | Bin 0 -> 483 bytes ggml/examples/yolo/data/labels/126_6.png | Bin 0 -> 524 bytes ggml/examples/yolo/data/labels/126_7.png | Bin 0 -> 561 bytes ggml/examples/yolo/data/labels/32_0.png | Bin 0 -> 280 bytes ggml/examples/yolo/data/labels/32_1.png | Bin 0 -> 281 bytes ggml/examples/yolo/data/labels/32_2.png | Bin 0 -> 281 bytes ggml/examples/yolo/data/labels/32_3.png | Bin 0 -> 282 bytes ggml/examples/yolo/data/labels/32_4.png | Bin 0 -> 282 bytes ggml/examples/yolo/data/labels/32_5.png | Bin 0 -> 285 bytes ggml/examples/yolo/data/labels/32_6.png | Bin 0 -> 285 bytes ggml/examples/yolo/data/labels/32_7.png | Bin 0 -> 285 bytes ggml/examples/yolo/data/labels/33_0.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/33_1.png | Bin 0 -> 313 bytes ggml/examples/yolo/data/labels/33_2.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/33_3.png | Bin 0 -> 339 bytes ggml/examples/yolo/data/labels/33_4.png | Bin 0 -> 348 bytes ggml/examples/yolo/data/labels/33_5.png | Bin 0 -> 365 bytes ggml/examples/yolo/data/labels/33_6.png | Bin 0 -> 382 bytes ggml/examples/yolo/data/labels/33_7.png | Bin 0 -> 393 bytes ggml/examples/yolo/data/labels/34_0.png | Bin 0 -> 295 bytes ggml/examples/yolo/data/labels/34_1.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/34_2.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/34_3.png | Bin 0 -> 316 bytes ggml/examples/yolo/data/labels/34_4.png | Bin 0 -> 318 bytes ggml/examples/yolo/data/labels/34_5.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/34_6.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/34_7.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/35_0.png | Bin 0 -> 338 bytes ggml/examples/yolo/data/labels/35_1.png | Bin 0 -> 445 bytes ggml/examples/yolo/data/labels/35_2.png | Bin 0 -> 535 bytes ggml/examples/yolo/data/labels/35_3.png | Bin 0 -> 636 bytes ggml/examples/yolo/data/labels/35_4.png | Bin 0 -> 739 bytes ggml/examples/yolo/data/labels/35_5.png | Bin 0 -> 830 bytes ggml/examples/yolo/data/labels/35_6.png | Bin 0 -> 905 bytes ggml/examples/yolo/data/labels/35_7.png | Bin 0 -> 999 bytes ggml/examples/yolo/data/labels/36_0.png | Bin 0 -> 344 bytes ggml/examples/yolo/data/labels/36_1.png | Bin 0 -> 441 bytes ggml/examples/yolo/data/labels/36_2.png | Bin 0 -> 517 bytes ggml/examples/yolo/data/labels/36_3.png | Bin 0 -> 599 bytes ggml/examples/yolo/data/labels/36_4.png | Bin 0 -> 687 bytes ggml/examples/yolo/data/labels/36_5.png | Bin 0 -> 762 bytes ggml/examples/yolo/data/labels/36_6.png | Bin 0 -> 871 bytes ggml/examples/yolo/data/labels/36_7.png | Bin 0 -> 949 bytes ggml/examples/yolo/data/labels/37_0.png | Bin 0 -> 351 bytes ggml/examples/yolo/data/labels/37_1.png | Bin 0 -> 491 bytes ggml/examples/yolo/data/labels/37_2.png | Bin 0 -> 658 bytes ggml/examples/yolo/data/labels/37_3.png | Bin 0 -> 858 bytes ggml/examples/yolo/data/labels/37_4.png | Bin 0 -> 998 bytes ggml/examples/yolo/data/labels/37_5.png | Bin 0 -> 1157 bytes ggml/examples/yolo/data/labels/37_6.png | Bin 0 -> 1259 bytes ggml/examples/yolo/data/labels/37_7.png | Bin 0 -> 1478 bytes ggml/examples/yolo/data/labels/38_0.png | Bin 0 -> 342 bytes ggml/examples/yolo/data/labels/38_1.png | Bin 0 -> 466 bytes ggml/examples/yolo/data/labels/38_2.png | Bin 0 -> 585 bytes ggml/examples/yolo/data/labels/38_3.png | Bin 0 -> 698 bytes ggml/examples/yolo/data/labels/38_4.png | Bin 0 -> 823 bytes ggml/examples/yolo/data/labels/38_5.png | Bin 0 -> 942 bytes ggml/examples/yolo/data/labels/38_6.png | Bin 0 -> 1053 bytes ggml/examples/yolo/data/labels/38_7.png | Bin 0 -> 1159 bytes ggml/examples/yolo/data/labels/39_0.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/39_1.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/39_2.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/39_3.png | Bin 0 -> 368 bytes ggml/examples/yolo/data/labels/39_4.png | Bin 0 -> 388 bytes ggml/examples/yolo/data/labels/39_5.png | Bin 0 -> 418 bytes ggml/examples/yolo/data/labels/39_6.png | Bin 0 -> 444 bytes ggml/examples/yolo/data/labels/39_7.png | Bin 0 -> 470 bytes ggml/examples/yolo/data/labels/40_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/40_1.png | Bin 0 -> 390 bytes ggml/examples/yolo/data/labels/40_2.png | Bin 0 -> 449 bytes ggml/examples/yolo/data/labels/40_3.png | Bin 0 -> 520 bytes ggml/examples/yolo/data/labels/40_4.png | Bin 0 -> 584 bytes ggml/examples/yolo/data/labels/40_5.png | Bin 0 -> 641 bytes ggml/examples/yolo/data/labels/40_6.png | Bin 0 -> 681 bytes ggml/examples/yolo/data/labels/40_7.png | Bin 0 -> 758 bytes ggml/examples/yolo/data/labels/41_0.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/41_1.png | Bin 0 -> 385 bytes ggml/examples/yolo/data/labels/41_2.png | Bin 0 -> 441 bytes ggml/examples/yolo/data/labels/41_3.png | Bin 0 -> 522 bytes ggml/examples/yolo/data/labels/41_4.png | Bin 0 -> 577 bytes ggml/examples/yolo/data/labels/41_5.png | Bin 0 -> 637 bytes ggml/examples/yolo/data/labels/41_6.png | Bin 0 -> 690 bytes ggml/examples/yolo/data/labels/41_7.png | Bin 0 -> 753 bytes ggml/examples/yolo/data/labels/42_0.png | Bin 0 -> 311 bytes ggml/examples/yolo/data/labels/42_1.png | Bin 0 -> 352 bytes ggml/examples/yolo/data/labels/42_2.png | Bin 0 -> 400 bytes ggml/examples/yolo/data/labels/42_3.png | Bin 0 -> 456 bytes ggml/examples/yolo/data/labels/42_4.png | Bin 0 -> 520 bytes ggml/examples/yolo/data/labels/42_5.png | Bin 0 -> 564 bytes ggml/examples/yolo/data/labels/42_6.png | Bin 0 -> 606 bytes ggml/examples/yolo/data/labels/42_7.png | Bin 0 -> 677 bytes ggml/examples/yolo/data/labels/43_0.png | Bin 0 -> 301 bytes ggml/examples/yolo/data/labels/43_1.png | Bin 0 -> 312 bytes ggml/examples/yolo/data/labels/43_2.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/43_3.png | Bin 0 -> 326 bytes ggml/examples/yolo/data/labels/43_4.png | Bin 0 -> 333 bytes ggml/examples/yolo/data/labels/43_5.png | Bin 0 -> 339 bytes ggml/examples/yolo/data/labels/43_6.png | Bin 0 -> 347 bytes ggml/examples/yolo/data/labels/43_7.png | Bin 0 -> 354 bytes ggml/examples/yolo/data/labels/44_0.png | Bin 0 -> 296 bytes ggml/examples/yolo/data/labels/44_1.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/44_2.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/44_3.png | Bin 0 -> 368 bytes ggml/examples/yolo/data/labels/44_4.png | Bin 0 -> 400 bytes ggml/examples/yolo/data/labels/44_5.png | Bin 0 -> 416 bytes ggml/examples/yolo/data/labels/44_6.png | Bin 0 -> 451 bytes ggml/examples/yolo/data/labels/44_7.png | Bin 0 -> 480 bytes ggml/examples/yolo/data/labels/45_0.png | Bin 0 -> 289 bytes ggml/examples/yolo/data/labels/45_1.png | Bin 0 -> 298 bytes ggml/examples/yolo/data/labels/45_2.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/45_3.png | Bin 0 -> 302 bytes ggml/examples/yolo/data/labels/45_4.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/45_5.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/45_6.png | Bin 0 -> 311 bytes ggml/examples/yolo/data/labels/45_7.png | Bin 0 -> 316 bytes ggml/examples/yolo/data/labels/46_0.png | Bin 0 -> 289 bytes ggml/examples/yolo/data/labels/46_1.png | Bin 0 -> 296 bytes ggml/examples/yolo/data/labels/46_2.png | Bin 0 -> 308 bytes ggml/examples/yolo/data/labels/46_3.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/46_4.png | Bin 0 -> 336 bytes ggml/examples/yolo/data/labels/46_5.png | Bin 0 -> 348 bytes ggml/examples/yolo/data/labels/46_6.png | Bin 0 -> 362 bytes ggml/examples/yolo/data/labels/46_7.png | Bin 0 -> 377 bytes ggml/examples/yolo/data/labels/47_0.png | Bin 0 -> 326 bytes ggml/examples/yolo/data/labels/47_1.png | Bin 0 -> 372 bytes ggml/examples/yolo/data/labels/47_2.png | Bin 0 -> 448 bytes ggml/examples/yolo/data/labels/47_3.png | Bin 0 -> 518 bytes ggml/examples/yolo/data/labels/47_4.png | Bin 0 -> 561 bytes ggml/examples/yolo/data/labels/47_5.png | Bin 0 -> 629 bytes ggml/examples/yolo/data/labels/47_6.png | Bin 0 -> 672 bytes ggml/examples/yolo/data/labels/47_7.png | Bin 0 -> 724 bytes ggml/examples/yolo/data/labels/48_0.png | Bin 0 -> 337 bytes ggml/examples/yolo/data/labels/48_1.png | Bin 0 -> 440 bytes ggml/examples/yolo/data/labels/48_2.png | Bin 0 -> 550 bytes ggml/examples/yolo/data/labels/48_3.png | Bin 0 -> 640 bytes ggml/examples/yolo/data/labels/48_4.png | Bin 0 -> 753 bytes ggml/examples/yolo/data/labels/48_5.png | Bin 0 -> 838 bytes ggml/examples/yolo/data/labels/48_6.png | Bin 0 -> 938 bytes ggml/examples/yolo/data/labels/48_7.png | Bin 0 -> 1023 bytes ggml/examples/yolo/data/labels/49_0.png | Bin 0 -> 297 bytes ggml/examples/yolo/data/labels/49_1.png | Bin 0 -> 306 bytes ggml/examples/yolo/data/labels/49_2.png | Bin 0 -> 313 bytes ggml/examples/yolo/data/labels/49_3.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/49_4.png | Bin 0 -> 331 bytes ggml/examples/yolo/data/labels/49_5.png | Bin 0 -> 335 bytes ggml/examples/yolo/data/labels/49_6.png | Bin 0 -> 349 bytes ggml/examples/yolo/data/labels/49_7.png | Bin 0 -> 352 bytes ggml/examples/yolo/data/labels/50_0.png | Bin 0 -> 334 bytes ggml/examples/yolo/data/labels/50_1.png | Bin 0 -> 414 bytes ggml/examples/yolo/data/labels/50_2.png | Bin 0 -> 477 bytes ggml/examples/yolo/data/labels/50_3.png | Bin 0 -> 550 bytes ggml/examples/yolo/data/labels/50_4.png | Bin 0 -> 615 bytes ggml/examples/yolo/data/labels/50_5.png | Bin 0 -> 689 bytes ggml/examples/yolo/data/labels/50_6.png | Bin 0 -> 744 bytes ggml/examples/yolo/data/labels/50_7.png | Bin 0 -> 840 bytes ggml/examples/yolo/data/labels/51_0.png | Bin 0 -> 336 bytes ggml/examples/yolo/data/labels/51_1.png | Bin 0 -> 428 bytes ggml/examples/yolo/data/labels/51_2.png | Bin 0 -> 510 bytes ggml/examples/yolo/data/labels/51_3.png | Bin 0 -> 608 bytes ggml/examples/yolo/data/labels/51_4.png | Bin 0 -> 693 bytes ggml/examples/yolo/data/labels/51_5.png | Bin 0 -> 787 bytes ggml/examples/yolo/data/labels/51_6.png | Bin 0 -> 872 bytes ggml/examples/yolo/data/labels/51_7.png | Bin 0 -> 958 bytes ggml/examples/yolo/data/labels/52_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/52_1.png | Bin 0 -> 368 bytes ggml/examples/yolo/data/labels/52_2.png | Bin 0 -> 412 bytes ggml/examples/yolo/data/labels/52_3.png | Bin 0 -> 455 bytes ggml/examples/yolo/data/labels/52_4.png | Bin 0 -> 502 bytes ggml/examples/yolo/data/labels/52_5.png | Bin 0 -> 547 bytes ggml/examples/yolo/data/labels/52_6.png | Bin 0 -> 589 bytes ggml/examples/yolo/data/labels/52_7.png | Bin 0 -> 633 bytes ggml/examples/yolo/data/labels/53_0.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/53_1.png | Bin 0 -> 415 bytes ggml/examples/yolo/data/labels/53_2.png | Bin 0 -> 495 bytes ggml/examples/yolo/data/labels/53_3.png | Bin 0 -> 577 bytes ggml/examples/yolo/data/labels/53_4.png | Bin 0 -> 658 bytes ggml/examples/yolo/data/labels/53_5.png | Bin 0 -> 742 bytes ggml/examples/yolo/data/labels/53_6.png | Bin 0 -> 820 bytes ggml/examples/yolo/data/labels/53_7.png | Bin 0 -> 891 bytes ggml/examples/yolo/data/labels/54_0.png | Bin 0 -> 339 bytes ggml/examples/yolo/data/labels/54_1.png | Bin 0 -> 434 bytes ggml/examples/yolo/data/labels/54_2.png | Bin 0 -> 543 bytes ggml/examples/yolo/data/labels/54_3.png | Bin 0 -> 633 bytes ggml/examples/yolo/data/labels/54_4.png | Bin 0 -> 723 bytes ggml/examples/yolo/data/labels/54_5.png | Bin 0 -> 818 bytes ggml/examples/yolo/data/labels/54_6.png | Bin 0 -> 907 bytes ggml/examples/yolo/data/labels/54_7.png | Bin 0 -> 1006 bytes ggml/examples/yolo/data/labels/55_0.png | Bin 0 -> 327 bytes ggml/examples/yolo/data/labels/55_1.png | Bin 0 -> 385 bytes ggml/examples/yolo/data/labels/55_2.png | Bin 0 -> 449 bytes ggml/examples/yolo/data/labels/55_3.png | Bin 0 -> 503 bytes ggml/examples/yolo/data/labels/55_4.png | Bin 0 -> 548 bytes ggml/examples/yolo/data/labels/55_5.png | Bin 0 -> 598 bytes ggml/examples/yolo/data/labels/55_6.png | Bin 0 -> 640 bytes ggml/examples/yolo/data/labels/55_7.png | Bin 0 -> 710 bytes ggml/examples/yolo/data/labels/56_0.png | Bin 0 -> 337 bytes ggml/examples/yolo/data/labels/56_1.png | Bin 0 -> 449 bytes ggml/examples/yolo/data/labels/56_2.png | Bin 0 -> 553 bytes ggml/examples/yolo/data/labels/56_3.png | Bin 0 -> 671 bytes ggml/examples/yolo/data/labels/56_4.png | Bin 0 -> 774 bytes ggml/examples/yolo/data/labels/56_5.png | Bin 0 -> 888 bytes ggml/examples/yolo/data/labels/56_6.png | Bin 0 -> 996 bytes ggml/examples/yolo/data/labels/56_7.png | Bin 0 -> 1088 bytes ggml/examples/yolo/data/labels/57_0.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/57_1.png | Bin 0 -> 435 bytes ggml/examples/yolo/data/labels/57_2.png | Bin 0 -> 532 bytes ggml/examples/yolo/data/labels/57_3.png | Bin 0 -> 629 bytes ggml/examples/yolo/data/labels/57_4.png | Bin 0 -> 725 bytes ggml/examples/yolo/data/labels/57_5.png | Bin 0 -> 823 bytes ggml/examples/yolo/data/labels/57_6.png | Bin 0 -> 924 bytes ggml/examples/yolo/data/labels/57_7.png | Bin 0 -> 1012 bytes ggml/examples/yolo/data/labels/58_0.png | Bin 0 -> 301 bytes ggml/examples/yolo/data/labels/58_1.png | Bin 0 -> 318 bytes ggml/examples/yolo/data/labels/58_2.png | Bin 0 -> 332 bytes ggml/examples/yolo/data/labels/58_3.png | Bin 0 -> 360 bytes ggml/examples/yolo/data/labels/58_4.png | Bin 0 -> 387 bytes ggml/examples/yolo/data/labels/58_5.png | Bin 0 -> 402 bytes ggml/examples/yolo/data/labels/58_6.png | Bin 0 -> 426 bytes ggml/examples/yolo/data/labels/58_7.png | Bin 0 -> 453 bytes ggml/examples/yolo/data/labels/59_0.png | Bin 0 -> 306 bytes ggml/examples/yolo/data/labels/59_1.png | Bin 0 -> 335 bytes ggml/examples/yolo/data/labels/59_2.png | Bin 0 -> 367 bytes ggml/examples/yolo/data/labels/59_3.png | Bin 0 -> 408 bytes ggml/examples/yolo/data/labels/59_4.png | Bin 0 -> 452 bytes ggml/examples/yolo/data/labels/59_5.png | Bin 0 -> 491 bytes ggml/examples/yolo/data/labels/59_6.png | Bin 0 -> 530 bytes ggml/examples/yolo/data/labels/59_7.png | Bin 0 -> 560 bytes ggml/examples/yolo/data/labels/60_0.png | Bin 0 -> 323 bytes ggml/examples/yolo/data/labels/60_1.png | Bin 0 -> 387 bytes ggml/examples/yolo/data/labels/60_2.png | Bin 0 -> 447 bytes ggml/examples/yolo/data/labels/60_3.png | Bin 0 -> 499 bytes ggml/examples/yolo/data/labels/60_4.png | Bin 0 -> 578 bytes ggml/examples/yolo/data/labels/60_5.png | Bin 0 -> 648 bytes ggml/examples/yolo/data/labels/60_6.png | Bin 0 -> 717 bytes ggml/examples/yolo/data/labels/60_7.png | Bin 0 -> 770 bytes ggml/examples/yolo/data/labels/61_0.png | Bin 0 -> 297 bytes ggml/examples/yolo/data/labels/61_1.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/61_2.png | Bin 0 -> 310 bytes ggml/examples/yolo/data/labels/61_3.png | Bin 0 -> 315 bytes ggml/examples/yolo/data/labels/61_4.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/61_5.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/61_6.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/61_7.png | Bin 0 -> 345 bytes ggml/examples/yolo/data/labels/62_0.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/62_1.png | Bin 0 -> 384 bytes ggml/examples/yolo/data/labels/62_2.png | Bin 0 -> 444 bytes ggml/examples/yolo/data/labels/62_3.png | Bin 0 -> 496 bytes ggml/examples/yolo/data/labels/62_4.png | Bin 0 -> 580 bytes ggml/examples/yolo/data/labels/62_5.png | Bin 0 -> 641 bytes ggml/examples/yolo/data/labels/62_6.png | Bin 0 -> 702 bytes ggml/examples/yolo/data/labels/62_7.png | Bin 0 -> 761 bytes ggml/examples/yolo/data/labels/63_0.png | Bin 0 -> 329 bytes ggml/examples/yolo/data/labels/63_1.png | Bin 0 -> 410 bytes ggml/examples/yolo/data/labels/63_2.png | Bin 0 -> 488 bytes ggml/examples/yolo/data/labels/63_3.png | Bin 0 -> 573 bytes ggml/examples/yolo/data/labels/63_4.png | Bin 0 -> 641 bytes ggml/examples/yolo/data/labels/63_5.png | Bin 0 -> 729 bytes ggml/examples/yolo/data/labels/63_6.png | Bin 0 -> 826 bytes ggml/examples/yolo/data/labels/63_7.png | Bin 0 -> 893 bytes ggml/examples/yolo/data/labels/64_0.png | Bin 0 -> 384 bytes ggml/examples/yolo/data/labels/64_1.png | Bin 0 -> 588 bytes ggml/examples/yolo/data/labels/64_2.png | Bin 0 -> 805 bytes ggml/examples/yolo/data/labels/64_3.png | Bin 0 -> 1038 bytes ggml/examples/yolo/data/labels/64_4.png | Bin 0 -> 1211 bytes ggml/examples/yolo/data/labels/64_5.png | Bin 0 -> 1438 bytes ggml/examples/yolo/data/labels/64_6.png | Bin 0 -> 1647 bytes ggml/examples/yolo/data/labels/64_7.png | Bin 0 -> 1829 bytes ggml/examples/yolo/data/labels/65_0.png | Bin 0 -> 338 bytes ggml/examples/yolo/data/labels/65_1.png | Bin 0 -> 424 bytes ggml/examples/yolo/data/labels/65_2.png | Bin 0 -> 525 bytes ggml/examples/yolo/data/labels/65_3.png | Bin 0 -> 613 bytes ggml/examples/yolo/data/labels/65_4.png | Bin 0 -> 699 bytes ggml/examples/yolo/data/labels/65_5.png | Bin 0 -> 788 bytes ggml/examples/yolo/data/labels/65_6.png | Bin 0 -> 851 bytes ggml/examples/yolo/data/labels/65_7.png | Bin 0 -> 961 bytes ggml/examples/yolo/data/labels/66_0.png | Bin 0 -> 325 bytes ggml/examples/yolo/data/labels/66_1.png | Bin 0 -> 386 bytes ggml/examples/yolo/data/labels/66_2.png | Bin 0 -> 450 bytes ggml/examples/yolo/data/labels/66_3.png | Bin 0 -> 498 bytes ggml/examples/yolo/data/labels/66_4.png | Bin 0 -> 570 bytes ggml/examples/yolo/data/labels/66_5.png | Bin 0 -> 624 bytes ggml/examples/yolo/data/labels/66_6.png | Bin 0 -> 691 bytes ggml/examples/yolo/data/labels/66_7.png | Bin 0 -> 740 bytes ggml/examples/yolo/data/labels/67_0.png | Bin 0 -> 331 bytes ggml/examples/yolo/data/labels/67_1.png | Bin 0 -> 415 bytes ggml/examples/yolo/data/labels/67_2.png | Bin 0 -> 513 bytes ggml/examples/yolo/data/labels/67_3.png | Bin 0 -> 608 bytes ggml/examples/yolo/data/labels/67_4.png | Bin 0 -> 699 bytes ggml/examples/yolo/data/labels/67_5.png | Bin 0 -> 789 bytes ggml/examples/yolo/data/labels/67_6.png | Bin 0 -> 876 bytes ggml/examples/yolo/data/labels/67_7.png | Bin 0 -> 958 bytes ggml/examples/yolo/data/labels/68_0.png | Bin 0 -> 329 bytes ggml/examples/yolo/data/labels/68_1.png | Bin 0 -> 398 bytes ggml/examples/yolo/data/labels/68_2.png | Bin 0 -> 457 bytes ggml/examples/yolo/data/labels/68_3.png | Bin 0 -> 521 bytes ggml/examples/yolo/data/labels/68_4.png | Bin 0 -> 594 bytes ggml/examples/yolo/data/labels/68_5.png | Bin 0 -> 649 bytes ggml/examples/yolo/data/labels/68_6.png | Bin 0 -> 702 bytes ggml/examples/yolo/data/labels/68_7.png | Bin 0 -> 773 bytes ggml/examples/yolo/data/labels/69_0.png | Bin 0 -> 310 bytes ggml/examples/yolo/data/labels/69_1.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/69_2.png | Bin 0 -> 325 bytes ggml/examples/yolo/data/labels/69_3.png | Bin 0 -> 327 bytes ggml/examples/yolo/data/labels/69_4.png | Bin 0 -> 331 bytes ggml/examples/yolo/data/labels/69_5.png | Bin 0 -> 341 bytes ggml/examples/yolo/data/labels/69_6.png | Bin 0 -> 346 bytes ggml/examples/yolo/data/labels/69_7.png | Bin 0 -> 357 bytes ggml/examples/yolo/data/labels/70_0.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/70_1.png | Bin 0 -> 311 bytes ggml/examples/yolo/data/labels/70_2.png | Bin 0 -> 310 bytes ggml/examples/yolo/data/labels/70_3.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/70_4.png | Bin 0 -> 329 bytes ggml/examples/yolo/data/labels/70_5.png | Bin 0 -> 328 bytes ggml/examples/yolo/data/labels/70_6.png | Bin 0 -> 337 bytes ggml/examples/yolo/data/labels/70_7.png | Bin 0 -> 335 bytes ggml/examples/yolo/data/labels/71_0.png | Bin 0 -> 345 bytes ggml/examples/yolo/data/labels/71_1.png | Bin 0 -> 448 bytes ggml/examples/yolo/data/labels/71_2.png | Bin 0 -> 556 bytes ggml/examples/yolo/data/labels/71_3.png | Bin 0 -> 678 bytes ggml/examples/yolo/data/labels/71_4.png | Bin 0 -> 767 bytes ggml/examples/yolo/data/labels/71_5.png | Bin 0 -> 872 bytes ggml/examples/yolo/data/labels/71_6.png | Bin 0 -> 954 bytes ggml/examples/yolo/data/labels/71_7.png | Bin 0 -> 1067 bytes ggml/examples/yolo/data/labels/72_0.png | Bin 0 -> 306 bytes ggml/examples/yolo/data/labels/72_1.png | Bin 0 -> 310 bytes ggml/examples/yolo/data/labels/72_2.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/72_3.png | Bin 0 -> 320 bytes ggml/examples/yolo/data/labels/72_4.png | Bin 0 -> 338 bytes ggml/examples/yolo/data/labels/72_5.png | Bin 0 -> 348 bytes ggml/examples/yolo/data/labels/72_6.png | Bin 0 -> 355 bytes ggml/examples/yolo/data/labels/72_7.png | Bin 0 -> 360 bytes ggml/examples/yolo/data/labels/73_0.png | Bin 0 -> 288 bytes ggml/examples/yolo/data/labels/73_1.png | Bin 0 -> 290 bytes ggml/examples/yolo/data/labels/73_2.png | Bin 0 -> 293 bytes ggml/examples/yolo/data/labels/73_3.png | Bin 0 -> 295 bytes ggml/examples/yolo/data/labels/73_4.png | Bin 0 -> 302 bytes ggml/examples/yolo/data/labels/73_5.png | Bin 0 -> 304 bytes ggml/examples/yolo/data/labels/73_6.png | Bin 0 -> 306 bytes ggml/examples/yolo/data/labels/73_7.png | Bin 0 -> 310 bytes ggml/examples/yolo/data/labels/74_0.png | Bin 0 -> 301 bytes ggml/examples/yolo/data/labels/74_1.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/74_2.png | Bin 0 -> 344 bytes ggml/examples/yolo/data/labels/74_3.png | Bin 0 -> 370 bytes ggml/examples/yolo/data/labels/74_4.png | Bin 0 -> 392 bytes ggml/examples/yolo/data/labels/74_5.png | Bin 0 -> 418 bytes ggml/examples/yolo/data/labels/74_6.png | Bin 0 -> 445 bytes ggml/examples/yolo/data/labels/74_7.png | Bin 0 -> 481 bytes ggml/examples/yolo/data/labels/75_0.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/75_1.png | Bin 0 -> 393 bytes ggml/examples/yolo/data/labels/75_2.png | Bin 0 -> 454 bytes ggml/examples/yolo/data/labels/75_3.png | Bin 0 -> 524 bytes ggml/examples/yolo/data/labels/75_4.png | Bin 0 -> 596 bytes ggml/examples/yolo/data/labels/75_5.png | Bin 0 -> 647 bytes ggml/examples/yolo/data/labels/75_6.png | Bin 0 -> 698 bytes ggml/examples/yolo/data/labels/75_7.png | Bin 0 -> 760 bytes ggml/examples/yolo/data/labels/76_0.png | Bin 0 -> 298 bytes ggml/examples/yolo/data/labels/76_1.png | Bin 0 -> 302 bytes ggml/examples/yolo/data/labels/76_2.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/76_3.png | Bin 0 -> 309 bytes ggml/examples/yolo/data/labels/76_4.png | Bin 0 -> 318 bytes ggml/examples/yolo/data/labels/76_5.png | Bin 0 -> 316 bytes ggml/examples/yolo/data/labels/76_6.png | Bin 0 -> 322 bytes ggml/examples/yolo/data/labels/76_7.png | Bin 0 -> 336 bytes ggml/examples/yolo/data/labels/77_0.png | Bin 0 -> 373 bytes ggml/examples/yolo/data/labels/77_1.png | Bin 0 -> 518 bytes ggml/examples/yolo/data/labels/77_2.png | Bin 0 -> 674 bytes ggml/examples/yolo/data/labels/77_3.png | Bin 0 -> 820 bytes ggml/examples/yolo/data/labels/77_4.png | Bin 0 -> 961 bytes ggml/examples/yolo/data/labels/77_5.png | Bin 0 -> 1098 bytes ggml/examples/yolo/data/labels/77_6.png | Bin 0 -> 1272 bytes ggml/examples/yolo/data/labels/77_7.png | Bin 0 -> 1416 bytes ggml/examples/yolo/data/labels/78_0.png | Bin 0 -> 329 bytes ggml/examples/yolo/data/labels/78_1.png | Bin 0 -> 397 bytes ggml/examples/yolo/data/labels/78_2.png | Bin 0 -> 449 bytes ggml/examples/yolo/data/labels/78_3.png | Bin 0 -> 512 bytes ggml/examples/yolo/data/labels/78_4.png | Bin 0 -> 559 bytes ggml/examples/yolo/data/labels/78_5.png | Bin 0 -> 600 bytes ggml/examples/yolo/data/labels/78_6.png | Bin 0 -> 646 bytes ggml/examples/yolo/data/labels/78_7.png | Bin 0 -> 691 bytes ggml/examples/yolo/data/labels/79_0.png | Bin 0 -> 353 bytes ggml/examples/yolo/data/labels/79_1.png | Bin 0 -> 457 bytes ggml/examples/yolo/data/labels/79_2.png | Bin 0 -> 586 bytes ggml/examples/yolo/data/labels/79_3.png | Bin 0 -> 717 bytes ggml/examples/yolo/data/labels/79_4.png | Bin 0 -> 836 bytes ggml/examples/yolo/data/labels/79_5.png | Bin 0 -> 950 bytes ggml/examples/yolo/data/labels/79_6.png | Bin 0 -> 1057 bytes ggml/examples/yolo/data/labels/79_7.png | Bin 0 -> 1167 bytes ggml/examples/yolo/data/labels/80_0.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/80_1.png | Bin 0 -> 349 bytes ggml/examples/yolo/data/labels/80_2.png | Bin 0 -> 382 bytes ggml/examples/yolo/data/labels/80_3.png | Bin 0 -> 421 bytes ggml/examples/yolo/data/labels/80_4.png | Bin 0 -> 458 bytes ggml/examples/yolo/data/labels/80_5.png | Bin 0 -> 484 bytes ggml/examples/yolo/data/labels/80_6.png | Bin 0 -> 528 bytes ggml/examples/yolo/data/labels/80_7.png | Bin 0 -> 556 bytes ggml/examples/yolo/data/labels/81_0.png | Bin 0 -> 360 bytes ggml/examples/yolo/data/labels/81_1.png | Bin 0 -> 484 bytes ggml/examples/yolo/data/labels/81_2.png | Bin 0 -> 614 bytes ggml/examples/yolo/data/labels/81_3.png | Bin 0 -> 756 bytes ggml/examples/yolo/data/labels/81_4.png | Bin 0 -> 899 bytes ggml/examples/yolo/data/labels/81_5.png | Bin 0 -> 1011 bytes ggml/examples/yolo/data/labels/81_6.png | Bin 0 -> 1123 bytes ggml/examples/yolo/data/labels/81_7.png | Bin 0 -> 1271 bytes ggml/examples/yolo/data/labels/82_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/82_1.png | Bin 0 -> 391 bytes ggml/examples/yolo/data/labels/82_2.png | Bin 0 -> 450 bytes ggml/examples/yolo/data/labels/82_3.png | Bin 0 -> 508 bytes ggml/examples/yolo/data/labels/82_4.png | Bin 0 -> 560 bytes ggml/examples/yolo/data/labels/82_5.png | Bin 0 -> 657 bytes ggml/examples/yolo/data/labels/82_6.png | Bin 0 -> 698 bytes ggml/examples/yolo/data/labels/82_7.png | Bin 0 -> 739 bytes ggml/examples/yolo/data/labels/83_0.png | Bin 0 -> 336 bytes ggml/examples/yolo/data/labels/83_1.png | Bin 0 -> 426 bytes ggml/examples/yolo/data/labels/83_2.png | Bin 0 -> 505 bytes ggml/examples/yolo/data/labels/83_3.png | Bin 0 -> 607 bytes ggml/examples/yolo/data/labels/83_4.png | Bin 0 -> 688 bytes ggml/examples/yolo/data/labels/83_5.png | Bin 0 -> 795 bytes ggml/examples/yolo/data/labels/83_6.png | Bin 0 -> 872 bytes ggml/examples/yolo/data/labels/83_7.png | Bin 0 -> 963 bytes ggml/examples/yolo/data/labels/84_0.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/84_1.png | Bin 0 -> 312 bytes ggml/examples/yolo/data/labels/84_2.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/84_3.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/84_4.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/84_5.png | Bin 0 -> 331 bytes ggml/examples/yolo/data/labels/84_6.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/84_7.png | Bin 0 -> 345 bytes ggml/examples/yolo/data/labels/85_0.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/85_1.png | Bin 0 -> 369 bytes ggml/examples/yolo/data/labels/85_2.png | Bin 0 -> 420 bytes ggml/examples/yolo/data/labels/85_3.png | Bin 0 -> 475 bytes ggml/examples/yolo/data/labels/85_4.png | Bin 0 -> 519 bytes ggml/examples/yolo/data/labels/85_5.png | Bin 0 -> 571 bytes ggml/examples/yolo/data/labels/85_6.png | Bin 0 -> 629 bytes ggml/examples/yolo/data/labels/85_7.png | Bin 0 -> 676 bytes ggml/examples/yolo/data/labels/86_0.png | Bin 0 -> 344 bytes ggml/examples/yolo/data/labels/86_1.png | Bin 0 -> 434 bytes ggml/examples/yolo/data/labels/86_2.png | Bin 0 -> 529 bytes ggml/examples/yolo/data/labels/86_3.png | Bin 0 -> 613 bytes ggml/examples/yolo/data/labels/86_4.png | Bin 0 -> 708 bytes ggml/examples/yolo/data/labels/86_5.png | Bin 0 -> 784 bytes ggml/examples/yolo/data/labels/86_6.png | Bin 0 -> 880 bytes ggml/examples/yolo/data/labels/86_7.png | Bin 0 -> 958 bytes ggml/examples/yolo/data/labels/87_0.png | Bin 0 -> 376 bytes ggml/examples/yolo/data/labels/87_1.png | Bin 0 -> 521 bytes ggml/examples/yolo/data/labels/87_2.png | Bin 0 -> 693 bytes ggml/examples/yolo/data/labels/87_3.png | Bin 0 -> 833 bytes ggml/examples/yolo/data/labels/87_4.png | Bin 0 -> 989 bytes ggml/examples/yolo/data/labels/87_5.png | Bin 0 -> 1122 bytes ggml/examples/yolo/data/labels/87_6.png | Bin 0 -> 1289 bytes ggml/examples/yolo/data/labels/87_7.png | Bin 0 -> 1451 bytes ggml/examples/yolo/data/labels/88_0.png | Bin 0 -> 338 bytes ggml/examples/yolo/data/labels/88_1.png | Bin 0 -> 443 bytes ggml/examples/yolo/data/labels/88_2.png | Bin 0 -> 540 bytes ggml/examples/yolo/data/labels/88_3.png | Bin 0 -> 638 bytes ggml/examples/yolo/data/labels/88_4.png | Bin 0 -> 742 bytes ggml/examples/yolo/data/labels/88_5.png | Bin 0 -> 844 bytes ggml/examples/yolo/data/labels/88_6.png | Bin 0 -> 933 bytes ggml/examples/yolo/data/labels/88_7.png | Bin 0 -> 1013 bytes ggml/examples/yolo/data/labels/89_0.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/89_1.png | Bin 0 -> 387 bytes ggml/examples/yolo/data/labels/89_2.png | Bin 0 -> 447 bytes ggml/examples/yolo/data/labels/89_3.png | Bin 0 -> 502 bytes ggml/examples/yolo/data/labels/89_4.png | Bin 0 -> 569 bytes ggml/examples/yolo/data/labels/89_5.png | Bin 0 -> 620 bytes ggml/examples/yolo/data/labels/89_6.png | Bin 0 -> 679 bytes ggml/examples/yolo/data/labels/89_7.png | Bin 0 -> 743 bytes ggml/examples/yolo/data/labels/90_0.png | Bin 0 -> 334 bytes ggml/examples/yolo/data/labels/90_1.png | Bin 0 -> 387 bytes ggml/examples/yolo/data/labels/90_2.png | Bin 0 -> 436 bytes ggml/examples/yolo/data/labels/90_3.png | Bin 0 -> 500 bytes ggml/examples/yolo/data/labels/90_4.png | Bin 0 -> 549 bytes ggml/examples/yolo/data/labels/90_5.png | Bin 0 -> 595 bytes ggml/examples/yolo/data/labels/90_6.png | Bin 0 -> 644 bytes ggml/examples/yolo/data/labels/90_7.png | Bin 0 -> 697 bytes ggml/examples/yolo/data/labels/91_0.png | Bin 0 -> 292 bytes ggml/examples/yolo/data/labels/91_1.png | Bin 0 -> 301 bytes ggml/examples/yolo/data/labels/91_2.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/91_3.png | Bin 0 -> 312 bytes ggml/examples/yolo/data/labels/91_4.png | Bin 0 -> 316 bytes ggml/examples/yolo/data/labels/91_5.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/91_6.png | Bin 0 -> 327 bytes ggml/examples/yolo/data/labels/91_7.png | Bin 0 -> 331 bytes ggml/examples/yolo/data/labels/92_0.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/92_1.png | Bin 0 -> 375 bytes ggml/examples/yolo/data/labels/92_2.png | Bin 0 -> 438 bytes ggml/examples/yolo/data/labels/92_3.png | Bin 0 -> 509 bytes ggml/examples/yolo/data/labels/92_4.png | Bin 0 -> 567 bytes ggml/examples/yolo/data/labels/92_5.png | Bin 0 -> 612 bytes ggml/examples/yolo/data/labels/92_6.png | Bin 0 -> 653 bytes ggml/examples/yolo/data/labels/92_7.png | Bin 0 -> 741 bytes ggml/examples/yolo/data/labels/93_0.png | Bin 0 -> 292 bytes ggml/examples/yolo/data/labels/93_1.png | Bin 0 -> 308 bytes ggml/examples/yolo/data/labels/93_2.png | Bin 0 -> 308 bytes ggml/examples/yolo/data/labels/93_3.png | Bin 0 -> 315 bytes ggml/examples/yolo/data/labels/93_4.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/93_5.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/93_6.png | Bin 0 -> 324 bytes ggml/examples/yolo/data/labels/93_7.png | Bin 0 -> 330 bytes ggml/examples/yolo/data/labels/94_0.png | Bin 0 -> 307 bytes ggml/examples/yolo/data/labels/94_1.png | Bin 0 -> 345 bytes ggml/examples/yolo/data/labels/94_2.png | Bin 0 -> 390 bytes ggml/examples/yolo/data/labels/94_3.png | Bin 0 -> 429 bytes ggml/examples/yolo/data/labels/94_4.png | Bin 0 -> 493 bytes ggml/examples/yolo/data/labels/94_5.png | Bin 0 -> 510 bytes ggml/examples/yolo/data/labels/94_6.png | Bin 0 -> 574 bytes ggml/examples/yolo/data/labels/94_7.png | Bin 0 -> 594 bytes ggml/examples/yolo/data/labels/95_0.png | Bin 0 -> 288 bytes ggml/examples/yolo/data/labels/95_1.png | Bin 0 -> 295 bytes ggml/examples/yolo/data/labels/95_2.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/95_3.png | Bin 0 -> 305 bytes ggml/examples/yolo/data/labels/95_4.png | Bin 0 -> 309 bytes ggml/examples/yolo/data/labels/95_5.png | Bin 0 -> 314 bytes ggml/examples/yolo/data/labels/95_6.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/95_7.png | Bin 0 -> 321 bytes ggml/examples/yolo/data/labels/96_0.png | Bin 0 -> 300 bytes ggml/examples/yolo/data/labels/96_1.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/96_2.png | Bin 0 -> 340 bytes ggml/examples/yolo/data/labels/96_3.png | Bin 0 -> 368 bytes ggml/examples/yolo/data/labels/96_4.png | Bin 0 -> 388 bytes ggml/examples/yolo/data/labels/96_5.png | Bin 0 -> 418 bytes ggml/examples/yolo/data/labels/96_6.png | Bin 0 -> 444 bytes ggml/examples/yolo/data/labels/96_7.png | Bin 0 -> 470 bytes ggml/examples/yolo/data/labels/97_0.png | Bin 0 -> 317 bytes ggml/examples/yolo/data/labels/97_1.png | Bin 0 -> 374 bytes ggml/examples/yolo/data/labels/97_2.png | Bin 0 -> 448 bytes ggml/examples/yolo/data/labels/97_3.png | Bin 0 -> 490 bytes ggml/examples/yolo/data/labels/97_4.png | Bin 0 -> 562 bytes ggml/examples/yolo/data/labels/97_5.png | Bin 0 -> 637 bytes ggml/examples/yolo/data/labels/97_6.png | Bin 0 -> 697 bytes ggml/examples/yolo/data/labels/97_7.png | Bin 0 -> 752 bytes ggml/examples/yolo/data/labels/98_0.png | Bin 0 -> 319 bytes ggml/examples/yolo/data/labels/98_1.png | Bin 0 -> 377 bytes ggml/examples/yolo/data/labels/98_2.png | Bin 0 -> 454 bytes ggml/examples/yolo/data/labels/98_3.png | Bin 0 -> 509 bytes ggml/examples/yolo/data/labels/98_4.png | Bin 0 -> 568 bytes ggml/examples/yolo/data/labels/98_5.png | Bin 0 -> 635 bytes ggml/examples/yolo/data/labels/98_6.png | Bin 0 -> 700 bytes ggml/examples/yolo/data/labels/98_7.png | Bin 0 -> 754 bytes ggml/examples/yolo/data/labels/99_0.png | Bin 0 -> 313 bytes ggml/examples/yolo/data/labels/99_1.png | Bin 0 -> 367 bytes ggml/examples/yolo/data/labels/99_2.png | Bin 0 -> 423 bytes ggml/examples/yolo/data/labels/99_3.png | Bin 0 -> 464 bytes ggml/examples/yolo/data/labels/99_4.png | Bin 0 -> 521 bytes ggml/examples/yolo/data/labels/99_5.png | Bin 0 -> 576 bytes ggml/examples/yolo/data/labels/99_6.png | Bin 0 -> 629 bytes ggml/examples/yolo/data/labels/99_7.png | Bin 0 -> 692 bytes ggml/examples/yolo/yolo-image.cpp | 210 + ggml/examples/yolo/yolo-image.h | 39 + ggml/examples/yolo/yolov3-tiny.cpp | 661 + ggml/ggml.pc.in | 10 + ggml/include/ggml-alloc.h | 86 + ggml/include/ggml-backend.h | 435 + ggml/include/ggml-blas.h | 25 + ggml/include/ggml-cann.h | 123 + ggml/include/ggml-cpp.h | 39 + ggml/include/ggml-cpu.h | 151 + ggml/include/ggml-cuda.h | 50 + ggml/include/ggml-hexagon.h | 19 + ggml/include/ggml-metal.h | 61 + ggml/include/ggml-opencl.h | 26 + ggml/include/ggml-openvino.h | 37 + ggml/include/ggml-opt.h | 256 + ggml/include/ggml-rpc.h | 35 + ggml/include/ggml-sycl.h | 49 + ggml/include/ggml-virtgpu.h | 14 + ggml/include/ggml-vulkan.h | 29 + ggml/include/ggml-webgpu.h | 19 + ggml/include/ggml-zdnn.h | 17 + ggml/include/ggml-zendnn.h | 22 + ggml/include/ggml.h | 2863 +++ ggml/include/gguf.h | 210 + ggml/requirements.txt | 12 + ggml/scripts/gen-authors.sh | 9 + ggml/scripts/release.sh | 296 + ggml/scripts/sync-llama-am.sh | 167 + ggml/scripts/sync-llama.last | 1 + ggml/scripts/sync-llama.sh | 21 + ggml/scripts/sync-whisper-am.sh | 138 + ggml/scripts/sync-whisper.last | 1 + ggml/scripts/sync-whisper.sh | 17 + ggml/src/CMakeLists.txt | 517 + ggml/src/ggml-alloc.c | 1248 + ggml/src/ggml-backend-dl.cpp | 48 + ggml/src/ggml-backend-dl.h | 45 + ggml/src/ggml-backend-impl.h | 275 + ggml/src/ggml-backend-meta.cpp | 2263 ++ ggml/src/ggml-backend-reg.cpp | 586 + ggml/src/ggml-backend.cpp | 2371 ++ ggml/src/ggml-blas/CMakeLists.txt | 101 + ggml/src/ggml-blas/ggml-blas.cpp | 522 + ggml/src/ggml-cann/CMakeLists.txt | 89 + ggml/src/ggml-cann/acl_tensor.cpp | 195 + ggml/src/ggml-cann/acl_tensor.h | 349 + ggml/src/ggml-cann/aclnn_ops.cpp | 4436 ++++ ggml/src/ggml-cann/aclnn_ops.h | 1190 + ggml/src/ggml-cann/common.h | 651 + ggml/src/ggml-cann/ggml-cann.cpp | 3062 +++ ggml/src/ggml-common.h | 1900 ++ ggml/src/ggml-cpu/CMakeLists.txt | 723 + ggml/src/ggml-cpu/amx/amx.cpp | 249 + ggml/src/ggml-cpu/amx/amx.h | 8 + ggml/src/ggml-cpu/amx/common.h | 115 + ggml/src/ggml-cpu/amx/mmq.cpp | 2512 ++ ggml/src/ggml-cpu/amx/mmq.h | 10 + ggml/src/ggml-cpu/arch-fallback.h | 347 + ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp | 98 + ggml/src/ggml-cpu/arch/arm/quants.c | 4245 ++++ ggml/src/ggml-cpu/arch/arm/repack.cpp | 5156 +++++ ggml/src/ggml-cpu/arch/loongarch/quants.c | 2309 ++ ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp | 82 + ggml/src/ggml-cpu/arch/powerpc/quants.c | 2304 ++ ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp | 38 + ggml/src/ggml-cpu/arch/riscv/quants.c | 6596 ++++++ ggml/src/ggml-cpu/arch/riscv/repack.cpp | 1703 ++ ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp | 50 + ggml/src/ggml-cpu/arch/s390/quants.c | 1465 ++ ggml/src/ggml-cpu/arch/wasm/quants.c | 1292 ++ ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp | 327 + ggml/src/ggml-cpu/arch/x86/quants.c | 3970 ++++ ggml/src/ggml-cpu/arch/x86/repack.cpp | 6407 ++++++ ggml/src/ggml-cpu/binary-ops.cpp | 154 + ggml/src/ggml-cpu/binary-ops.h | 16 + ggml/src/ggml-cpu/cmake/FindSIMD.cmake | 100 + ggml/src/ggml-cpu/cmake/FindSMTIME.cmake | 32 + ggml/src/ggml-cpu/common.h | 95 + ggml/src/ggml-cpu/ggml-cpu-impl.h | 539 + ggml/src/ggml-cpu/ggml-cpu.c | 3887 ++++ ggml/src/ggml-cpu/ggml-cpu.cpp | 703 + ggml/src/ggml-cpu/hbm.cpp | 55 + ggml/src/ggml-cpu/hbm.h | 8 + ggml/src/ggml-cpu/kleidiai/kernels.cpp | 939 + ggml/src/ggml-cpu/kleidiai/kernels.h | 90 + ggml/src/ggml-cpu/kleidiai/kleidiai.cpp | 1523 ++ ggml/src/ggml-cpu/kleidiai/kleidiai.h | 17 + ggml/src/ggml-cpu/llamafile/sgemm.cpp | 4052 ++++ ggml/src/ggml-cpu/llamafile/sgemm.h | 25 + ggml/src/ggml-cpu/ops.cpp | 11540 ++++++++++ ggml/src/ggml-cpu/ops.h | 125 + ggml/src/ggml-cpu/quants.c | 1288 ++ ggml/src/ggml-cpu/quants.h | 103 + ggml/src/ggml-cpu/repack.cpp | 4836 ++++ ggml/src/ggml-cpu/repack.h | 245 + ggml/src/ggml-cpu/simd-gemm.h | 226 + ggml/src/ggml-cpu/simd-mappings.h | 1306 ++ ggml/src/ggml-cpu/spacemit/ime.cpp | 1740 ++ ggml/src/ggml-cpu/spacemit/ime.h | 21 + ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp | 1027 + ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp | 5768 +++++ ggml/src/ggml-cpu/spacemit/ime_env.cpp | 320 + ggml/src/ggml-cpu/spacemit/ime_env.h | 55 + ggml/src/ggml-cpu/spacemit/ime_kernels.h | 189 + ggml/src/ggml-cpu/spacemit/repack.cpp | 1795 ++ ggml/src/ggml-cpu/spacemit/repack.h | 14 + ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp | 3178 +++ ggml/src/ggml-cpu/spacemit/rvv_kernels.h | 95 + ggml/src/ggml-cpu/spacemit/spine_barrier.h | 34 + ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp | 760 + ggml/src/ggml-cpu/spacemit/spine_mem_pool.h | 32 + ggml/src/ggml-cpu/spacemit/spine_tcm.h | 409 + ggml/src/ggml-cpu/traits.cpp | 36 + ggml/src/ggml-cpu/traits.h | 38 + ggml/src/ggml-cpu/unary-ops.cpp | 337 + ggml/src/ggml-cpu/unary-ops.h | 35 + ggml/src/ggml-cpu/vec.cpp | 613 + ggml/src/ggml-cpu/vec.h | 1570 ++ ggml/src/ggml-cuda/CMakeLists.txt | 269 + ggml/src/ggml-cuda/acc.cu | 61 + ggml/src/ggml-cuda/acc.cuh | 5 + ggml/src/ggml-cuda/add-id.cu | 58 + ggml/src/ggml-cuda/add-id.cuh | 3 + ggml/src/ggml-cuda/allreduce.cu | 971 + ggml/src/ggml-cuda/allreduce.cuh | 29 + ggml/src/ggml-cuda/arange.cu | 34 + ggml/src/ggml-cuda/arange.cuh | 5 + ggml/src/ggml-cuda/argmax.cu | 91 + ggml/src/ggml-cuda/argmax.cuh | 3 + ggml/src/ggml-cuda/argsort.cu | 266 + ggml/src/ggml-cuda/argsort.cuh | 19 + ggml/src/ggml-cuda/binbcast.cu | 530 + ggml/src/ggml-cuda/binbcast.cuh | 12 + ggml/src/ggml-cuda/clamp.cu | 45 + ggml/src/ggml-cuda/clamp.cuh | 5 + ggml/src/ggml-cuda/col2im-1d.cu | 81 + ggml/src/ggml-cuda/col2im-1d.cuh | 3 + ggml/src/ggml-cuda/common.cuh | 1641 ++ ggml/src/ggml-cuda/concat.cu | 227 + ggml/src/ggml-cuda/concat.cuh | 5 + ggml/src/ggml-cuda/conv-transpose-1d.cu | 86 + ggml/src/ggml-cuda/conv-transpose-1d.cuh | 5 + ggml/src/ggml-cuda/conv2d-dw.cu | 161 + ggml/src/ggml-cuda/conv2d-dw.cuh | 5 + ggml/src/ggml-cuda/conv2d-transpose.cu | 115 + ggml/src/ggml-cuda/conv2d-transpose.cuh | 5 + ggml/src/ggml-cuda/conv2d.cu | 166 + ggml/src/ggml-cuda/conv2d.cuh | 5 + ggml/src/ggml-cuda/convert.cu | 892 + ggml/src/ggml-cuda/convert.cuh | 66 + ggml/src/ggml-cuda/count-equal.cu | 64 + ggml/src/ggml-cuda/count-equal.cuh | 5 + ggml/src/ggml-cuda/cp-async.cuh | 57 + ggml/src/ggml-cuda/cpy-utils.cuh | 217 + ggml/src/ggml-cuda/cpy.cu | 566 + ggml/src/ggml-cuda/cpy.cuh | 7 + ggml/src/ggml-cuda/cross-entropy-loss.cu | 177 + ggml/src/ggml-cuda/cross-entropy-loss.cuh | 7 + ggml/src/ggml-cuda/cumsum.cu | 307 + ggml/src/ggml-cuda/cumsum.cuh | 5 + ggml/src/ggml-cuda/dequantize.cuh | 99 + ggml/src/ggml-cuda/diag.cu | 77 + ggml/src/ggml-cuda/diag.cuh | 5 + ggml/src/ggml-cuda/diagmask.cu | 40 + ggml/src/ggml-cuda/diagmask.cuh | 5 + ggml/src/ggml-cuda/fattn-common.cuh | 1270 ++ ggml/src/ggml-cuda/fattn-mma-f16.cuh | 2029 ++ ggml/src/ggml-cuda/fattn-tile.cu | 61 + ggml/src/ggml-cuda/fattn-tile.cuh | 1357 ++ ggml/src/ggml-cuda/fattn-vec.cuh | 611 + ggml/src/ggml-cuda/fattn-wmma-f16.cu | 705 + ggml/src/ggml-cuda/fattn-wmma-f16.cuh | 51 + ggml/src/ggml-cuda/fattn.cu | 597 + ggml/src/ggml-cuda/fattn.cuh | 7 + ggml/src/ggml-cuda/fill.cu | 37 + ggml/src/ggml-cuda/fill.cuh | 3 + ggml/src/ggml-cuda/fwht.cu | 101 + ggml/src/ggml-cuda/fwht.cuh | 4 + ggml/src/ggml-cuda/gated_delta_net.cu | 312 + ggml/src/ggml-cuda/gated_delta_net.cuh | 4 + ggml/src/ggml-cuda/getrows.cu | 308 + ggml/src/ggml-cuda/getrows.cuh | 15 + ggml/src/ggml-cuda/ggml-cuda.cu | 5720 +++++ ggml/src/ggml-cuda/gla.cu | 93 + ggml/src/ggml-cuda/gla.cuh | 3 + ggml/src/ggml-cuda/im2col.cu | 267 + ggml/src/ggml-cuda/im2col.cuh | 6 + ggml/src/ggml-cuda/mean.cu | 77 + ggml/src/ggml-cuda/mean.cuh | 3 + ggml/src/ggml-cuda/mma.cuh | 1456 ++ ggml/src/ggml-cuda/mmf.cu | 191 + ggml/src/ggml-cuda/mmf.cuh | 908 + ggml/src/ggml-cuda/mmid.cu | 164 + ggml/src/ggml-cuda/mmid.cuh | 5 + ggml/src/ggml-cuda/mmq.cu | 372 + ggml/src/ggml-cuda/mmq.cuh | 4176 ++++ ggml/src/ggml-cuda/mmvf.cu | 869 + ggml/src/ggml-cuda/mmvf.cuh | 14 + ggml/src/ggml-cuda/mmvq.cu | 1252 + ggml/src/ggml-cuda/mmvq.cuh | 18 + ggml/src/ggml-cuda/norm.cu | 698 + ggml/src/ggml-cuda/norm.cuh | 18 + ggml/src/ggml-cuda/opt-step-adamw.cu | 78 + ggml/src/ggml-cuda/opt-step-adamw.cuh | 5 + ggml/src/ggml-cuda/opt-step-sgd.cu | 49 + ggml/src/ggml-cuda/opt-step-sgd.cuh | 5 + ggml/src/ggml-cuda/out-prod.cu | 84 + ggml/src/ggml-cuda/out-prod.cuh | 3 + ggml/src/ggml-cuda/pad.cu | 106 + ggml/src/ggml-cuda/pad.cuh | 5 + ggml/src/ggml-cuda/pad_reflect_1d.cu | 91 + ggml/src/ggml-cuda/pad_reflect_1d.cuh | 5 + ggml/src/ggml-cuda/pool2d.cu | 94 + ggml/src/ggml-cuda/pool2d.cuh | 5 + ggml/src/ggml-cuda/quantize.cu | 450 + ggml/src/ggml-cuda/quantize.cuh | 41 + ggml/src/ggml-cuda/reduce_rows.cuh | 43 + ggml/src/ggml-cuda/roll.cu | 67 + ggml/src/ggml-cuda/roll.cuh | 5 + ggml/src/ggml-cuda/rope.cu | 672 + ggml/src/ggml-cuda/rope.cuh | 9 + ggml/src/ggml-cuda/scale.cu | 37 + ggml/src/ggml-cuda/scale.cuh | 5 + ggml/src/ggml-cuda/set-rows.cu | 338 + ggml/src/ggml-cuda/set-rows.cuh | 7 + ggml/src/ggml-cuda/set.cu | 39 + ggml/src/ggml-cuda/set.cuh | 7 + ggml/src/ggml-cuda/snake.cu | 72 + ggml/src/ggml-cuda/snake.cuh | 8 + ggml/src/ggml-cuda/softcap.cu | 37 + ggml/src/ggml-cuda/softcap.cuh | 5 + ggml/src/ggml-cuda/softmax.cu | 472 + ggml/src/ggml-cuda/softmax.cuh | 7 + ggml/src/ggml-cuda/solve_tri.cu | 275 + ggml/src/ggml-cuda/solve_tri.cuh | 3 + ggml/src/ggml-cuda/ssm-conv.cu | 206 + ggml/src/ggml-cuda/ssm-conv.cuh | 3 + ggml/src/ggml-cuda/ssm-scan.cu | 364 + ggml/src/ggml-cuda/ssm-scan.cuh | 3 + ggml/src/ggml-cuda/sum.cu | 41 + ggml/src/ggml-cuda/sum.cuh | 5 + ggml/src/ggml-cuda/sumrows.cu | 47 + ggml/src/ggml-cuda/sumrows.cuh | 4 + ...ttn-mma-f16-instance-ncols1_1-ncols2_16.cu | 6 + ...ttn-mma-f16-instance-ncols1_1-ncols2_32.cu | 6 + ...attn-mma-f16-instance-ncols1_1-ncols2_8.cu | 12 + ...ttn-mma-f16-instance-ncols1_16-ncols2_1.cu | 10 + ...ttn-mma-f16-instance-ncols1_16-ncols2_2.cu | 10 + ...ttn-mma-f16-instance-ncols1_16-ncols2_4.cu | 12 + ...ttn-mma-f16-instance-ncols1_2-ncols2_16.cu | 6 + ...ttn-mma-f16-instance-ncols1_2-ncols2_32.cu | 6 + ...attn-mma-f16-instance-ncols1_2-ncols2_4.cu | 12 + ...attn-mma-f16-instance-ncols1_2-ncols2_8.cu | 12 + ...ttn-mma-f16-instance-ncols1_32-ncols2_1.cu | 10 + ...ttn-mma-f16-instance-ncols1_32-ncols2_2.cu | 10 + ...ttn-mma-f16-instance-ncols1_4-ncols2_16.cu | 6 + ...attn-mma-f16-instance-ncols1_4-ncols2_2.cu | 10 + ...attn-mma-f16-instance-ncols1_4-ncols2_4.cu | 12 + ...attn-mma-f16-instance-ncols1_4-ncols2_8.cu | 12 + ...ttn-mma-f16-instance-ncols1_64-ncols2_1.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_1.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_2.cu | 10 + ...attn-mma-f16-instance-ncols1_8-ncols2_4.cu | 12 + ...attn-mma-f16-instance-ncols1_8-ncols2_8.cu | 12 + .../fattn-tile-instance-dkq112-dv112.cu | 5 + .../fattn-tile-instance-dkq128-dv128.cu | 5 + .../fattn-tile-instance-dkq192-dv128.cu | 5 + .../fattn-tile-instance-dkq256-dv256.cu | 5 + .../fattn-tile-instance-dkq320-dv256.cu | 5 + .../fattn-tile-instance-dkq40-dv40.cu | 5 + .../fattn-tile-instance-dkq512-dv512.cu | 5 + .../fattn-tile-instance-dkq576-dv512.cu | 5 + .../fattn-tile-instance-dkq64-dv64.cu | 5 + .../fattn-tile-instance-dkq72-dv72.cu | 5 + .../fattn-tile-instance-dkq80-dv80.cu | 5 + .../fattn-tile-instance-dkq96-dv96.cu | 5 + .../fattn-vec-instance-bf16-bf16.cu | 7 + .../fattn-vec-instance-bf16-f16.cu | 7 + .../fattn-vec-instance-bf16-q4_0.cu | 7 + .../fattn-vec-instance-bf16-q4_1.cu | 7 + .../fattn-vec-instance-bf16-q5_0.cu | 7 + .../fattn-vec-instance-bf16-q5_1.cu | 7 + .../fattn-vec-instance-bf16-q8_0.cu | 7 + .../fattn-vec-instance-f16-bf16.cu | 7 + .../fattn-vec-instance-f16-f16.cu | 7 + .../fattn-vec-instance-f16-q4_0.cu | 7 + .../fattn-vec-instance-f16-q4_1.cu | 7 + .../fattn-vec-instance-f16-q5_0.cu | 7 + .../fattn-vec-instance-f16-q5_1.cu | 7 + .../fattn-vec-instance-f16-q8_0.cu | 7 + .../fattn-vec-instance-q4_0-bf16.cu | 7 + .../fattn-vec-instance-q4_0-f16.cu | 7 + .../fattn-vec-instance-q4_0-q4_0.cu | 7 + .../fattn-vec-instance-q4_0-q4_1.cu | 7 + .../fattn-vec-instance-q4_0-q5_0.cu | 7 + .../fattn-vec-instance-q4_0-q5_1.cu | 7 + .../fattn-vec-instance-q4_0-q8_0.cu | 7 + .../fattn-vec-instance-q4_1-bf16.cu | 7 + .../fattn-vec-instance-q4_1-f16.cu | 7 + .../fattn-vec-instance-q4_1-q4_0.cu | 7 + .../fattn-vec-instance-q4_1-q4_1.cu | 7 + .../fattn-vec-instance-q4_1-q5_0.cu | 7 + .../fattn-vec-instance-q4_1-q5_1.cu | 7 + .../fattn-vec-instance-q4_1-q8_0.cu | 7 + .../fattn-vec-instance-q5_0-bf16.cu | 7 + .../fattn-vec-instance-q5_0-f16.cu | 7 + .../fattn-vec-instance-q5_0-q4_0.cu | 7 + .../fattn-vec-instance-q5_0-q4_1.cu | 7 + .../fattn-vec-instance-q5_0-q5_0.cu | 7 + .../fattn-vec-instance-q5_0-q5_1.cu | 7 + .../fattn-vec-instance-q5_0-q8_0.cu | 7 + .../fattn-vec-instance-q5_1-bf16.cu | 7 + .../fattn-vec-instance-q5_1-f16.cu | 7 + .../fattn-vec-instance-q5_1-q4_0.cu | 7 + .../fattn-vec-instance-q5_1-q4_1.cu | 7 + .../fattn-vec-instance-q5_1-q5_0.cu | 7 + .../fattn-vec-instance-q5_1-q5_1.cu | 7 + .../fattn-vec-instance-q5_1-q8_0.cu | 7 + .../fattn-vec-instance-q8_0-bf16.cu | 7 + .../fattn-vec-instance-q8_0-f16.cu | 7 + .../fattn-vec-instance-q8_0-q4_0.cu | 7 + .../fattn-vec-instance-q8_0-q4_1.cu | 7 + .../fattn-vec-instance-q8_0-q5_0.cu | 7 + .../fattn-vec-instance-q8_0-q5_1.cu | 7 + .../fattn-vec-instance-q8_0-q8_0.cu | 7 + .../template-instances/generate_cu_files.py | 110 + .../mmf-instance-ncols_1.cu | 5 + .../mmf-instance-ncols_10.cu | 5 + .../mmf-instance-ncols_11.cu | 5 + .../mmf-instance-ncols_12.cu | 5 + .../mmf-instance-ncols_13.cu | 5 + .../mmf-instance-ncols_14.cu | 5 + .../mmf-instance-ncols_15.cu | 5 + .../mmf-instance-ncols_16.cu | 5 + .../mmf-instance-ncols_2.cu | 5 + .../mmf-instance-ncols_3.cu | 5 + .../mmf-instance-ncols_4.cu | 5 + .../mmf-instance-ncols_5.cu | 5 + .../mmf-instance-ncols_6.cu | 5 + .../mmf-instance-ncols_7.cu | 5 + .../mmf-instance-ncols_8.cu | 5 + .../mmf-instance-ncols_9.cu | 5 + .../template-instances/mmq-instance-iq1_s.cu | 5 + .../template-instances/mmq-instance-iq2_s.cu | 5 + .../template-instances/mmq-instance-iq2_xs.cu | 5 + .../mmq-instance-iq2_xxs.cu | 5 + .../template-instances/mmq-instance-iq3_s.cu | 5 + .../mmq-instance-iq3_xxs.cu | 5 + .../template-instances/mmq-instance-iq4_nl.cu | 5 + .../template-instances/mmq-instance-iq4_xs.cu | 5 + .../template-instances/mmq-instance-mxfp4.cu | 5 + .../template-instances/mmq-instance-nvfp4.cu | 5 + .../template-instances/mmq-instance-q1_0.cu | 5 + .../template-instances/mmq-instance-q2_k.cu | 5 + .../template-instances/mmq-instance-q3_k.cu | 5 + .../template-instances/mmq-instance-q4_0.cu | 5 + .../template-instances/mmq-instance-q4_1.cu | 5 + .../template-instances/mmq-instance-q4_k.cu | 5 + .../template-instances/mmq-instance-q5_0.cu | 5 + .../template-instances/mmq-instance-q5_1.cu | 5 + .../template-instances/mmq-instance-q5_k.cu | 5 + .../template-instances/mmq-instance-q6_k.cu | 5 + .../template-instances/mmq-instance-q8_0.cu | 5 + ggml/src/ggml-cuda/top-k.cu | 96 + ggml/src/ggml-cuda/top-k.cuh | 3 + ggml/src/ggml-cuda/topk-moe.cu | 418 + ggml/src/ggml-cuda/topk-moe.cuh | 27 + ggml/src/ggml-cuda/tri.cu | 136 + ggml/src/ggml-cuda/tri.cuh | 5 + ggml/src/ggml-cuda/tsembd.cu | 47 + ggml/src/ggml-cuda/tsembd.cuh | 5 + ggml/src/ggml-cuda/unary.cu | 646 + ggml/src/ggml-cuda/unary.cuh | 114 + ggml/src/ggml-cuda/upscale.cu | 293 + ggml/src/ggml-cuda/upscale.cuh | 5 + ggml/src/ggml-cuda/vecdotq.cuh | 1317 ++ ggml/src/ggml-cuda/vendors/cuda.h | 28 + ggml/src/ggml-cuda/vendors/hip.h | 304 + ggml/src/ggml-cuda/vendors/musa.h | 150 + ggml/src/ggml-cuda/wkv.cu | 199 + ggml/src/ggml-cuda/wkv.cuh | 7 + ggml/src/ggml-hexagon/CMakeLists.txt | 118 + ggml/src/ggml-hexagon/ggml-hexagon.cpp | 4102 ++++ ggml/src/ggml-hexagon/htp-drv.cpp | 418 + ggml/src/ggml-hexagon/htp-drv.h | 121 + ggml/src/ggml-hexagon/htp-opnode.h | 272 + ggml/src/ggml-hexagon/htp/CMakeLists.txt | 84 + ggml/src/ggml-hexagon/htp/act-ops.c | 782 + ggml/src/ggml-hexagon/htp/argsort-ops.c | 294 + ggml/src/ggml-hexagon/htp/binary-ops.c | 872 + .../ggml-hexagon/htp/cmake-toolchain.cmake | 157 + ggml/src/ggml-hexagon/htp/concat-ops.c | 277 + ggml/src/ggml-hexagon/htp/cpy-ops.c | 295 + ggml/src/ggml-hexagon/htp/cumsum-ops.c | 270 + ggml/src/ggml-hexagon/htp/diag-ops.c | 216 + ggml/src/ggml-hexagon/htp/fill-ops.c | 123 + ggml/src/ggml-hexagon/htp/flash-attn-ops.c | 745 + .../ggml-hexagon/htp/gated-delta-net-ops.c | 1148 + ggml/src/ggml-hexagon/htp/get-rows-ops.c | 218 + ggml/src/ggml-hexagon/htp/hex-dma.c | 63 + ggml/src/ggml-hexagon/htp/hex-dma.h | 379 + ggml/src/ggml-hexagon/htp/hex-dump.h | 86 + ggml/src/ggml-hexagon/htp/hex-fastdiv.h | 37 + ggml/src/ggml-hexagon/htp/hex-profile.h | 64 + ggml/src/ggml-hexagon/htp/hex-utils.h | 110 + .../src/ggml-hexagon/htp/hmx-flash-attn-ops.c | 1838 ++ ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 2080 ++ ggml/src/ggml-hexagon/htp/hmx-ops.c | 6 + ggml/src/ggml-hexagon/htp/hmx-ops.h | 88 + ggml/src/ggml-hexagon/htp/hmx-queue.c | 160 + ggml/src/ggml-hexagon/htp/hmx-queue.h | 136 + ggml/src/ggml-hexagon/htp/hmx-utils.h | 200 + ggml/src/ggml-hexagon/htp/htp-ctx.h | 120 + ggml/src/ggml-hexagon/htp/htp-ops.h | 215 + ggml/src/ggml-hexagon/htp/htp_iface.idl | 22 + ggml/src/ggml-hexagon/htp/hvx-arith.h | 443 + ggml/src/ggml-hexagon/htp/hvx-base.h | 308 + ggml/src/ggml-hexagon/htp/hvx-copy.h | 262 + ggml/src/ggml-hexagon/htp/hvx-div.h | 291 + ggml/src/ggml-hexagon/htp/hvx-dump.h | 129 + ggml/src/ggml-hexagon/htp/hvx-exp.h | 216 + ggml/src/ggml-hexagon/htp/hvx-flash-attn.h | 47 + ggml/src/ggml-hexagon/htp/hvx-floor.h | 100 + ggml/src/ggml-hexagon/htp/hvx-inverse.h | 210 + ggml/src/ggml-hexagon/htp/hvx-log.h | 65 + ggml/src/ggml-hexagon/htp/hvx-pow.h | 42 + ggml/src/ggml-hexagon/htp/hvx-reduce.h | 296 + ggml/src/ggml-hexagon/htp/hvx-repl.h | 74 + ggml/src/ggml-hexagon/htp/hvx-scale.h | 133 + ggml/src/ggml-hexagon/htp/hvx-sigmoid.h | 142 + ggml/src/ggml-hexagon/htp/hvx-sin-cos.h | 90 + ggml/src/ggml-hexagon/htp/hvx-sqrt.h | 126 + ggml/src/ggml-hexagon/htp/hvx-types.h | 36 + ggml/src/ggml-hexagon/htp/hvx-utils.h | 23 + ggml/src/ggml-hexagon/htp/main.c | 945 + ggml/src/ggml-hexagon/htp/matmul-ops.c | 5004 ++++ ggml/src/ggml-hexagon/htp/pad-ops.c | 547 + ggml/src/ggml-hexagon/htp/repeat-ops.c | 148 + ggml/src/ggml-hexagon/htp/rope-ops.c | 693 + ggml/src/ggml-hexagon/htp/set-rows-ops.c | 190 + ggml/src/ggml-hexagon/htp/softmax-ops.c | 407 + ggml/src/ggml-hexagon/htp/solve-tri-ops.c | 267 + ggml/src/ggml-hexagon/htp/ssm-conv.c | 432 + ggml/src/ggml-hexagon/htp/sum-rows-ops.c | 128 + ggml/src/ggml-hexagon/htp/unary-ops.c | 1056 + ggml/src/ggml-hexagon/htp/vtcm-utils.h | 16 + ggml/src/ggml-hexagon/htp/worker-pool.c | 293 + ggml/src/ggml-hexagon/htp/worker-pool.h | 57 + ggml/src/ggml-hexagon/libdl.h | 79 + ggml/src/ggml-hexagon/libggml-htp.inf | 40 + ggml/src/ggml-hip/CMakeLists.txt | 157 + ggml/src/ggml-impl.h | 783 + ggml/src/ggml-metal/CMakeLists.txt | 124 + ggml/src/ggml-metal/ggml-metal-common.cpp | 457 + ggml/src/ggml-metal/ggml-metal-common.h | 52 + ggml/src/ggml-metal/ggml-metal-context.h | 41 + ggml/src/ggml-metal/ggml-metal-context.m | 739 + ggml/src/ggml-metal/ggml-metal-device.cpp | 2123 ++ ggml/src/ggml-metal/ggml-metal-device.h | 325 + ggml/src/ggml-metal/ggml-metal-device.m | 1905 ++ ggml/src/ggml-metal/ggml-metal-impl.h | 1190 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 4766 ++++ ggml/src/ggml-metal/ggml-metal-ops.h | 99 + ggml/src/ggml-metal/ggml-metal.cpp | 950 + ggml/src/ggml-metal/ggml-metal.metal | 10822 +++++++++ ggml/src/ggml-musa/CMakeLists.txt | 124 + ggml/src/ggml-musa/mudnn.cu | 112 + ggml/src/ggml-musa/mudnn.cuh | 12 + ggml/src/ggml-opencl/CMakeLists.txt | 206 + ggml/src/ggml-opencl/ggml-opencl.cpp | 18931 ++++++++++++++++ ggml/src/ggml-opencl/kernels/add.cl | 190 + ggml/src/ggml-opencl/kernels/add_id.cl | 42 + ggml/src/ggml-opencl/kernels/argsort.cl | 86 + ggml/src/ggml-opencl/kernels/clamp.cl | 20 + ggml/src/ggml-opencl/kernels/concat.cl | 118 + ggml/src/ggml-opencl/kernels/conv2d.cl | 185 + .../src/ggml-opencl/kernels/conv2d_f16_f32.cl | 176 + ggml/src/ggml-opencl/kernels/cpy.cl | 288 + ggml/src/ggml-opencl/kernels/cumsum.cl | 139 + ggml/src/ggml-opencl/kernels/cvt.cl | 2176 ++ ggml/src/ggml-opencl/kernels/diag.cl | 27 + ggml/src/ggml-opencl/kernels/diag_mask_inf.cl | 58 + ggml/src/ggml-opencl/kernels/div.cl | 138 + ggml/src/ggml-opencl/kernels/embed_kernel.py | 26 + ggml/src/ggml-opencl/kernels/exp.cl | 125 + ggml/src/ggml-opencl/kernels/expm1.cl | 113 + ggml/src/ggml-opencl/kernels/fill.cl | 17 + .../src/ggml-opencl/kernels/flash_attn_f16.cl | 370 + .../src/ggml-opencl/kernels/flash_attn_f32.cl | 371 + .../ggml-opencl/kernels/flash_attn_f32_f16.cl | 373 + .../ggml-opencl/kernels/gated_delta_net.cl | 249 + ggml/src/ggml-opencl/kernels/gelu.cl | 89 + .../ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl | 162 + .../kernels/gemm_moe_mxfp4_f32_ns.cl | 306 + .../kernels/gemm_moe_q4_0_f32_ns.cl | 256 + .../kernels/gemm_moe_q4_1_f32_ns.cl | 258 + .../kernels/gemm_moe_q4_k_f32_ns.cl | 283 + .../kernels/gemm_moe_q5_0_f32_ns.cl | 260 + .../kernels/gemm_moe_q5_1_f32_ns.cl | 262 + .../kernels/gemm_moe_q5_k_f32_ns.cl | 288 + .../kernels/gemm_moe_q6_k_f32_ns.cl | 267 + .../kernels/gemm_noshuffle_iq4_nl_f32.cl | 150 + .../kernels/gemm_noshuffle_q4_0_f32.cl | 139 + .../kernels/gemm_noshuffle_q4_1_f32.cl | 132 + .../kernels/gemm_noshuffle_q4_k_f32.cl | 172 + .../kernels/gemm_noshuffle_q5_0_f32.cl | 131 + .../kernels/gemm_noshuffle_q5_1_f32.cl | 134 + .../kernels/gemm_noshuffle_q5_k_f32.cl | 176 + .../kernels/gemm_noshuffle_q6_k_f32.cl | 140 + .../kernels/gemm_noshuffle_q8_0_f32.cl | 129 + .../kernels/gemm_xmem_f16_f32_os8.cl | 233 + .../ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl | 156 + .../kernels/gemv_moe_mxfp4_f32_ns.cl | 165 + .../kernels/gemv_moe_q4_0_f32_ns.cl | 120 + .../kernels/gemv_moe_q4_1_f32_ns.cl | 123 + .../kernels/gemv_moe_q4_k_f32_ns.cl | 155 + .../kernels/gemv_moe_q5_0_f32_ns.cl | 123 + .../kernels/gemv_moe_q5_1_f32_ns.cl | 125 + .../kernels/gemv_moe_q5_k_f32_ns.cl | 160 + .../kernels/gemv_moe_q6_k_f32_ns.cl | 141 + .../kernels/gemv_noshuffle_iq4_nl_f32.cl | 302 + .../kernels/gemv_noshuffle_q4_0_f32.cl | 274 + .../kernels/gemv_noshuffle_q4_0_f32_spec.cl | 268 + .../kernels/gemv_noshuffle_q4_1_f32.cl | 283 + .../kernels/gemv_noshuffle_q4_k_f32.cl | 318 + .../kernels/gemv_noshuffle_q5_0_f32.cl | 291 + .../kernels/gemv_noshuffle_q5_1_f32.cl | 294 + .../kernels/gemv_noshuffle_q5_k_f32.cl | 326 + .../kernels/gemv_noshuffle_q6_k_f32.cl | 293 + .../kernels/gemv_noshuffle_q8_0_f32.cl | 195 + ggml/src/ggml-opencl/kernels/get_rows.cl | 193 + ggml/src/ggml-opencl/kernels/glu.cl | 378 + ggml/src/ggml-opencl/kernels/group_norm.cl | 121 + ggml/src/ggml-opencl/kernels/im2col_f16.cl | 57 + ggml/src/ggml-opencl/kernels/im2col_f32.cl | 57 + ggml/src/ggml-opencl/kernels/l2_norm.cl | 71 + ggml/src/ggml-opencl/kernels/mean.cl | 140 + ggml/src/ggml-opencl/kernels/moe_reorder_b.cl | 30 + .../ggml-opencl/kernels/moe_sort_by_expert.cl | 82 + ggml/src/ggml-opencl/kernels/mul.cl | 152 + .../ggml-opencl/kernels/mul_mat_f16_f32.cl | 130 + .../kernels/mul_mm_f16_f32_kq_kqv.cl | 273 + .../kernels/mul_mm_f16_f32_l4_lm.cl | 146 + .../kernels/mul_mm_f32_f32_l4_lm.cl | 147 + .../kernels/mul_mm_iq4_nl_f32_l4_lm.cl | 171 + .../kernels/mul_mm_q4_0_f32_l4_lm.cl | 163 + .../kernels/mul_mm_q4_1_f32_l4_lm.cl | 165 + .../kernels/mul_mm_q4_k_f32_l4_lm.cl | 179 + .../kernels/mul_mm_q5_0_f32_l4_lm.cl | 173 + .../kernels/mul_mm_q5_1_f32_l4_lm.cl | 175 + .../kernels/mul_mm_q5_k_f32_l4_lm.cl | 192 + .../kernels/mul_mm_q6_k_f32_l4_lm.cl | 158 + .../kernels/mul_mm_q8_0_f32_l4_lm.cl | 154 + .../src/ggml-opencl/kernels/mul_mv_f16_f16.cl | 118 + .../src/ggml-opencl/kernels/mul_mv_f16_f32.cl | 118 + .../kernels/mul_mv_f16_f32_1row.cl | 94 + .../ggml-opencl/kernels/mul_mv_f16_f32_l4.cl | 380 + .../src/ggml-opencl/kernels/mul_mv_f32_f32.cl | 118 + .../kernels/mul_mv_id_mxfp4_f32.cl | 189 + .../kernels/mul_mv_id_mxfp4_f32_flat.cl | 176 + .../kernels/mul_mv_id_q4_0_f32_8x_flat.cl | 283 + .../ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl | 140 + .../kernels/mul_mv_id_q8_0_f32_flat.cl | 222 + .../ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl | 164 + .../kernels/mul_mv_iq4_nl_f32_flat.cl | 202 + .../ggml-opencl/kernels/mul_mv_mxfp4_f32.cl | 144 + .../kernels/mul_mv_mxfp4_f32_flat.cl | 167 + .../ggml-opencl/kernels/mul_mv_q4_0_f32.cl | 192 + .../kernels/mul_mv_q4_0_f32_1d_16x_flat.cl | 307 + .../kernels/mul_mv_q4_0_f32_1d_8x_flat.cl | 265 + .../kernels/mul_mv_q4_0_f32_8x_flat.cl | 272 + .../ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl | 254 + .../ggml-opencl/kernels/mul_mv_q4_1_f32.cl | 219 + .../kernels/mul_mv_q4_1_f32_flat.cl | 229 + .../ggml-opencl/kernels/mul_mv_q4_k_f32.cl | 180 + .../kernels/mul_mv_q4_k_f32_flat.cl | 196 + .../ggml-opencl/kernels/mul_mv_q5_0_f32.cl | 241 + .../kernels/mul_mv_q5_0_f32_flat.cl | 243 + .../ggml-opencl/kernels/mul_mv_q5_1_f32.cl | 243 + .../kernels/mul_mv_q5_1_f32_flat.cl | 247 + .../ggml-opencl/kernels/mul_mv_q5_k_f32.cl | 187 + .../kernels/mul_mv_q5_k_f32_flat.cl | 203 + .../ggml-opencl/kernels/mul_mv_q6_k_f32.cl | 194 + .../kernels/mul_mv_q6_k_f32_flat.cl | 178 + .../ggml-opencl/kernels/mul_mv_q8_0_f32.cl | 125 + .../kernels/mul_mv_q8_0_f32_flat.cl | 202 + ggml/src/ggml-opencl/kernels/neg.cl | 125 + ggml/src/ggml-opencl/kernels/norm.cl | 161 + ggml/src/ggml-opencl/kernels/pad.cl | 39 + ggml/src/ggml-opencl/kernels/relu.cl | 16 + ggml/src/ggml-opencl/kernels/repeat.cl | 38 + ggml/src/ggml-opencl/kernels/rms_norm.cl | 190 + ggml/src/ggml-opencl/kernels/rope.cl | 747 + ggml/src/ggml-opencl/kernels/scale.cl | 27 + ggml/src/ggml-opencl/kernels/set_rows.cl | 208 + ggml/src/ggml-opencl/kernels/sigmoid.cl | 29 + ggml/src/ggml-opencl/kernels/silu.cl | 30 + ggml/src/ggml-opencl/kernels/softmax_4_f16.cl | 108 + ggml/src/ggml-opencl/kernels/softmax_4_f32.cl | 108 + ggml/src/ggml-opencl/kernels/softmax_f16.cl | 107 + ggml/src/ggml-opencl/kernels/softmax_f32.cl | 107 + ggml/src/ggml-opencl/kernels/softplus.cl | 116 + ggml/src/ggml-opencl/kernels/solve_tri.cl | 51 + ggml/src/ggml-opencl/kernels/sqr.cl | 53 + ggml/src/ggml-opencl/kernels/sqrt.cl | 53 + ggml/src/ggml-opencl/kernels/ssm_conv.cl | 77 + ggml/src/ggml-opencl/kernels/sub.cl | 138 + ggml/src/ggml-opencl/kernels/sum_rows.cl | 140 + ggml/src/ggml-opencl/kernels/tanh.cl | 109 + ggml/src/ggml-opencl/kernels/transpose.cl | 143 + ggml/src/ggml-opencl/kernels/tri.cl | 32 + ggml/src/ggml-opencl/kernels/tsembd.cl | 48 + ggml/src/ggml-opencl/kernels/upscale.cl | 120 + ggml/src/ggml-openvino/.clang-format | 149 + ggml/src/ggml-openvino/CMakeLists.txt | 20 + ggml/src/ggml-openvino/ggml-decoder.cpp | 1651 ++ ggml/src/ggml-openvino/ggml-decoder.h | 347 + .../src/ggml-openvino/ggml-openvino-extra.cpp | 434 + ggml/src/ggml-openvino/ggml-openvino-extra.h | 203 + ggml/src/ggml-openvino/ggml-openvino.cpp | 1376 ++ ggml/src/ggml-openvino/ggml-quants.cpp | 1022 + ggml/src/ggml-openvino/ggml-quants.h | 159 + ggml/src/ggml-openvino/openvino/decoder.h | 114 + ggml/src/ggml-openvino/openvino/frontend.cpp | 27 + ggml/src/ggml-openvino/openvino/frontend.h | 23 + .../ggml-openvino/openvino/input_model.cpp | 17 + ggml/src/ggml-openvino/openvino/input_model.h | 29 + .../src/ggml-openvino/openvino/node_context.h | 169 + ggml/src/ggml-openvino/openvino/op/add_id.cpp | 62 + .../src/ggml-openvino/openvino/op/argsort.cpp | 47 + ggml/src/ggml-openvino/openvino/op/clamp.cpp | 33 + ggml/src/ggml-openvino/openvino/op/concat.cpp | 48 + ggml/src/ggml-openvino/openvino/op/cont.cpp | 40 + ggml/src/ggml-openvino/openvino/op/cpy.cpp | 34 + ggml/src/ggml-openvino/openvino/op/div.cpp | 146 + .../openvino/op/flash_attn_ext.cpp | 177 + .../openvino/op/gated_delta_net.cpp | 282 + .../openvino/op/gated_delta_net.hpp | 65 + .../ggml-openvino/openvino/op/get_rows.cpp | 62 + .../ggml-openvino/openvino/op/glu_geglu.cpp | 75 + .../ggml-openvino/openvino/op/glu_swiglu.cpp | 65 + ggml/src/ggml-openvino/openvino/op/im2col.cpp | 120 + .../src/ggml-openvino/openvino/op/l2_norm.cpp | 44 + .../ggml-openvino/openvino/op/mul_mat_id.cpp | 108 + ggml/src/ggml-openvino/openvino/op/mulmat.cpp | 100 + ggml/src/ggml-openvino/openvino/op/norm.cpp | 58 + ggml/src/ggml-openvino/openvino/op/pad.cpp | 95 + .../src/ggml-openvino/openvino/op/permute.cpp | 147 + ggml/src/ggml-openvino/openvino/op/repeat.cpp | 74 + .../src/ggml-openvino/openvino/op/reshape.cpp | 90 + .../ggml-openvino/openvino/op/rms_norm.cpp | 46 + ggml/src/ggml-openvino/openvino/op/rope.cpp | 245 + ggml/src/ggml-openvino/openvino/op/scale.cpp | 41 + .../ggml-openvino/openvino/op/set_rows.cpp | 76 + .../src/ggml-openvino/openvino/op/softmax.cpp | 106 + .../ggml-openvino/openvino/op/ssm_conv.cpp | 59 + .../ggml-openvino/openvino/op/sum_rows.cpp | 27 + .../ggml-openvino/openvino/op/transpose.cpp | 54 + .../ggml-openvino/openvino/op/unary_silu.cpp | 27 + .../openvino/op/unary_softplus.cpp | 38 + ggml/src/ggml-openvino/openvino/op/view.cpp | 118 + ggml/src/ggml-openvino/openvino/op_table.cpp | 64 + ggml/src/ggml-openvino/openvino/op_table.h | 52 + .../openvino/pass/fuse_to_sdpa.cpp | 60 + .../openvino/pass/fuse_to_sdpa.h | 17 + ...k_decompression_convert_constant_folding.h | 29 + .../openvino/pass/squeeze_matmul.cpp | 58 + .../openvino/pass/squeeze_matmul.h | 17 + .../rt_info/weightless_caching_attributes.hpp | 41 + .../openvino/translate_session.cpp | 320 + .../openvino/translate_session.h | 29 + ggml/src/ggml-openvino/openvino/utils.cpp | 802 + ggml/src/ggml-openvino/openvino/utils.h | 88 + ggml/src/ggml-openvino/utils.cpp | 1169 + ggml/src/ggml-openvino/utils.h | 146 + ggml/src/ggml-opt.cpp | 1094 + ggml/src/ggml-quants.c | 5591 +++++ ggml/src/ggml-quants.h | 112 + ggml/src/ggml-rpc/CMakeLists.txt | 33 + ggml/src/ggml-rpc/ggml-rpc.cpp | 1974 ++ ggml/src/ggml-rpc/transport.cpp | 683 + ggml/src/ggml-rpc/transport.h | 34 + ggml/src/ggml-sycl/CMakeLists.txt | 207 + ggml/src/ggml-sycl/add-id.cpp | 81 + ggml/src/ggml-sycl/add-id.hpp | 8 + ggml/src/ggml-sycl/backend.hpp | 50 + ggml/src/ggml-sycl/binbcast.cpp | 353 + ggml/src/ggml-sycl/binbcast.hpp | 39 + ggml/src/ggml-sycl/col2im-1d.cpp | 97 + ggml/src/ggml-sycl/col2im-1d.hpp | 8 + ggml/src/ggml-sycl/common.cpp | 153 + ggml/src/ggml-sycl/common.hpp | 1025 + ggml/src/ggml-sycl/concat.cpp | 222 + ggml/src/ggml-sycl/concat.hpp | 20 + ggml/src/ggml-sycl/conv.cpp | 101 + ggml/src/ggml-sycl/conv.hpp | 20 + ggml/src/ggml-sycl/conv2d-dw.cpp | 158 + ggml/src/ggml-sycl/conv2d-dw.hpp | 10 + ggml/src/ggml-sycl/conv2d-transpose.cpp | 125 + ggml/src/ggml-sycl/conv2d-transpose.hpp | 10 + ggml/src/ggml-sycl/conv2d.cpp | 150 + ggml/src/ggml-sycl/conv2d.hpp | 10 + ggml/src/ggml-sycl/conv3d.cpp | 218 + ggml/src/ggml-sycl/conv3d.hpp | 8 + ggml/src/ggml-sycl/convert.cpp | 852 + ggml/src/ggml-sycl/convert.hpp | 64 + ggml/src/ggml-sycl/count-equal.cpp | 79 + ggml/src/ggml-sycl/count-equal.hpp | 9 + ggml/src/ggml-sycl/cpy.cpp | 602 + ggml/src/ggml-sycl/cpy.hpp | 504 + ggml/src/ggml-sycl/cumsum.cpp | 148 + ggml/src/ggml-sycl/cumsum.hpp | 5 + ggml/src/ggml-sycl/dequantize.hpp | 1519 ++ ggml/src/ggml-sycl/diag.cpp | 67 + ggml/src/ggml-sycl/diag.hpp | 5 + ggml/src/ggml-sycl/dmmv.cpp | 1912 ++ ggml/src/ggml-sycl/dmmv.hpp | 27 + ggml/src/ggml-sycl/dpct/helper.hpp | 3782 +++ ggml/src/ggml-sycl/element_wise.cpp | 1098 + ggml/src/ggml-sycl/element_wise.hpp | 96 + ggml/src/ggml-sycl/fattn-buffers.cpp | 56 + ggml/src/ggml-sycl/fattn-buffers.hpp | 63 + ggml/src/ggml-sycl/fattn-common.hpp | 1181 + ggml/src/ggml-sycl/fattn-tile.cpp | 59 + ggml/src/ggml-sycl/fattn-tile.hpp | 1246 + ggml/src/ggml-sycl/fattn-vec.hpp | 674 + ggml/src/ggml-sycl/fattn.cpp | 227 + ggml/src/ggml-sycl/fattn.hpp | 22 + ggml/src/ggml-sycl/fill.cpp | 55 + ggml/src/ggml-sycl/fill.hpp | 5 + ggml/src/ggml-sycl/gated_delta_net.cpp | 347 + ggml/src/ggml-sycl/gated_delta_net.hpp | 9 + ggml/src/ggml-sycl/gemm.hpp | 93 + ggml/src/ggml-sycl/getrows.cpp | 291 + ggml/src/ggml-sycl/getrows.hpp | 20 + ggml/src/ggml-sycl/ggml-sycl.cpp | 5969 +++++ ggml/src/ggml-sycl/gla.cpp | 106 + ggml/src/ggml-sycl/gla.hpp | 8 + ggml/src/ggml-sycl/im2col.cpp | 400 + ggml/src/ggml-sycl/im2col.hpp | 23 + ggml/src/ggml-sycl/mmq.cpp | 3030 +++ ggml/src/ggml-sycl/mmq.hpp | 33 + ggml/src/ggml-sycl/mmvq.cpp | 2659 +++ ggml/src/ggml-sycl/mmvq.hpp | 60 + ggml/src/ggml-sycl/norm.cpp | 656 + ggml/src/ggml-sycl/norm.hpp | 28 + ggml/src/ggml-sycl/outprod.cpp | 83 + ggml/src/ggml-sycl/outprod.hpp | 10 + ggml/src/ggml-sycl/pad.cpp | 97 + ggml/src/ggml-sycl/pad.hpp | 24 + ggml/src/ggml-sycl/pad_reflect_1d.cpp | 100 + ggml/src/ggml-sycl/pad_reflect_1d.hpp | 10 + ggml/src/ggml-sycl/pool.cpp | 185 + ggml/src/ggml-sycl/pool.hpp | 22 + ggml/src/ggml-sycl/presets.hpp | 81 + ggml/src/ggml-sycl/quantize.hpp | 133 + ggml/src/ggml-sycl/quants.hpp | 181 + ggml/src/ggml-sycl/repeat_back.cpp | 76 + ggml/src/ggml-sycl/repeat_back.hpp | 8 + ggml/src/ggml-sycl/roll.cpp | 122 + ggml/src/ggml-sycl/roll.hpp | 20 + ggml/src/ggml-sycl/rope.cpp | 641 + ggml/src/ggml-sycl/rope.hpp | 26 + ggml/src/ggml-sycl/set.cpp | 73 + ggml/src/ggml-sycl/set.hpp | 5 + ggml/src/ggml-sycl/set_rows.cpp | 248 + ggml/src/ggml-sycl/set_rows.hpp | 8 + ggml/src/ggml-sycl/softmax.cpp | 425 + ggml/src/ggml-sycl/softmax.hpp | 24 + ggml/src/ggml-sycl/solve_tri.cpp | 172 + ggml/src/ggml-sycl/solve_tri.hpp | 8 + ggml/src/ggml-sycl/ssm_conv.cpp | 132 + ggml/src/ggml-sycl/ssm_conv.hpp | 5 + ggml/src/ggml-sycl/ssm_scan.cpp | 156 + ggml/src/ggml-sycl/ssm_scan.hpp | 5 + ggml/src/ggml-sycl/sycl_hw.cpp | 67 + ggml/src/ggml-sycl/sycl_hw.hpp | 38 + .../fattn-tile-instance-dkq112-dv112.cpp | 5 + .../fattn-tile-instance-dkq128-dv128.cpp | 5 + .../fattn-tile-instance-dkq256-dv256.cpp | 5 + .../fattn-tile-instance-dkq40-dv40.cpp | 5 + .../fattn-tile-instance-dkq512-dv512.cpp | 6 + .../fattn-tile-instance-dkq576-dv512.cpp | 5 + .../fattn-tile-instance-dkq64-dv64.cpp | 5 + .../fattn-tile-instance-dkq72-dv72.cpp | 5 + .../fattn-tile-instance-dkq80-dv80.cpp | 5 + .../fattn-tile-instance-dkq96-dv96.cpp | 5 + .../fattn-vec-instance-f16-f16.cpp | 8 + .../fattn-vec-instance-f16-q4_0.cpp | 8 + .../fattn-vec-instance-f16-q4_1.cpp | 8 + .../fattn-vec-instance-f16-q5_0.cpp | 8 + .../fattn-vec-instance-f16-q5_1.cpp | 8 + .../fattn-vec-instance-f16-q8_0.cpp | 8 + .../fattn-vec-instance-q4_0-f16.cpp | 8 + .../fattn-vec-instance-q4_0-q4_0.cpp | 8 + .../fattn-vec-instance-q4_0-q4_1.cpp | 8 + .../fattn-vec-instance-q4_0-q5_0.cpp | 8 + .../fattn-vec-instance-q4_0-q5_1.cpp | 8 + .../fattn-vec-instance-q4_0-q8_0.cpp | 8 + .../fattn-vec-instance-q4_1-f16.cpp | 8 + .../fattn-vec-instance-q4_1-q4_0.cpp | 8 + .../fattn-vec-instance-q4_1-q4_1.cpp | 8 + .../fattn-vec-instance-q4_1-q5_0.cpp | 8 + .../fattn-vec-instance-q4_1-q5_1.cpp | 8 + .../fattn-vec-instance-q4_1-q8_0.cpp | 8 + .../fattn-vec-instance-q5_0-f16.cpp | 8 + .../fattn-vec-instance-q5_0-q4_0.cpp | 8 + .../fattn-vec-instance-q5_0-q4_1.cpp | 8 + .../fattn-vec-instance-q5_0-q5_0.cpp | 8 + .../fattn-vec-instance-q5_0-q5_1.cpp | 8 + .../fattn-vec-instance-q5_0-q8_0.cpp | 8 + .../fattn-vec-instance-q5_1-f16.cpp | 8 + .../fattn-vec-instance-q5_1-q4_0.cpp | 8 + .../fattn-vec-instance-q5_1-q4_1.cpp | 8 + .../fattn-vec-instance-q5_1-q5_0.cpp | 8 + .../fattn-vec-instance-q5_1-q5_1.cpp | 8 + .../fattn-vec-instance-q5_1-q8_0.cpp | 8 + .../fattn-vec-instance-q8_0-f16.cpp | 8 + .../fattn-vec-instance-q8_0-q4_0.cpp | 8 + .../fattn-vec-instance-q8_0-q4_1.cpp | 8 + .../fattn-vec-instance-q8_0-q5_0.cpp | 8 + .../fattn-vec-instance-q8_0-q5_1.cpp | 8 + .../fattn-vec-instance-q8_0-q8_0.cpp | 8 + ggml/src/ggml-sycl/tsembd.cpp | 73 + ggml/src/ggml-sycl/tsembd.hpp | 20 + ggml/src/ggml-sycl/type.hpp | 112 + ggml/src/ggml-sycl/upscale.cpp | 410 + ggml/src/ggml-sycl/upscale.hpp | 9 + ggml/src/ggml-sycl/vecdotq.hpp | 1571 ++ ggml/src/ggml-sycl/wkv.cpp | 293 + ggml/src/ggml-sycl/wkv.hpp | 10 + ggml/src/ggml-threading.cpp | 12 + ggml/src/ggml-threading.h | 14 + ggml/src/ggml-virtgpu/CMakeLists.txt | 70 + .../ggml-virtgpu/apir_cs_ggml-rpc-front.cpp | 87 + ggml/src/ggml-virtgpu/backend/CMakeLists.txt | 21 + .../backend/apir_cs_ggml-rpc-back.cpp | 115 + .../ggml-virtgpu/backend/backend-convert.h | 13 + .../backend/backend-dispatched-backend.cpp | 102 + .../backend-dispatched-buffer-type.cpp | 105 + .../backend/backend-dispatched-buffer.cpp | 179 + .../backend/backend-dispatched-device.cpp | 148 + .../backend/backend-dispatched.cpp | 51 + .../backend/backend-dispatched.gen.h | 73 + .../ggml-virtgpu/backend/backend-dispatched.h | 27 + .../ggml-virtgpu/backend/backend-virgl-apir.h | 32 + ggml/src/ggml-virtgpu/backend/backend.cpp | 144 + .../backend/shared/api_remoting.h | 95 + .../backend/shared/apir_backend.gen.h | 94 + .../backend/shared/apir_backend.h | 50 + .../src/ggml-virtgpu/backend/shared/apir_cs.h | 378 + .../backend/shared/apir_cs_ggml.h | 232 + .../ggml-virtgpu/backend/shared/apir_cs_rpc.h | 58 + .../ggml-virtgpu/ggml-backend-buffer-type.cpp | 81 + ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp | 123 + ggml/src/ggml-virtgpu/ggml-backend-device.cpp | 160 + ggml/src/ggml-virtgpu/ggml-backend-reg.cpp | 213 + ggml/src/ggml-virtgpu/ggml-backend.cpp | 71 + ggml/src/ggml-virtgpu/ggml-remoting.h | 71 + .../ggml-virtgpu/ggmlremoting_functions.yaml | 166 + ggml/src/ggml-virtgpu/include/apir_hw.h | 9 + ggml/src/ggml-virtgpu/regenerate_remoting.py | 333 + ggml/src/ggml-virtgpu/virtgpu-apir.h | 15 + .../ggml-virtgpu/virtgpu-forward-backend.cpp | 58 + .../virtgpu-forward-buffer-type.cpp | 110 + .../ggml-virtgpu/virtgpu-forward-buffer.cpp | 173 + .../ggml-virtgpu/virtgpu-forward-device.cpp | 192 + ggml/src/ggml-virtgpu/virtgpu-forward-impl.h | 36 + ggml/src/ggml-virtgpu/virtgpu-forward.gen.h | 53 + ggml/src/ggml-virtgpu/virtgpu-shm.cpp | 99 + ggml/src/ggml-virtgpu/virtgpu-shm.h | 23 + ggml/src/ggml-virtgpu/virtgpu-utils.cpp | 179 + ggml/src/ggml-virtgpu/virtgpu-utils.h | 86 + ggml/src/ggml-virtgpu/virtgpu.cpp | 545 + ggml/src/ggml-virtgpu/virtgpu.h | 115 + ggml/src/ggml-vulkan/CMakeLists.txt | 231 + .../ggml-vulkan/cmake/host-toolchain.cmake.in | 15 + ggml/src/ggml-vulkan/ggml-vulkan.cpp | 18393 +++++++++++++++ .../ggml-vulkan/vulkan-shaders/CMakeLists.txt | 35 + ggml/src/ggml-vulkan/vulkan-shaders/acc.comp | 37 + ggml/src/ggml-vulkan/vulkan-shaders/add.comp | 69 + ggml/src/ggml-vulkan/vulkan-shaders/add1.comp | 28 + .../ggml-vulkan/vulkan-shaders/add_id.comp | 42 + .../ggml-vulkan/vulkan-shaders/arange.comp | 20 + .../ggml-vulkan/vulkan-shaders/argmax.comp | 60 + .../ggml-vulkan/vulkan-shaders/argsort.comp | 86 + .../vulkan-shaders/argsort_large.comp | 114 + .../src/ggml-vulkan/vulkan-shaders/clamp.comp | 17 + .../ggml-vulkan/vulkan-shaders/col2im_1d.comp | 61 + .../ggml-vulkan/vulkan-shaders/concat.comp | 41 + .../vulkan-shaders/contig_copy.comp | 53 + .../ggml-vulkan/vulkan-shaders/conv2d_dw.comp | 105 + .../ggml-vulkan/vulkan-shaders/conv2d_mm.comp | 480 + .../vulkan-shaders/conv_transpose_1d.comp | 98 + ggml/src/ggml-vulkan/vulkan-shaders/copy.comp | 25 + .../vulkan-shaders/copy_from_quant.comp | 51 + .../vulkan-shaders/copy_to_quant.comp | 320 + .../vulkan-shaders/copy_transpose.comp | 67 + ggml/src/ggml-vulkan/vulkan-shaders/cos.comp | 17 + .../vulkan-shaders/count_equal.comp | 31 + .../vulkan-shaders/count_experts.comp | 51 + .../ggml-vulkan/vulkan-shaders/cumsum.comp | 83 + .../vulkan-shaders/cumsum_multipass1.comp | 60 + .../vulkan-shaders/cumsum_multipass2.comp | 66 + .../vulkan-shaders/dequant_f32.comp | 20 + .../vulkan-shaders/dequant_funcs.glsl | 692 + .../vulkan-shaders/dequant_funcs_cm2.glsl | 1376 ++ .../vulkan-shaders/dequant_head.glsl | 13 + .../vulkan-shaders/dequant_iq1_m.comp | 42 + .../vulkan-shaders/dequant_iq1_s.comp | 35 + .../vulkan-shaders/dequant_iq2_s.comp | 44 + .../vulkan-shaders/dequant_iq2_xs.comp | 43 + .../vulkan-shaders/dequant_iq2_xxs.comp | 49 + .../vulkan-shaders/dequant_iq3_s.comp | 40 + .../vulkan-shaders/dequant_iq3_xxs.comp | 51 + .../vulkan-shaders/dequant_iq4_nl.comp | 32 + .../vulkan-shaders/dequant_iq4_xs.comp | 34 + .../vulkan-shaders/dequant_mxfp4.comp | 32 + .../vulkan-shaders/dequant_nvfp4.comp | 32 + .../vulkan-shaders/dequant_q1_0.comp | 29 + .../vulkan-shaders/dequant_q2_k.comp | 34 + .../vulkan-shaders/dequant_q3_k.comp | 42 + .../vulkan-shaders/dequant_q4_0.comp | 30 + .../vulkan-shaders/dequant_q4_1.comp | 32 + .../vulkan-shaders/dequant_q4_k.comp | 68 + .../vulkan-shaders/dequant_q5_0.comp | 34 + .../vulkan-shaders/dequant_q5_1.comp | 35 + .../vulkan-shaders/dequant_q5_k.comp | 70 + .../vulkan-shaders/dequant_q6_k.comp | 33 + .../vulkan-shaders/dequant_q8_0.comp | 31 + ggml/src/ggml-vulkan/vulkan-shaders/diag.comp | 28 + .../vulkan-shaders/diag_mask_inf.comp | 34 + ggml/src/ggml-vulkan/vulkan-shaders/div.comp | 27 + .../vulkan-shaders/dot_product_funcs.glsl | 27 + .../feature-tests/bfloat16.comp | 7 + .../vulkan-shaders/feature-tests/coopmat.comp | 7 + .../feature-tests/coopmat2.comp | 7 + .../feature-tests/coopmat2_decode_vector.comp | 7 + .../feature-tests/integer_dot.comp | 7 + ggml/src/ggml-vulkan/vulkan-shaders/fill.comp | 19 + .../vulkan-shaders/flash_attn.comp | 757 + .../vulkan-shaders/flash_attn_base.glsl | 265 + .../vulkan-shaders/flash_attn_cm1.comp | 644 + .../vulkan-shaders/flash_attn_cm2.comp | 481 + .../vulkan-shaders/flash_attn_dequant.glsl | 131 + .../vulkan-shaders/flash_attn_mask_opt.comp | 162 + .../vulkan-shaders/flash_attn_mmq_funcs.glsl | 203 + .../flash_attn_split_k_reduce.comp | 121 + ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp | 115 + .../vulkan-shaders/gated_delta_net.comp | 189 + .../src/ggml-vulkan/vulkan-shaders/geglu.comp | 13 + .../ggml-vulkan/vulkan-shaders/geglu_erf.comp | 27 + .../vulkan-shaders/geglu_quick.comp | 11 + .../vulkan-shaders/generic_binary_head.glsl | 65 + .../vulkan-shaders/generic_head.glsl | 11 + .../vulkan-shaders/generic_unary_head.glsl | 85 + .../ggml-vulkan/vulkan-shaders/get_rows.comp | 42 + .../vulkan-shaders/get_rows_quant.comp | 51 + .../ggml-vulkan/vulkan-shaders/glu_head.glsl | 47 + .../ggml-vulkan/vulkan-shaders/glu_main.glsl | 35 + .../vulkan-shaders/group_norm.comp | 66 + .../ggml-vulkan/vulkan-shaders/im2col.comp | 138 + .../ggml-vulkan/vulkan-shaders/im2col_3d.comp | 124 + .../ggml-vulkan/vulkan-shaders/l2_norm.comp | 44 + .../vulkan-shaders/leaky_relu.comp | 22 + ggml/src/ggml-vulkan/vulkan-shaders/log.comp | 17 + ggml/src/ggml-vulkan/vulkan-shaders/mul.comp | 27 + .../mul_mat_split_k_reduce.comp | 48 + .../vulkan-shaders/mul_mat_vec.comp | 264 + .../vulkan-shaders/mul_mat_vec_base.glsl | 230 + .../vulkan-shaders/mul_mat_vec_iface.glsl | 35 + .../vulkan-shaders/mul_mat_vec_iq1_m.comp | 132 + .../vulkan-shaders/mul_mat_vec_iq1_s.comp | 95 + .../vulkan-shaders/mul_mat_vec_iq2_s.comp | 90 + .../vulkan-shaders/mul_mat_vec_iq2_xs.comp | 105 + .../vulkan-shaders/mul_mat_vec_iq2_xxs.comp | 87 + .../vulkan-shaders/mul_mat_vec_iq3_s.comp | 90 + .../vulkan-shaders/mul_mat_vec_iq3_xxs.comp | 88 + .../vulkan-shaders/mul_mat_vec_nc.comp | 124 + .../vulkan-shaders/mul_mat_vec_p021.comp | 156 + .../vulkan-shaders/mul_mat_vec_q2_k.comp | 128 + .../vulkan-shaders/mul_mat_vec_q3_k.comp | 132 + .../vulkan-shaders/mul_mat_vec_q4_k.comp | 134 + .../vulkan-shaders/mul_mat_vec_q5_k.comp | 165 + .../vulkan-shaders/mul_mat_vec_q6_k.comp | 130 + .../vulkan-shaders/mul_mat_vecq.comp | 144 + .../vulkan-shaders/mul_mat_vecq_funcs.glsl | 527 + .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 458 + .../vulkan-shaders/mul_mm_cm2.comp | 657 + .../vulkan-shaders/mul_mm_funcs.glsl | 600 + .../vulkan-shaders/mul_mm_id_funcs.glsl | 74 + .../ggml-vulkan/vulkan-shaders/mul_mmq.comp | 311 + .../vulkan-shaders/mul_mmq_funcs.glsl | 454 + .../vulkan-shaders/mul_mmq_shmem_types.glsl | 93 + .../ggml-vulkan/vulkan-shaders/multi_add.comp | 194 + ggml/src/ggml-vulkan/vulkan-shaders/norm.comp | 44 + .../vulkan-shaders/opt_step_adamw.comp | 42 + .../vulkan-shaders/opt_step_sgd.comp | 22 + ggml/src/ggml-vulkan/vulkan-shaders/pad.comp | 64 + .../ggml-vulkan/vulkan-shaders/pool2d.comp | 74 + .../vulkan-shaders/quantize_q8_1.comp | 127 + .../src/ggml-vulkan/vulkan-shaders/reglu.comp | 9 + .../ggml-vulkan/vulkan-shaders/repeat.comp | 26 + .../vulkan-shaders/repeat_back.comp | 37 + .../ggml-vulkan/vulkan-shaders/rms_norm.comp | 150 + .../vulkan-shaders/rms_norm_back.comp | 55 + .../vulkan-shaders/rms_norm_partials.comp | 65 + ggml/src/ggml-vulkan/vulkan-shaders/roll.comp | 46 + .../vulkan-shaders/rope_funcs.glsl | 210 + .../ggml-vulkan/vulkan-shaders/rope_head.glsl | 19 + .../vulkan-shaders/rope_multi.comp | 17 + .../ggml-vulkan/vulkan-shaders/rope_neox.comp | 17 + .../ggml-vulkan/vulkan-shaders/rope_norm.comp | 17 + .../vulkan-shaders/rope_params.glsl | 34 + .../vulkan-shaders/rope_vision.comp | 17 + .../src/ggml-vulkan/vulkan-shaders/scale.comp | 24 + .../ggml-vulkan/vulkan-shaders/silu_back.comp | 26 + ggml/src/ggml-vulkan/vulkan-shaders/sin.comp | 17 + .../src/ggml-vulkan/vulkan-shaders/snake.comp | 49 + .../ggml-vulkan/vulkan-shaders/soft_max.comp | 195 + .../vulkan-shaders/soft_max_back.comp | 54 + .../vulkan-shaders/soft_max_large1.comp | 62 + .../vulkan-shaders/soft_max_large2.comp | 79 + .../vulkan-shaders/soft_max_large3.comp | 65 + .../vulkan-shaders/soft_max_large_common.glsl | 53 + .../ggml-vulkan/vulkan-shaders/solve_tri.comp | 81 + ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp | 17 + .../ggml-vulkan/vulkan-shaders/square.comp | 17 + .../ggml-vulkan/vulkan-shaders/ssm_conv.comp | 60 + .../ggml-vulkan/vulkan-shaders/ssm_scan.comp | 124 + ggml/src/ggml-vulkan/vulkan-shaders/sub.comp | 29 + .../ggml-vulkan/vulkan-shaders/sum_rows.comp | 47 + .../ggml-vulkan/vulkan-shaders/sum_rows.glsl | 25 + .../ggml-vulkan/vulkan-shaders/swiglu.comp | 9 + .../vulkan-shaders/swiglu_oai.comp | 14 + .../vulkan-shaders/timestep_embedding.comp | 42 + .../vulkan-shaders/topk_argsort.comp | 118 + .../ggml-vulkan/vulkan-shaders/topk_moe.comp | 213 + .../vulkan-shaders/topk_nary_search.comp | 246 + ggml/src/ggml-vulkan/vulkan-shaders/tri.comp | 42 + .../src/ggml-vulkan/vulkan-shaders/types.glsl | 1861 ++ .../src/ggml-vulkan/vulkan-shaders/unary.comp | 144 + .../ggml-vulkan/vulkan-shaders/upscale.comp | 178 + .../src/ggml-vulkan/vulkan-shaders/utils.glsl | 25 + .../vulkan-shaders/vulkan-shaders-gen.cpp | 1257 + ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp | 87 + ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp | 91 + ggml/src/ggml-webgpu/CMakeLists.txt | 83 + .../ggml-webgpu/ggml-webgpu-shader-lib.hpp | 3293 +++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 4583 ++++ ggml/src/ggml-webgpu/pre_wgsl.hpp | 808 + ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl | 64 + ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl | 72 + .../src/ggml-webgpu/wgsl-shaders/argsort.wgsl | 106 + .../wgsl-shaders/argsort_merge.wgsl | 134 + ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl | 142 + .../wgsl-shaders/common_decls.tmpl | 904 + ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl | 93 + ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl | 165 + ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl | 82 + ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl | 66 + .../ggml-webgpu/wgsl-shaders/embed_wgsl.py | 89 + .../ggml-webgpu/wgsl-shaders/flash_attn.wgsl | 559 + .../flash_attn_quant_staging.tmpl | 124 + .../wgsl-shaders/flash_attn_tile.wgsl | 397 + .../wgsl-shaders/flash_attn_vec_blk.wgsl | 101 + .../wgsl-shaders/flash_attn_vec_reduce.wgsl | 84 + .../wgsl-shaders/flash_attn_vec_split.wgsl | 619 + .../wgsl-shaders/gated_delta_net.wgsl | 149 + .../ggml-webgpu/wgsl-shaders/get_rows.wgsl | 773 + ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl | 155 + ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl | 101 + ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl | 40 + .../wgsl-shaders/mul_mat_decls.tmpl | 1004 + .../ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl | 195 + .../wgsl-shaders/mul_mat_id_gather.wgsl | 52 + .../wgsl-shaders/mul_mat_id_vec.wgsl | 154 + .../wgsl-shaders/mul_mat_reg_tile.wgsl | 149 + .../wgsl-shaders/mul_mat_subgroup_matrix.wgsl | 200 + .../ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl | 151 + .../wgsl-shaders/mul_mat_vec_acc.tmpl | 1432 ++ .../wgsl-shaders/mul_mat_vec_q_acc.tmpl | 303 + ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl | 86 + .../wgsl-shaders/quant_inner_loops.tmpl | 21 + .../ggml-webgpu/wgsl-shaders/quantize_q8.wgsl | 173 + ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl | 67 + .../wgsl-shaders/rms_norm_mul.wgsl | 152 + ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl | 224 + .../ggml-webgpu/wgsl-shaders/row_norm.wgsl | 153 + ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl | 65 + ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl | 109 + .../ggml-webgpu/wgsl-shaders/set_rows.wgsl | 108 + .../wgsl-shaders/set_rows_quant.wgsl | 224 + .../ggml-webgpu/wgsl-shaders/soft_max.wgsl | 245 + .../ggml-webgpu/wgsl-shaders/solve_tri.wgsl | 121 + .../ggml-webgpu/wgsl-shaders/ssm_conv.wgsl | 65 + .../ggml-webgpu/wgsl-shaders/ssm_scan.wgsl | 193 + .../ggml-webgpu/wgsl-shaders/sum_rows.wgsl | 55 + ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl | 213 + .../src/ggml-webgpu/wgsl-shaders/upscale.wgsl | 240 + ggml/src/ggml-zdnn/.gitignore | 1 + ggml/src/ggml-zdnn/CMakeLists.txt | 36 + ggml/src/ggml-zdnn/common.hpp | 59 + ggml/src/ggml-zdnn/ggml-zdnn.cpp | 637 + ggml/src/ggml-zdnn/mmf.cpp | 80 + ggml/src/ggml-zdnn/mmf.hpp | 12 + ggml/src/ggml-zdnn/utils.cpp | 79 + ggml/src/ggml-zdnn/utils.hpp | 19 + ggml/src/ggml-zendnn/CMakeLists.txt | 91 + ggml/src/ggml-zendnn/ggml-zendnn.cpp | 703 + ggml/src/ggml.c | 7818 +++++++ ggml/src/ggml.cpp | 26 + ggml/src/gguf.cpp | 1688 ++ ggml/tests/CMakeLists.txt | 356 + ggml/tests/test-arange.cpp | 100 + ggml/tests/test-backend-ops.cpp | 10035 ++++++++ ggml/tests/test-cont.c | 170 + ggml/tests/test-conv-transpose-1d.cpp | 691 + ggml/tests/test-conv-transpose.c | 248 + ggml/tests/test-conv1d-dw-c1.cpp | 243 + ggml/tests/test-conv1d-dw-c2.cpp | 243 + ggml/tests/test-conv1d.cpp | 289 + ggml/tests/test-conv2d-dw.cpp | 153 + ggml/tests/test-conv2d.cpp | 391 + ggml/tests/test-customop.c | 300 + ggml/tests/test-dup.c | 111 + ggml/tests/test-interpolate.cpp | 166 + ggml/tests/test-opt.cpp | 1003 + ggml/tests/test-pad-reflect-1d.cpp | 213 + ggml/tests/test-pool.c | 274 + ggml/tests/test-quantize-fns.cpp | 196 + ggml/tests/test-quantize-perf.cpp | 356 + ggml/tests/test-rel-pos.c | 87 + ggml/tests/test-roll.cpp | 128 + ggml/tests/test-timestep_embedding.cpp | 180 + models.sh | 22 + quantize.sh | 37 + src/audio-io.h | 487 + src/audio-postproc-stream.h | 458 + src/audio-postproc.h | 423 + src/audio-resample.h | 177 + src/backend.h | 171 + src/bpe.h | 633 + src/dac-decoder.h | 428 + src/dac-encoder.h | 192 + src/debug.h | 120 + src/duration-estimator.h | 1233 + src/gguf-weights.h | 445 + src/hubert-enc.h | 691 + src/lang-map.h | 698 + src/maskgit-tts.h | 366 + src/omnivoice-llm.h | 109 + src/omnivoice.cpp | 441 + src/omnivoice.h | 313 + src/ov-error.h | 59 + src/philox.h | 158 + src/pipeline-codec.cpp | 799 + src/pipeline-codec.h | 92 + src/pipeline-tts.cpp | 1447 ++ src/pipeline-tts.h | 178 + src/prompt-tts.h | 348 + src/qwen3-enc.h | 371 + src/rvq-codec.h | 251 + src/rvq-file.h | 111 + src/semantic-enc.h | 166 + src/srt.h | 178 + src/text-chunker-stream.h | 133 + src/text-chunker.h | 505 + src/timer.h | 18 + src/tts-server.h | 319 + src/utf8.h | 140 + src/voice-design.h | 460 + src/wav.h | 183 + src/weight-ctx.h | 83 + tests/abi-c.c | 167 + tests/clone-CPU-BF16.log | 190 + tests/clone-CPU-F32.log | 191 + tests/clone-CPU-Q4_K_M.log | 191 + tests/clone-CPU-Q8_0.log | 191 + tests/clone-CUDA0-BF16.log | 191 + tests/clone-CUDA0-F32.log | 191 + tests/clone-CUDA0-Q4_K_M.log | 191 + tests/clone-CUDA0-Q8_0.log | 191 + tests/clone-Vulkan0-BF16.log | 191 + tests/clone-Vulkan0-F32.log | 191 + tests/clone-Vulkan0-Q4_K_M.log | 191 + tests/clone-Vulkan0-Q8_0.log | 191 + tests/cross-decode.py | 140 + tests/debug-clone-cossim.py | 585 + tests/debug-clone-cossim.sh | 8 + tests/debug-tts-cossim.py | 408 + tests/debug-tts-cossim.sh | 8 + tests/tts-CPU-BF16.log | 143 + tests/tts-CPU-F32.log | 143 + tests/tts-CPU-Q4_K_M.log | 143 + tests/tts-CPU-Q8_0.log | 143 + tests/tts-CUDA0-BF16.log | 143 + tests/tts-CUDA0-F32.log | 143 + tests/tts-CUDA0-Q4_K_M.log | 143 + tests/tts-CUDA0-Q8_0.log | 143 + tests/tts-Vulkan0-BF16.log | 143 + tests/tts-Vulkan0-F32.log | 143 + tests/tts-Vulkan0-Q4_K_M.log | 143 + tests/tts-Vulkan0-Q8_0.log | 143 + tools/omnivoice-codec.cpp | 204 + tools/omnivoice-tts.cpp | 932 + tools/quantize.cpp | 447 + tools/tts-server.cpp | 132 + tools/version.cmake | 38 + update.cmd | 6 + update.sh | 6 + vendor/cpp-httplib/CMakeLists.txt | 15 + vendor/cpp-httplib/LICENSE | 22 + vendor/cpp-httplib/httplib.cpp | 16526 ++++++++++++++ vendor/cpp-httplib/httplib.h | 3903 ++++ vendor/yyjson/yyjson.c | 11224 +++++++++ vendor/yyjson/yyjson.h | 8349 +++++++ 2105 files changed, 499091 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 buildall.cmd create mode 100755 buildall.sh create mode 100755 buildcpu.sh create mode 100644 buildcuda.cmd create mode 100755 buildcuda.sh create mode 100755 buildsycl.sh create mode 100755 buildtermux.sh create mode 100644 buildvulkan.cmd create mode 100755 buildvulkan.sh create mode 100755 checkpoints.sh create mode 100755 convert.py create mode 100644 docker-compose.yml create mode 100644 docs/ARCHITECTURE.md create mode 100755 examples/client.sh create mode 100644 examples/clone.cmd create mode 100755 examples/clone.sh create mode 100644 examples/freeman.rvq create mode 100644 examples/freeman.txt create mode 100644 examples/prompt.txt create mode 100755 examples/server.sh create mode 100644 examples/tts.cmd create mode 100755 examples/tts.sh create mode 100755 format.sh create mode 100644 ggml/.editorconfig create mode 100644 ggml/.github/pull_request_template.md create mode 100644 ggml/.github/workflows/ci.yml create mode 100644 ggml/.github/workflows/release.yml create mode 100644 ggml/.gitignore create mode 100644 ggml/.gitmodules create mode 100644 ggml/AUTHORS create mode 100644 ggml/CMakeLists.txt create mode 100644 ggml/CONTRIBUTING.md create mode 100644 ggml/LICENSE create mode 100644 ggml/README.md create mode 100644 ggml/ci/run.sh create mode 100644 ggml/cmake/FindNCCL.cmake create mode 100644 ggml/cmake/GitVars.cmake create mode 100644 ggml/cmake/common.cmake create mode 100644 ggml/cmake/ggml-config.cmake.in create mode 100644 ggml/docs/gguf.md create mode 100644 ggml/examples/CMakeLists.txt create mode 100644 ggml/examples/common-ggml.cpp create mode 100644 ggml/examples/common-ggml.h create mode 100644 ggml/examples/common.cpp create mode 100644 ggml/examples/common.h create mode 100644 ggml/examples/gpt-2/CMakeLists.txt create mode 100644 ggml/examples/gpt-2/README.md create mode 100644 ggml/examples/gpt-2/convert-cerebras-to-ggml.py create mode 100644 ggml/examples/gpt-2/convert-ckpt-to-ggml.py create mode 100644 ggml/examples/gpt-2/convert-h5-to-ggml.py create mode 100755 ggml/examples/gpt-2/download-ggml-model.sh create mode 100755 ggml/examples/gpt-2/download-model.sh create mode 100644 ggml/examples/gpt-2/main-alloc.cpp create mode 100644 ggml/examples/gpt-2/main-backend.cpp create mode 100644 ggml/examples/gpt-2/main-batched.cpp create mode 100644 ggml/examples/gpt-2/main-ctx.cpp create mode 100644 ggml/examples/gpt-2/main-sched.cpp create mode 100644 ggml/examples/gpt-2/quantize.cpp create mode 100644 ggml/examples/gpt-j/CMakeLists.txt create mode 100644 ggml/examples/gpt-j/README.md create mode 100644 ggml/examples/gpt-j/convert-h5-to-ggml.py create mode 100755 ggml/examples/gpt-j/download-ggml-model.sh create mode 100755 ggml/examples/gpt-j/download-model.sh create mode 100644 ggml/examples/gpt-j/main.cpp create mode 100644 ggml/examples/gpt-j/quantize.cpp create mode 100644 ggml/examples/magika/CMakeLists.txt create mode 100644 ggml/examples/magika/README.md create mode 100644 ggml/examples/magika/convert.py create mode 100644 ggml/examples/magika/main.cpp create mode 100644 ggml/examples/mnist/.gitignore create mode 100644 ggml/examples/mnist/CMakeLists.txt create mode 100644 ggml/examples/mnist/README.md create mode 100644 ggml/examples/mnist/mnist-common.cpp create mode 100644 ggml/examples/mnist/mnist-common.h create mode 100644 ggml/examples/mnist/mnist-eval.cpp create mode 100755 ggml/examples/mnist/mnist-train-cnn.py create mode 100644 ggml/examples/mnist/mnist-train-fc.py create mode 100644 ggml/examples/mnist/mnist-train.cpp create mode 100644 ggml/examples/mnist/server.py create mode 100644 ggml/examples/perf-metal/CMakeLists.txt create mode 100644 ggml/examples/perf-metal/perf-metal.cpp create mode 100644 ggml/examples/prompts/dolly-v2.txt create mode 100644 ggml/examples/prompts/gpt-2-chinese.txt create mode 100644 ggml/examples/prompts/gpt-2.txt create mode 100644 ggml/examples/prompts/gpt-j.txt create mode 100644 ggml/examples/prompts/gpt-neox-japanese.txt create mode 100644 ggml/examples/prompts/gpt-neox.txt create mode 100644 ggml/examples/prompts/polyglot-ko.txt create mode 100644 ggml/examples/prompts/replit.txt create mode 100644 ggml/examples/prompts/starcoder.txt create mode 100644 ggml/examples/prompts/test-cases.txt create mode 100644 ggml/examples/prompts/tokenize_huggingface.py create mode 100644 ggml/examples/prompts/whisper.txt create mode 100644 ggml/examples/python/README.md create mode 100644 ggml/examples/python/api.h create mode 100644 ggml/examples/python/example_add_quant.py create mode 100644 ggml/examples/python/example_test_all_quants.py create mode 100644 ggml/examples/python/ggml/__init__.py create mode 100644 ggml/examples/python/ggml/__init__.pyi create mode 100644 ggml/examples/python/ggml/cffi.py create mode 100644 ggml/examples/python/ggml/ffi/__init__.pyi create mode 100644 ggml/examples/python/ggml/utils.py create mode 100644 ggml/examples/python/regenerate.py create mode 100644 ggml/examples/python/stubs.py create mode 100644 ggml/examples/python/test_tensor.py create mode 100644 ggml/examples/sam/CMakeLists.txt create mode 100644 ggml/examples/sam/README.md create mode 100644 ggml/examples/sam/convert-pth-to-ggml.py create mode 100644 ggml/examples/sam/example.jpg create mode 100644 ggml/examples/sam/sam.cpp create mode 100644 ggml/examples/simple/CMakeLists.txt create mode 100644 ggml/examples/simple/README.md create mode 100644 ggml/examples/simple/simple-backend.cpp create mode 100644 ggml/examples/simple/simple-ctx.cpp create mode 100644 ggml/examples/stb_image.h create mode 100644 ggml/examples/stb_image_write.h create mode 100644 ggml/examples/test-cmake/CMakeLists.txt create mode 100644 ggml/examples/test-cmake/README.md create mode 100644 ggml/examples/test-cmake/test-cmake.cpp create mode 100644 ggml/examples/yolo/CMakeLists.txt create mode 100644 ggml/examples/yolo/README.md create mode 100755 ggml/examples/yolo/convert-yolov3-tiny.py create mode 100644 ggml/examples/yolo/data/coco.names create mode 100644 ggml/examples/yolo/data/labels/100_0.png create mode 100644 ggml/examples/yolo/data/labels/100_1.png create mode 100644 ggml/examples/yolo/data/labels/100_2.png create mode 100644 ggml/examples/yolo/data/labels/100_3.png create mode 100644 ggml/examples/yolo/data/labels/100_4.png create mode 100644 ggml/examples/yolo/data/labels/100_5.png create mode 100644 ggml/examples/yolo/data/labels/100_6.png create mode 100644 ggml/examples/yolo/data/labels/100_7.png create mode 100644 ggml/examples/yolo/data/labels/101_0.png create mode 100644 ggml/examples/yolo/data/labels/101_1.png create mode 100644 ggml/examples/yolo/data/labels/101_2.png create mode 100644 ggml/examples/yolo/data/labels/101_3.png create mode 100644 ggml/examples/yolo/data/labels/101_4.png create mode 100644 ggml/examples/yolo/data/labels/101_5.png create mode 100644 ggml/examples/yolo/data/labels/101_6.png create mode 100644 ggml/examples/yolo/data/labels/101_7.png create mode 100644 ggml/examples/yolo/data/labels/102_0.png create mode 100644 ggml/examples/yolo/data/labels/102_1.png create mode 100644 ggml/examples/yolo/data/labels/102_2.png create mode 100644 ggml/examples/yolo/data/labels/102_3.png create mode 100644 ggml/examples/yolo/data/labels/102_4.png create mode 100644 ggml/examples/yolo/data/labels/102_5.png create mode 100644 ggml/examples/yolo/data/labels/102_6.png create mode 100644 ggml/examples/yolo/data/labels/102_7.png create mode 100644 ggml/examples/yolo/data/labels/103_0.png create mode 100644 ggml/examples/yolo/data/labels/103_1.png create mode 100644 ggml/examples/yolo/data/labels/103_2.png create mode 100644 ggml/examples/yolo/data/labels/103_3.png create mode 100644 ggml/examples/yolo/data/labels/103_4.png create mode 100644 ggml/examples/yolo/data/labels/103_5.png create mode 100644 ggml/examples/yolo/data/labels/103_6.png create mode 100644 ggml/examples/yolo/data/labels/103_7.png create mode 100644 ggml/examples/yolo/data/labels/104_0.png create mode 100644 ggml/examples/yolo/data/labels/104_1.png create mode 100644 ggml/examples/yolo/data/labels/104_2.png create mode 100644 ggml/examples/yolo/data/labels/104_3.png create mode 100644 ggml/examples/yolo/data/labels/104_4.png create mode 100644 ggml/examples/yolo/data/labels/104_5.png create mode 100644 ggml/examples/yolo/data/labels/104_6.png create mode 100644 ggml/examples/yolo/data/labels/104_7.png create mode 100644 ggml/examples/yolo/data/labels/105_0.png create mode 100644 ggml/examples/yolo/data/labels/105_1.png create mode 100644 ggml/examples/yolo/data/labels/105_2.png create mode 100644 ggml/examples/yolo/data/labels/105_3.png create mode 100644 ggml/examples/yolo/data/labels/105_4.png create mode 100644 ggml/examples/yolo/data/labels/105_5.png create mode 100644 ggml/examples/yolo/data/labels/105_6.png create mode 100644 ggml/examples/yolo/data/labels/105_7.png create mode 100644 ggml/examples/yolo/data/labels/106_0.png create mode 100644 ggml/examples/yolo/data/labels/106_1.png create mode 100644 ggml/examples/yolo/data/labels/106_2.png create mode 100644 ggml/examples/yolo/data/labels/106_3.png create mode 100644 ggml/examples/yolo/data/labels/106_4.png create mode 100644 ggml/examples/yolo/data/labels/106_5.png create mode 100644 ggml/examples/yolo/data/labels/106_6.png create mode 100644 ggml/examples/yolo/data/labels/106_7.png create mode 100644 ggml/examples/yolo/data/labels/107_0.png create mode 100644 ggml/examples/yolo/data/labels/107_1.png create mode 100644 ggml/examples/yolo/data/labels/107_2.png create mode 100644 ggml/examples/yolo/data/labels/107_3.png create mode 100644 ggml/examples/yolo/data/labels/107_4.png create mode 100644 ggml/examples/yolo/data/labels/107_5.png create mode 100644 ggml/examples/yolo/data/labels/107_6.png create mode 100644 ggml/examples/yolo/data/labels/107_7.png create mode 100644 ggml/examples/yolo/data/labels/108_0.png create mode 100644 ggml/examples/yolo/data/labels/108_1.png create mode 100644 ggml/examples/yolo/data/labels/108_2.png create mode 100644 ggml/examples/yolo/data/labels/108_3.png create mode 100644 ggml/examples/yolo/data/labels/108_4.png create mode 100644 ggml/examples/yolo/data/labels/108_5.png create mode 100644 ggml/examples/yolo/data/labels/108_6.png create mode 100644 ggml/examples/yolo/data/labels/108_7.png create mode 100644 ggml/examples/yolo/data/labels/109_0.png create mode 100644 ggml/examples/yolo/data/labels/109_1.png create mode 100644 ggml/examples/yolo/data/labels/109_2.png create mode 100644 ggml/examples/yolo/data/labels/109_3.png create mode 100644 ggml/examples/yolo/data/labels/109_4.png create mode 100644 ggml/examples/yolo/data/labels/109_5.png create mode 100644 ggml/examples/yolo/data/labels/109_6.png create mode 100644 ggml/examples/yolo/data/labels/109_7.png create mode 100644 ggml/examples/yolo/data/labels/110_0.png create mode 100644 ggml/examples/yolo/data/labels/110_1.png create mode 100644 ggml/examples/yolo/data/labels/110_2.png create mode 100644 ggml/examples/yolo/data/labels/110_3.png create mode 100644 ggml/examples/yolo/data/labels/110_4.png create mode 100644 ggml/examples/yolo/data/labels/110_5.png create mode 100644 ggml/examples/yolo/data/labels/110_6.png create mode 100644 ggml/examples/yolo/data/labels/110_7.png create mode 100644 ggml/examples/yolo/data/labels/111_0.png create mode 100644 ggml/examples/yolo/data/labels/111_1.png create mode 100644 ggml/examples/yolo/data/labels/111_2.png create mode 100644 ggml/examples/yolo/data/labels/111_3.png create mode 100644 ggml/examples/yolo/data/labels/111_4.png create mode 100644 ggml/examples/yolo/data/labels/111_5.png create mode 100644 ggml/examples/yolo/data/labels/111_6.png create mode 100644 ggml/examples/yolo/data/labels/111_7.png create mode 100644 ggml/examples/yolo/data/labels/112_0.png create mode 100644 ggml/examples/yolo/data/labels/112_1.png create mode 100644 ggml/examples/yolo/data/labels/112_2.png create mode 100644 ggml/examples/yolo/data/labels/112_3.png create mode 100644 ggml/examples/yolo/data/labels/112_4.png create mode 100644 ggml/examples/yolo/data/labels/112_5.png create mode 100644 ggml/examples/yolo/data/labels/112_6.png create mode 100644 ggml/examples/yolo/data/labels/112_7.png create mode 100644 ggml/examples/yolo/data/labels/113_0.png create mode 100644 ggml/examples/yolo/data/labels/113_1.png create mode 100644 ggml/examples/yolo/data/labels/113_2.png create mode 100644 ggml/examples/yolo/data/labels/113_3.png create mode 100644 ggml/examples/yolo/data/labels/113_4.png create mode 100644 ggml/examples/yolo/data/labels/113_5.png create mode 100644 ggml/examples/yolo/data/labels/113_6.png create mode 100644 ggml/examples/yolo/data/labels/113_7.png create mode 100644 ggml/examples/yolo/data/labels/114_0.png create mode 100644 ggml/examples/yolo/data/labels/114_1.png create mode 100644 ggml/examples/yolo/data/labels/114_2.png create mode 100644 ggml/examples/yolo/data/labels/114_3.png create mode 100644 ggml/examples/yolo/data/labels/114_4.png create mode 100644 ggml/examples/yolo/data/labels/114_5.png create mode 100644 ggml/examples/yolo/data/labels/114_6.png create mode 100644 ggml/examples/yolo/data/labels/114_7.png create mode 100644 ggml/examples/yolo/data/labels/115_0.png create mode 100644 ggml/examples/yolo/data/labels/115_1.png create mode 100644 ggml/examples/yolo/data/labels/115_2.png create mode 100644 ggml/examples/yolo/data/labels/115_3.png create mode 100644 ggml/examples/yolo/data/labels/115_4.png create mode 100644 ggml/examples/yolo/data/labels/115_5.png create mode 100644 ggml/examples/yolo/data/labels/115_6.png create mode 100644 ggml/examples/yolo/data/labels/115_7.png create mode 100644 ggml/examples/yolo/data/labels/116_0.png create mode 100644 ggml/examples/yolo/data/labels/116_1.png create mode 100644 ggml/examples/yolo/data/labels/116_2.png create mode 100644 ggml/examples/yolo/data/labels/116_3.png create mode 100644 ggml/examples/yolo/data/labels/116_4.png create mode 100644 ggml/examples/yolo/data/labels/116_5.png create mode 100644 ggml/examples/yolo/data/labels/116_6.png create mode 100644 ggml/examples/yolo/data/labels/116_7.png create mode 100644 ggml/examples/yolo/data/labels/117_0.png create mode 100644 ggml/examples/yolo/data/labels/117_1.png create mode 100644 ggml/examples/yolo/data/labels/117_2.png create mode 100644 ggml/examples/yolo/data/labels/117_3.png create mode 100644 ggml/examples/yolo/data/labels/117_4.png create mode 100644 ggml/examples/yolo/data/labels/117_5.png create mode 100644 ggml/examples/yolo/data/labels/117_6.png create mode 100644 ggml/examples/yolo/data/labels/117_7.png create mode 100644 ggml/examples/yolo/data/labels/118_0.png create mode 100644 ggml/examples/yolo/data/labels/118_1.png create mode 100644 ggml/examples/yolo/data/labels/118_2.png create mode 100644 ggml/examples/yolo/data/labels/118_3.png create mode 100644 ggml/examples/yolo/data/labels/118_4.png create mode 100644 ggml/examples/yolo/data/labels/118_5.png create mode 100644 ggml/examples/yolo/data/labels/118_6.png create mode 100644 ggml/examples/yolo/data/labels/118_7.png create mode 100644 ggml/examples/yolo/data/labels/119_0.png create mode 100644 ggml/examples/yolo/data/labels/119_1.png create mode 100644 ggml/examples/yolo/data/labels/119_2.png create mode 100644 ggml/examples/yolo/data/labels/119_3.png create mode 100644 ggml/examples/yolo/data/labels/119_4.png create mode 100644 ggml/examples/yolo/data/labels/119_5.png create mode 100644 ggml/examples/yolo/data/labels/119_6.png create mode 100644 ggml/examples/yolo/data/labels/119_7.png create mode 100644 ggml/examples/yolo/data/labels/120_0.png create mode 100644 ggml/examples/yolo/data/labels/120_1.png create mode 100644 ggml/examples/yolo/data/labels/120_2.png create mode 100644 ggml/examples/yolo/data/labels/120_3.png create mode 100644 ggml/examples/yolo/data/labels/120_4.png create mode 100644 ggml/examples/yolo/data/labels/120_5.png create mode 100644 ggml/examples/yolo/data/labels/120_6.png create mode 100644 ggml/examples/yolo/data/labels/120_7.png create mode 100644 ggml/examples/yolo/data/labels/121_0.png create mode 100644 ggml/examples/yolo/data/labels/121_1.png create mode 100644 ggml/examples/yolo/data/labels/121_2.png create mode 100644 ggml/examples/yolo/data/labels/121_3.png create mode 100644 ggml/examples/yolo/data/labels/121_4.png create mode 100644 ggml/examples/yolo/data/labels/121_5.png create mode 100644 ggml/examples/yolo/data/labels/121_6.png create mode 100644 ggml/examples/yolo/data/labels/121_7.png create mode 100644 ggml/examples/yolo/data/labels/122_0.png create mode 100644 ggml/examples/yolo/data/labels/122_1.png create mode 100644 ggml/examples/yolo/data/labels/122_2.png create mode 100644 ggml/examples/yolo/data/labels/122_3.png create mode 100644 ggml/examples/yolo/data/labels/122_4.png create mode 100644 ggml/examples/yolo/data/labels/122_5.png create mode 100644 ggml/examples/yolo/data/labels/122_6.png create mode 100644 ggml/examples/yolo/data/labels/122_7.png create mode 100644 ggml/examples/yolo/data/labels/123_0.png create mode 100644 ggml/examples/yolo/data/labels/123_1.png create mode 100644 ggml/examples/yolo/data/labels/123_2.png create mode 100644 ggml/examples/yolo/data/labels/123_3.png create mode 100644 ggml/examples/yolo/data/labels/123_4.png create mode 100644 ggml/examples/yolo/data/labels/123_5.png create mode 100644 ggml/examples/yolo/data/labels/123_6.png create mode 100644 ggml/examples/yolo/data/labels/123_7.png create mode 100644 ggml/examples/yolo/data/labels/124_0.png create mode 100644 ggml/examples/yolo/data/labels/124_1.png create mode 100644 ggml/examples/yolo/data/labels/124_2.png create mode 100644 ggml/examples/yolo/data/labels/124_3.png create mode 100644 ggml/examples/yolo/data/labels/124_4.png create mode 100644 ggml/examples/yolo/data/labels/124_5.png create mode 100644 ggml/examples/yolo/data/labels/124_6.png create mode 100644 ggml/examples/yolo/data/labels/124_7.png create mode 100644 ggml/examples/yolo/data/labels/125_0.png create mode 100644 ggml/examples/yolo/data/labels/125_1.png create mode 100644 ggml/examples/yolo/data/labels/125_2.png create mode 100644 ggml/examples/yolo/data/labels/125_3.png create mode 100644 ggml/examples/yolo/data/labels/125_4.png create mode 100644 ggml/examples/yolo/data/labels/125_5.png create mode 100644 ggml/examples/yolo/data/labels/125_6.png create mode 100644 ggml/examples/yolo/data/labels/125_7.png create mode 100644 ggml/examples/yolo/data/labels/126_0.png create mode 100644 ggml/examples/yolo/data/labels/126_1.png create mode 100644 ggml/examples/yolo/data/labels/126_2.png create mode 100644 ggml/examples/yolo/data/labels/126_3.png create mode 100644 ggml/examples/yolo/data/labels/126_4.png create mode 100644 ggml/examples/yolo/data/labels/126_5.png create mode 100644 ggml/examples/yolo/data/labels/126_6.png create mode 100644 ggml/examples/yolo/data/labels/126_7.png create mode 100644 ggml/examples/yolo/data/labels/32_0.png create mode 100644 ggml/examples/yolo/data/labels/32_1.png create mode 100644 ggml/examples/yolo/data/labels/32_2.png create mode 100644 ggml/examples/yolo/data/labels/32_3.png create mode 100644 ggml/examples/yolo/data/labels/32_4.png create mode 100644 ggml/examples/yolo/data/labels/32_5.png create mode 100644 ggml/examples/yolo/data/labels/32_6.png create mode 100644 ggml/examples/yolo/data/labels/32_7.png create mode 100644 ggml/examples/yolo/data/labels/33_0.png create mode 100644 ggml/examples/yolo/data/labels/33_1.png create mode 100644 ggml/examples/yolo/data/labels/33_2.png create mode 100644 ggml/examples/yolo/data/labels/33_3.png create mode 100644 ggml/examples/yolo/data/labels/33_4.png create mode 100644 ggml/examples/yolo/data/labels/33_5.png create mode 100644 ggml/examples/yolo/data/labels/33_6.png create mode 100644 ggml/examples/yolo/data/labels/33_7.png create mode 100644 ggml/examples/yolo/data/labels/34_0.png create mode 100644 ggml/examples/yolo/data/labels/34_1.png create mode 100644 ggml/examples/yolo/data/labels/34_2.png create mode 100644 ggml/examples/yolo/data/labels/34_3.png create mode 100644 ggml/examples/yolo/data/labels/34_4.png create mode 100644 ggml/examples/yolo/data/labels/34_5.png create mode 100644 ggml/examples/yolo/data/labels/34_6.png create mode 100644 ggml/examples/yolo/data/labels/34_7.png create mode 100644 ggml/examples/yolo/data/labels/35_0.png create mode 100644 ggml/examples/yolo/data/labels/35_1.png create mode 100644 ggml/examples/yolo/data/labels/35_2.png create mode 100644 ggml/examples/yolo/data/labels/35_3.png create mode 100644 ggml/examples/yolo/data/labels/35_4.png create mode 100644 ggml/examples/yolo/data/labels/35_5.png create mode 100644 ggml/examples/yolo/data/labels/35_6.png create mode 100644 ggml/examples/yolo/data/labels/35_7.png create mode 100644 ggml/examples/yolo/data/labels/36_0.png create mode 100644 ggml/examples/yolo/data/labels/36_1.png create mode 100644 ggml/examples/yolo/data/labels/36_2.png create mode 100644 ggml/examples/yolo/data/labels/36_3.png create mode 100644 ggml/examples/yolo/data/labels/36_4.png create mode 100644 ggml/examples/yolo/data/labels/36_5.png create mode 100644 ggml/examples/yolo/data/labels/36_6.png create mode 100644 ggml/examples/yolo/data/labels/36_7.png create mode 100644 ggml/examples/yolo/data/labels/37_0.png create mode 100644 ggml/examples/yolo/data/labels/37_1.png create mode 100644 ggml/examples/yolo/data/labels/37_2.png create mode 100644 ggml/examples/yolo/data/labels/37_3.png create mode 100644 ggml/examples/yolo/data/labels/37_4.png create mode 100644 ggml/examples/yolo/data/labels/37_5.png create mode 100644 ggml/examples/yolo/data/labels/37_6.png create mode 100644 ggml/examples/yolo/data/labels/37_7.png create mode 100644 ggml/examples/yolo/data/labels/38_0.png create mode 100644 ggml/examples/yolo/data/labels/38_1.png create mode 100644 ggml/examples/yolo/data/labels/38_2.png create mode 100644 ggml/examples/yolo/data/labels/38_3.png create mode 100644 ggml/examples/yolo/data/labels/38_4.png create mode 100644 ggml/examples/yolo/data/labels/38_5.png create mode 100644 ggml/examples/yolo/data/labels/38_6.png create mode 100644 ggml/examples/yolo/data/labels/38_7.png create mode 100644 ggml/examples/yolo/data/labels/39_0.png create mode 100644 ggml/examples/yolo/data/labels/39_1.png create mode 100644 ggml/examples/yolo/data/labels/39_2.png create mode 100644 ggml/examples/yolo/data/labels/39_3.png create mode 100644 ggml/examples/yolo/data/labels/39_4.png create mode 100644 ggml/examples/yolo/data/labels/39_5.png create mode 100644 ggml/examples/yolo/data/labels/39_6.png create mode 100644 ggml/examples/yolo/data/labels/39_7.png create mode 100644 ggml/examples/yolo/data/labels/40_0.png create mode 100644 ggml/examples/yolo/data/labels/40_1.png create mode 100644 ggml/examples/yolo/data/labels/40_2.png create mode 100644 ggml/examples/yolo/data/labels/40_3.png create mode 100644 ggml/examples/yolo/data/labels/40_4.png create mode 100644 ggml/examples/yolo/data/labels/40_5.png create mode 100644 ggml/examples/yolo/data/labels/40_6.png create mode 100644 ggml/examples/yolo/data/labels/40_7.png create mode 100644 ggml/examples/yolo/data/labels/41_0.png create mode 100644 ggml/examples/yolo/data/labels/41_1.png create mode 100644 ggml/examples/yolo/data/labels/41_2.png create mode 100644 ggml/examples/yolo/data/labels/41_3.png create mode 100644 ggml/examples/yolo/data/labels/41_4.png create mode 100644 ggml/examples/yolo/data/labels/41_5.png create mode 100644 ggml/examples/yolo/data/labels/41_6.png create mode 100644 ggml/examples/yolo/data/labels/41_7.png create mode 100644 ggml/examples/yolo/data/labels/42_0.png create mode 100644 ggml/examples/yolo/data/labels/42_1.png create mode 100644 ggml/examples/yolo/data/labels/42_2.png create mode 100644 ggml/examples/yolo/data/labels/42_3.png create mode 100644 ggml/examples/yolo/data/labels/42_4.png create mode 100644 ggml/examples/yolo/data/labels/42_5.png create mode 100644 ggml/examples/yolo/data/labels/42_6.png create mode 100644 ggml/examples/yolo/data/labels/42_7.png create mode 100644 ggml/examples/yolo/data/labels/43_0.png create mode 100644 ggml/examples/yolo/data/labels/43_1.png create mode 100644 ggml/examples/yolo/data/labels/43_2.png create mode 100644 ggml/examples/yolo/data/labels/43_3.png create mode 100644 ggml/examples/yolo/data/labels/43_4.png create mode 100644 ggml/examples/yolo/data/labels/43_5.png create mode 100644 ggml/examples/yolo/data/labels/43_6.png create mode 100644 ggml/examples/yolo/data/labels/43_7.png create mode 100644 ggml/examples/yolo/data/labels/44_0.png create mode 100644 ggml/examples/yolo/data/labels/44_1.png create mode 100644 ggml/examples/yolo/data/labels/44_2.png create mode 100644 ggml/examples/yolo/data/labels/44_3.png create mode 100644 ggml/examples/yolo/data/labels/44_4.png create mode 100644 ggml/examples/yolo/data/labels/44_5.png create mode 100644 ggml/examples/yolo/data/labels/44_6.png create mode 100644 ggml/examples/yolo/data/labels/44_7.png create mode 100644 ggml/examples/yolo/data/labels/45_0.png create mode 100644 ggml/examples/yolo/data/labels/45_1.png create mode 100644 ggml/examples/yolo/data/labels/45_2.png create mode 100644 ggml/examples/yolo/data/labels/45_3.png create mode 100644 ggml/examples/yolo/data/labels/45_4.png create mode 100644 ggml/examples/yolo/data/labels/45_5.png create mode 100644 ggml/examples/yolo/data/labels/45_6.png create mode 100644 ggml/examples/yolo/data/labels/45_7.png create mode 100644 ggml/examples/yolo/data/labels/46_0.png create mode 100644 ggml/examples/yolo/data/labels/46_1.png create mode 100644 ggml/examples/yolo/data/labels/46_2.png create mode 100644 ggml/examples/yolo/data/labels/46_3.png create mode 100644 ggml/examples/yolo/data/labels/46_4.png create mode 100644 ggml/examples/yolo/data/labels/46_5.png create mode 100644 ggml/examples/yolo/data/labels/46_6.png create mode 100644 ggml/examples/yolo/data/labels/46_7.png create mode 100644 ggml/examples/yolo/data/labels/47_0.png create mode 100644 ggml/examples/yolo/data/labels/47_1.png create mode 100644 ggml/examples/yolo/data/labels/47_2.png create mode 100644 ggml/examples/yolo/data/labels/47_3.png create mode 100644 ggml/examples/yolo/data/labels/47_4.png create mode 100644 ggml/examples/yolo/data/labels/47_5.png create mode 100644 ggml/examples/yolo/data/labels/47_6.png create mode 100644 ggml/examples/yolo/data/labels/47_7.png create mode 100644 ggml/examples/yolo/data/labels/48_0.png create mode 100644 ggml/examples/yolo/data/labels/48_1.png create mode 100644 ggml/examples/yolo/data/labels/48_2.png create mode 100644 ggml/examples/yolo/data/labels/48_3.png create mode 100644 ggml/examples/yolo/data/labels/48_4.png create mode 100644 ggml/examples/yolo/data/labels/48_5.png create mode 100644 ggml/examples/yolo/data/labels/48_6.png create mode 100644 ggml/examples/yolo/data/labels/48_7.png create mode 100644 ggml/examples/yolo/data/labels/49_0.png create mode 100644 ggml/examples/yolo/data/labels/49_1.png create mode 100644 ggml/examples/yolo/data/labels/49_2.png create mode 100644 ggml/examples/yolo/data/labels/49_3.png create mode 100644 ggml/examples/yolo/data/labels/49_4.png create mode 100644 ggml/examples/yolo/data/labels/49_5.png create mode 100644 ggml/examples/yolo/data/labels/49_6.png create mode 100644 ggml/examples/yolo/data/labels/49_7.png create mode 100644 ggml/examples/yolo/data/labels/50_0.png create mode 100644 ggml/examples/yolo/data/labels/50_1.png create mode 100644 ggml/examples/yolo/data/labels/50_2.png create mode 100644 ggml/examples/yolo/data/labels/50_3.png create mode 100644 ggml/examples/yolo/data/labels/50_4.png create mode 100644 ggml/examples/yolo/data/labels/50_5.png create mode 100644 ggml/examples/yolo/data/labels/50_6.png create mode 100644 ggml/examples/yolo/data/labels/50_7.png create mode 100644 ggml/examples/yolo/data/labels/51_0.png create mode 100644 ggml/examples/yolo/data/labels/51_1.png create mode 100644 ggml/examples/yolo/data/labels/51_2.png create mode 100644 ggml/examples/yolo/data/labels/51_3.png create mode 100644 ggml/examples/yolo/data/labels/51_4.png create mode 100644 ggml/examples/yolo/data/labels/51_5.png create mode 100644 ggml/examples/yolo/data/labels/51_6.png create mode 100644 ggml/examples/yolo/data/labels/51_7.png create mode 100644 ggml/examples/yolo/data/labels/52_0.png create mode 100644 ggml/examples/yolo/data/labels/52_1.png create mode 100644 ggml/examples/yolo/data/labels/52_2.png create mode 100644 ggml/examples/yolo/data/labels/52_3.png create mode 100644 ggml/examples/yolo/data/labels/52_4.png create mode 100644 ggml/examples/yolo/data/labels/52_5.png create mode 100644 ggml/examples/yolo/data/labels/52_6.png create mode 100644 ggml/examples/yolo/data/labels/52_7.png create mode 100644 ggml/examples/yolo/data/labels/53_0.png create mode 100644 ggml/examples/yolo/data/labels/53_1.png create mode 100644 ggml/examples/yolo/data/labels/53_2.png create mode 100644 ggml/examples/yolo/data/labels/53_3.png create mode 100644 ggml/examples/yolo/data/labels/53_4.png create mode 100644 ggml/examples/yolo/data/labels/53_5.png create mode 100644 ggml/examples/yolo/data/labels/53_6.png create mode 100644 ggml/examples/yolo/data/labels/53_7.png create mode 100644 ggml/examples/yolo/data/labels/54_0.png create mode 100644 ggml/examples/yolo/data/labels/54_1.png create mode 100644 ggml/examples/yolo/data/labels/54_2.png create mode 100644 ggml/examples/yolo/data/labels/54_3.png create mode 100644 ggml/examples/yolo/data/labels/54_4.png create mode 100644 ggml/examples/yolo/data/labels/54_5.png create mode 100644 ggml/examples/yolo/data/labels/54_6.png create mode 100644 ggml/examples/yolo/data/labels/54_7.png create mode 100644 ggml/examples/yolo/data/labels/55_0.png create mode 100644 ggml/examples/yolo/data/labels/55_1.png create mode 100644 ggml/examples/yolo/data/labels/55_2.png create mode 100644 ggml/examples/yolo/data/labels/55_3.png create mode 100644 ggml/examples/yolo/data/labels/55_4.png create mode 100644 ggml/examples/yolo/data/labels/55_5.png create mode 100644 ggml/examples/yolo/data/labels/55_6.png create mode 100644 ggml/examples/yolo/data/labels/55_7.png create mode 100644 ggml/examples/yolo/data/labels/56_0.png create mode 100644 ggml/examples/yolo/data/labels/56_1.png create mode 100644 ggml/examples/yolo/data/labels/56_2.png create mode 100644 ggml/examples/yolo/data/labels/56_3.png create mode 100644 ggml/examples/yolo/data/labels/56_4.png create mode 100644 ggml/examples/yolo/data/labels/56_5.png create mode 100644 ggml/examples/yolo/data/labels/56_6.png create mode 100644 ggml/examples/yolo/data/labels/56_7.png create mode 100644 ggml/examples/yolo/data/labels/57_0.png create mode 100644 ggml/examples/yolo/data/labels/57_1.png create mode 100644 ggml/examples/yolo/data/labels/57_2.png create mode 100644 ggml/examples/yolo/data/labels/57_3.png create mode 100644 ggml/examples/yolo/data/labels/57_4.png create mode 100644 ggml/examples/yolo/data/labels/57_5.png create mode 100644 ggml/examples/yolo/data/labels/57_6.png create mode 100644 ggml/examples/yolo/data/labels/57_7.png create mode 100644 ggml/examples/yolo/data/labels/58_0.png create mode 100644 ggml/examples/yolo/data/labels/58_1.png create mode 100644 ggml/examples/yolo/data/labels/58_2.png create mode 100644 ggml/examples/yolo/data/labels/58_3.png create mode 100644 ggml/examples/yolo/data/labels/58_4.png create mode 100644 ggml/examples/yolo/data/labels/58_5.png create mode 100644 ggml/examples/yolo/data/labels/58_6.png create mode 100644 ggml/examples/yolo/data/labels/58_7.png create mode 100644 ggml/examples/yolo/data/labels/59_0.png create mode 100644 ggml/examples/yolo/data/labels/59_1.png create mode 100644 ggml/examples/yolo/data/labels/59_2.png create mode 100644 ggml/examples/yolo/data/labels/59_3.png create mode 100644 ggml/examples/yolo/data/labels/59_4.png create mode 100644 ggml/examples/yolo/data/labels/59_5.png create mode 100644 ggml/examples/yolo/data/labels/59_6.png create mode 100644 ggml/examples/yolo/data/labels/59_7.png create mode 100644 ggml/examples/yolo/data/labels/60_0.png create mode 100644 ggml/examples/yolo/data/labels/60_1.png create mode 100644 ggml/examples/yolo/data/labels/60_2.png create mode 100644 ggml/examples/yolo/data/labels/60_3.png create mode 100644 ggml/examples/yolo/data/labels/60_4.png create mode 100644 ggml/examples/yolo/data/labels/60_5.png create mode 100644 ggml/examples/yolo/data/labels/60_6.png create mode 100644 ggml/examples/yolo/data/labels/60_7.png create mode 100644 ggml/examples/yolo/data/labels/61_0.png create mode 100644 ggml/examples/yolo/data/labels/61_1.png create mode 100644 ggml/examples/yolo/data/labels/61_2.png create mode 100644 ggml/examples/yolo/data/labels/61_3.png create mode 100644 ggml/examples/yolo/data/labels/61_4.png create mode 100644 ggml/examples/yolo/data/labels/61_5.png create mode 100644 ggml/examples/yolo/data/labels/61_6.png create mode 100644 ggml/examples/yolo/data/labels/61_7.png create mode 100644 ggml/examples/yolo/data/labels/62_0.png create mode 100644 ggml/examples/yolo/data/labels/62_1.png create mode 100644 ggml/examples/yolo/data/labels/62_2.png create mode 100644 ggml/examples/yolo/data/labels/62_3.png create mode 100644 ggml/examples/yolo/data/labels/62_4.png create mode 100644 ggml/examples/yolo/data/labels/62_5.png create mode 100644 ggml/examples/yolo/data/labels/62_6.png create mode 100644 ggml/examples/yolo/data/labels/62_7.png create mode 100644 ggml/examples/yolo/data/labels/63_0.png create mode 100644 ggml/examples/yolo/data/labels/63_1.png create mode 100644 ggml/examples/yolo/data/labels/63_2.png create mode 100644 ggml/examples/yolo/data/labels/63_3.png create mode 100644 ggml/examples/yolo/data/labels/63_4.png create mode 100644 ggml/examples/yolo/data/labels/63_5.png create mode 100644 ggml/examples/yolo/data/labels/63_6.png create mode 100644 ggml/examples/yolo/data/labels/63_7.png create mode 100644 ggml/examples/yolo/data/labels/64_0.png create mode 100644 ggml/examples/yolo/data/labels/64_1.png create mode 100644 ggml/examples/yolo/data/labels/64_2.png create mode 100644 ggml/examples/yolo/data/labels/64_3.png create mode 100644 ggml/examples/yolo/data/labels/64_4.png create mode 100644 ggml/examples/yolo/data/labels/64_5.png create mode 100644 ggml/examples/yolo/data/labels/64_6.png create mode 100644 ggml/examples/yolo/data/labels/64_7.png create mode 100644 ggml/examples/yolo/data/labels/65_0.png create mode 100644 ggml/examples/yolo/data/labels/65_1.png create mode 100644 ggml/examples/yolo/data/labels/65_2.png create mode 100644 ggml/examples/yolo/data/labels/65_3.png create mode 100644 ggml/examples/yolo/data/labels/65_4.png create mode 100644 ggml/examples/yolo/data/labels/65_5.png create mode 100644 ggml/examples/yolo/data/labels/65_6.png create mode 100644 ggml/examples/yolo/data/labels/65_7.png create mode 100644 ggml/examples/yolo/data/labels/66_0.png create mode 100644 ggml/examples/yolo/data/labels/66_1.png create mode 100644 ggml/examples/yolo/data/labels/66_2.png create mode 100644 ggml/examples/yolo/data/labels/66_3.png create mode 100644 ggml/examples/yolo/data/labels/66_4.png create mode 100644 ggml/examples/yolo/data/labels/66_5.png create mode 100644 ggml/examples/yolo/data/labels/66_6.png create mode 100644 ggml/examples/yolo/data/labels/66_7.png create mode 100644 ggml/examples/yolo/data/labels/67_0.png create mode 100644 ggml/examples/yolo/data/labels/67_1.png create mode 100644 ggml/examples/yolo/data/labels/67_2.png create mode 100644 ggml/examples/yolo/data/labels/67_3.png create mode 100644 ggml/examples/yolo/data/labels/67_4.png create mode 100644 ggml/examples/yolo/data/labels/67_5.png create mode 100644 ggml/examples/yolo/data/labels/67_6.png create mode 100644 ggml/examples/yolo/data/labels/67_7.png create mode 100644 ggml/examples/yolo/data/labels/68_0.png create mode 100644 ggml/examples/yolo/data/labels/68_1.png create mode 100644 ggml/examples/yolo/data/labels/68_2.png create mode 100644 ggml/examples/yolo/data/labels/68_3.png create mode 100644 ggml/examples/yolo/data/labels/68_4.png create mode 100644 ggml/examples/yolo/data/labels/68_5.png create mode 100644 ggml/examples/yolo/data/labels/68_6.png create mode 100644 ggml/examples/yolo/data/labels/68_7.png create mode 100644 ggml/examples/yolo/data/labels/69_0.png create mode 100644 ggml/examples/yolo/data/labels/69_1.png create mode 100644 ggml/examples/yolo/data/labels/69_2.png create mode 100644 ggml/examples/yolo/data/labels/69_3.png create mode 100644 ggml/examples/yolo/data/labels/69_4.png create mode 100644 ggml/examples/yolo/data/labels/69_5.png create mode 100644 ggml/examples/yolo/data/labels/69_6.png create mode 100644 ggml/examples/yolo/data/labels/69_7.png create mode 100644 ggml/examples/yolo/data/labels/70_0.png create mode 100644 ggml/examples/yolo/data/labels/70_1.png create mode 100644 ggml/examples/yolo/data/labels/70_2.png create mode 100644 ggml/examples/yolo/data/labels/70_3.png create mode 100644 ggml/examples/yolo/data/labels/70_4.png create mode 100644 ggml/examples/yolo/data/labels/70_5.png create mode 100644 ggml/examples/yolo/data/labels/70_6.png create mode 100644 ggml/examples/yolo/data/labels/70_7.png create mode 100644 ggml/examples/yolo/data/labels/71_0.png create mode 100644 ggml/examples/yolo/data/labels/71_1.png create mode 100644 ggml/examples/yolo/data/labels/71_2.png create mode 100644 ggml/examples/yolo/data/labels/71_3.png create mode 100644 ggml/examples/yolo/data/labels/71_4.png create mode 100644 ggml/examples/yolo/data/labels/71_5.png create mode 100644 ggml/examples/yolo/data/labels/71_6.png create mode 100644 ggml/examples/yolo/data/labels/71_7.png create mode 100644 ggml/examples/yolo/data/labels/72_0.png create mode 100644 ggml/examples/yolo/data/labels/72_1.png create mode 100644 ggml/examples/yolo/data/labels/72_2.png create mode 100644 ggml/examples/yolo/data/labels/72_3.png create mode 100644 ggml/examples/yolo/data/labels/72_4.png create mode 100644 ggml/examples/yolo/data/labels/72_5.png create mode 100644 ggml/examples/yolo/data/labels/72_6.png create mode 100644 ggml/examples/yolo/data/labels/72_7.png create mode 100644 ggml/examples/yolo/data/labels/73_0.png create mode 100644 ggml/examples/yolo/data/labels/73_1.png create mode 100644 ggml/examples/yolo/data/labels/73_2.png create mode 100644 ggml/examples/yolo/data/labels/73_3.png create mode 100644 ggml/examples/yolo/data/labels/73_4.png create mode 100644 ggml/examples/yolo/data/labels/73_5.png create mode 100644 ggml/examples/yolo/data/labels/73_6.png create mode 100644 ggml/examples/yolo/data/labels/73_7.png create mode 100644 ggml/examples/yolo/data/labels/74_0.png create mode 100644 ggml/examples/yolo/data/labels/74_1.png create mode 100644 ggml/examples/yolo/data/labels/74_2.png create mode 100644 ggml/examples/yolo/data/labels/74_3.png create mode 100644 ggml/examples/yolo/data/labels/74_4.png create mode 100644 ggml/examples/yolo/data/labels/74_5.png create mode 100644 ggml/examples/yolo/data/labels/74_6.png create mode 100644 ggml/examples/yolo/data/labels/74_7.png create mode 100644 ggml/examples/yolo/data/labels/75_0.png create mode 100644 ggml/examples/yolo/data/labels/75_1.png create mode 100644 ggml/examples/yolo/data/labels/75_2.png create mode 100644 ggml/examples/yolo/data/labels/75_3.png create mode 100644 ggml/examples/yolo/data/labels/75_4.png create mode 100644 ggml/examples/yolo/data/labels/75_5.png create mode 100644 ggml/examples/yolo/data/labels/75_6.png create mode 100644 ggml/examples/yolo/data/labels/75_7.png create mode 100644 ggml/examples/yolo/data/labels/76_0.png create mode 100644 ggml/examples/yolo/data/labels/76_1.png create mode 100644 ggml/examples/yolo/data/labels/76_2.png create mode 100644 ggml/examples/yolo/data/labels/76_3.png create mode 100644 ggml/examples/yolo/data/labels/76_4.png create mode 100644 ggml/examples/yolo/data/labels/76_5.png create mode 100644 ggml/examples/yolo/data/labels/76_6.png create mode 100644 ggml/examples/yolo/data/labels/76_7.png create mode 100644 ggml/examples/yolo/data/labels/77_0.png create mode 100644 ggml/examples/yolo/data/labels/77_1.png create mode 100644 ggml/examples/yolo/data/labels/77_2.png create mode 100644 ggml/examples/yolo/data/labels/77_3.png create mode 100644 ggml/examples/yolo/data/labels/77_4.png create mode 100644 ggml/examples/yolo/data/labels/77_5.png create mode 100644 ggml/examples/yolo/data/labels/77_6.png create mode 100644 ggml/examples/yolo/data/labels/77_7.png create mode 100644 ggml/examples/yolo/data/labels/78_0.png create mode 100644 ggml/examples/yolo/data/labels/78_1.png create mode 100644 ggml/examples/yolo/data/labels/78_2.png create mode 100644 ggml/examples/yolo/data/labels/78_3.png create mode 100644 ggml/examples/yolo/data/labels/78_4.png create mode 100644 ggml/examples/yolo/data/labels/78_5.png create mode 100644 ggml/examples/yolo/data/labels/78_6.png create mode 100644 ggml/examples/yolo/data/labels/78_7.png create mode 100644 ggml/examples/yolo/data/labels/79_0.png create mode 100644 ggml/examples/yolo/data/labels/79_1.png create mode 100644 ggml/examples/yolo/data/labels/79_2.png create mode 100644 ggml/examples/yolo/data/labels/79_3.png create mode 100644 ggml/examples/yolo/data/labels/79_4.png create mode 100644 ggml/examples/yolo/data/labels/79_5.png create mode 100644 ggml/examples/yolo/data/labels/79_6.png create mode 100644 ggml/examples/yolo/data/labels/79_7.png create mode 100644 ggml/examples/yolo/data/labels/80_0.png create mode 100644 ggml/examples/yolo/data/labels/80_1.png create mode 100644 ggml/examples/yolo/data/labels/80_2.png create mode 100644 ggml/examples/yolo/data/labels/80_3.png create mode 100644 ggml/examples/yolo/data/labels/80_4.png create mode 100644 ggml/examples/yolo/data/labels/80_5.png create mode 100644 ggml/examples/yolo/data/labels/80_6.png create mode 100644 ggml/examples/yolo/data/labels/80_7.png create mode 100644 ggml/examples/yolo/data/labels/81_0.png create mode 100644 ggml/examples/yolo/data/labels/81_1.png create mode 100644 ggml/examples/yolo/data/labels/81_2.png create mode 100644 ggml/examples/yolo/data/labels/81_3.png create mode 100644 ggml/examples/yolo/data/labels/81_4.png create mode 100644 ggml/examples/yolo/data/labels/81_5.png create mode 100644 ggml/examples/yolo/data/labels/81_6.png create mode 100644 ggml/examples/yolo/data/labels/81_7.png create mode 100644 ggml/examples/yolo/data/labels/82_0.png create mode 100644 ggml/examples/yolo/data/labels/82_1.png create mode 100644 ggml/examples/yolo/data/labels/82_2.png create mode 100644 ggml/examples/yolo/data/labels/82_3.png create mode 100644 ggml/examples/yolo/data/labels/82_4.png create mode 100644 ggml/examples/yolo/data/labels/82_5.png create mode 100644 ggml/examples/yolo/data/labels/82_6.png create mode 100644 ggml/examples/yolo/data/labels/82_7.png create mode 100644 ggml/examples/yolo/data/labels/83_0.png create mode 100644 ggml/examples/yolo/data/labels/83_1.png create mode 100644 ggml/examples/yolo/data/labels/83_2.png create mode 100644 ggml/examples/yolo/data/labels/83_3.png create mode 100644 ggml/examples/yolo/data/labels/83_4.png create mode 100644 ggml/examples/yolo/data/labels/83_5.png create mode 100644 ggml/examples/yolo/data/labels/83_6.png create mode 100644 ggml/examples/yolo/data/labels/83_7.png create mode 100644 ggml/examples/yolo/data/labels/84_0.png create mode 100644 ggml/examples/yolo/data/labels/84_1.png create mode 100644 ggml/examples/yolo/data/labels/84_2.png create mode 100644 ggml/examples/yolo/data/labels/84_3.png create mode 100644 ggml/examples/yolo/data/labels/84_4.png create mode 100644 ggml/examples/yolo/data/labels/84_5.png create mode 100644 ggml/examples/yolo/data/labels/84_6.png create mode 100644 ggml/examples/yolo/data/labels/84_7.png create mode 100644 ggml/examples/yolo/data/labels/85_0.png create mode 100644 ggml/examples/yolo/data/labels/85_1.png create mode 100644 ggml/examples/yolo/data/labels/85_2.png create mode 100644 ggml/examples/yolo/data/labels/85_3.png create mode 100644 ggml/examples/yolo/data/labels/85_4.png create mode 100644 ggml/examples/yolo/data/labels/85_5.png create mode 100644 ggml/examples/yolo/data/labels/85_6.png create mode 100644 ggml/examples/yolo/data/labels/85_7.png create mode 100644 ggml/examples/yolo/data/labels/86_0.png create mode 100644 ggml/examples/yolo/data/labels/86_1.png create mode 100644 ggml/examples/yolo/data/labels/86_2.png create mode 100644 ggml/examples/yolo/data/labels/86_3.png create mode 100644 ggml/examples/yolo/data/labels/86_4.png create mode 100644 ggml/examples/yolo/data/labels/86_5.png create mode 100644 ggml/examples/yolo/data/labels/86_6.png create mode 100644 ggml/examples/yolo/data/labels/86_7.png create mode 100644 ggml/examples/yolo/data/labels/87_0.png create mode 100644 ggml/examples/yolo/data/labels/87_1.png create mode 100644 ggml/examples/yolo/data/labels/87_2.png create mode 100644 ggml/examples/yolo/data/labels/87_3.png create mode 100644 ggml/examples/yolo/data/labels/87_4.png create mode 100644 ggml/examples/yolo/data/labels/87_5.png create mode 100644 ggml/examples/yolo/data/labels/87_6.png create mode 100644 ggml/examples/yolo/data/labels/87_7.png create mode 100644 ggml/examples/yolo/data/labels/88_0.png create mode 100644 ggml/examples/yolo/data/labels/88_1.png create mode 100644 ggml/examples/yolo/data/labels/88_2.png create mode 100644 ggml/examples/yolo/data/labels/88_3.png create mode 100644 ggml/examples/yolo/data/labels/88_4.png create mode 100644 ggml/examples/yolo/data/labels/88_5.png create mode 100644 ggml/examples/yolo/data/labels/88_6.png create mode 100644 ggml/examples/yolo/data/labels/88_7.png create mode 100644 ggml/examples/yolo/data/labels/89_0.png create mode 100644 ggml/examples/yolo/data/labels/89_1.png create mode 100644 ggml/examples/yolo/data/labels/89_2.png create mode 100644 ggml/examples/yolo/data/labels/89_3.png create mode 100644 ggml/examples/yolo/data/labels/89_4.png create mode 100644 ggml/examples/yolo/data/labels/89_5.png create mode 100644 ggml/examples/yolo/data/labels/89_6.png create mode 100644 ggml/examples/yolo/data/labels/89_7.png create mode 100644 ggml/examples/yolo/data/labels/90_0.png create mode 100644 ggml/examples/yolo/data/labels/90_1.png create mode 100644 ggml/examples/yolo/data/labels/90_2.png create mode 100644 ggml/examples/yolo/data/labels/90_3.png create mode 100644 ggml/examples/yolo/data/labels/90_4.png create mode 100644 ggml/examples/yolo/data/labels/90_5.png create mode 100644 ggml/examples/yolo/data/labels/90_6.png create mode 100644 ggml/examples/yolo/data/labels/90_7.png create mode 100644 ggml/examples/yolo/data/labels/91_0.png create mode 100644 ggml/examples/yolo/data/labels/91_1.png create mode 100644 ggml/examples/yolo/data/labels/91_2.png create mode 100644 ggml/examples/yolo/data/labels/91_3.png create mode 100644 ggml/examples/yolo/data/labels/91_4.png create mode 100644 ggml/examples/yolo/data/labels/91_5.png create mode 100644 ggml/examples/yolo/data/labels/91_6.png create mode 100644 ggml/examples/yolo/data/labels/91_7.png create mode 100644 ggml/examples/yolo/data/labels/92_0.png create mode 100644 ggml/examples/yolo/data/labels/92_1.png create mode 100644 ggml/examples/yolo/data/labels/92_2.png create mode 100644 ggml/examples/yolo/data/labels/92_3.png create mode 100644 ggml/examples/yolo/data/labels/92_4.png create mode 100644 ggml/examples/yolo/data/labels/92_5.png create mode 100644 ggml/examples/yolo/data/labels/92_6.png create mode 100644 ggml/examples/yolo/data/labels/92_7.png create mode 100644 ggml/examples/yolo/data/labels/93_0.png create mode 100644 ggml/examples/yolo/data/labels/93_1.png create mode 100644 ggml/examples/yolo/data/labels/93_2.png create mode 100644 ggml/examples/yolo/data/labels/93_3.png create mode 100644 ggml/examples/yolo/data/labels/93_4.png create mode 100644 ggml/examples/yolo/data/labels/93_5.png create mode 100644 ggml/examples/yolo/data/labels/93_6.png create mode 100644 ggml/examples/yolo/data/labels/93_7.png create mode 100644 ggml/examples/yolo/data/labels/94_0.png create mode 100644 ggml/examples/yolo/data/labels/94_1.png create mode 100644 ggml/examples/yolo/data/labels/94_2.png create mode 100644 ggml/examples/yolo/data/labels/94_3.png create mode 100644 ggml/examples/yolo/data/labels/94_4.png create mode 100644 ggml/examples/yolo/data/labels/94_5.png create mode 100644 ggml/examples/yolo/data/labels/94_6.png create mode 100644 ggml/examples/yolo/data/labels/94_7.png create mode 100644 ggml/examples/yolo/data/labels/95_0.png create mode 100644 ggml/examples/yolo/data/labels/95_1.png create mode 100644 ggml/examples/yolo/data/labels/95_2.png create mode 100644 ggml/examples/yolo/data/labels/95_3.png create mode 100644 ggml/examples/yolo/data/labels/95_4.png create mode 100644 ggml/examples/yolo/data/labels/95_5.png create mode 100644 ggml/examples/yolo/data/labels/95_6.png create mode 100644 ggml/examples/yolo/data/labels/95_7.png create mode 100644 ggml/examples/yolo/data/labels/96_0.png create mode 100644 ggml/examples/yolo/data/labels/96_1.png create mode 100644 ggml/examples/yolo/data/labels/96_2.png create mode 100644 ggml/examples/yolo/data/labels/96_3.png create mode 100644 ggml/examples/yolo/data/labels/96_4.png create mode 100644 ggml/examples/yolo/data/labels/96_5.png create mode 100644 ggml/examples/yolo/data/labels/96_6.png create mode 100644 ggml/examples/yolo/data/labels/96_7.png create mode 100644 ggml/examples/yolo/data/labels/97_0.png create mode 100644 ggml/examples/yolo/data/labels/97_1.png create mode 100644 ggml/examples/yolo/data/labels/97_2.png create mode 100644 ggml/examples/yolo/data/labels/97_3.png create mode 100644 ggml/examples/yolo/data/labels/97_4.png create mode 100644 ggml/examples/yolo/data/labels/97_5.png create mode 100644 ggml/examples/yolo/data/labels/97_6.png create mode 100644 ggml/examples/yolo/data/labels/97_7.png create mode 100644 ggml/examples/yolo/data/labels/98_0.png create mode 100644 ggml/examples/yolo/data/labels/98_1.png create mode 100644 ggml/examples/yolo/data/labels/98_2.png create mode 100644 ggml/examples/yolo/data/labels/98_3.png create mode 100644 ggml/examples/yolo/data/labels/98_4.png create mode 100644 ggml/examples/yolo/data/labels/98_5.png create mode 100644 ggml/examples/yolo/data/labels/98_6.png create mode 100644 ggml/examples/yolo/data/labels/98_7.png create mode 100644 ggml/examples/yolo/data/labels/99_0.png create mode 100644 ggml/examples/yolo/data/labels/99_1.png create mode 100644 ggml/examples/yolo/data/labels/99_2.png create mode 100644 ggml/examples/yolo/data/labels/99_3.png create mode 100644 ggml/examples/yolo/data/labels/99_4.png create mode 100644 ggml/examples/yolo/data/labels/99_5.png create mode 100644 ggml/examples/yolo/data/labels/99_6.png create mode 100644 ggml/examples/yolo/data/labels/99_7.png create mode 100644 ggml/examples/yolo/yolo-image.cpp create mode 100644 ggml/examples/yolo/yolo-image.h create mode 100644 ggml/examples/yolo/yolov3-tiny.cpp create mode 100644 ggml/ggml.pc.in create mode 100644 ggml/include/ggml-alloc.h create mode 100644 ggml/include/ggml-backend.h create mode 100644 ggml/include/ggml-blas.h create mode 100644 ggml/include/ggml-cann.h create mode 100644 ggml/include/ggml-cpp.h create mode 100644 ggml/include/ggml-cpu.h create mode 100644 ggml/include/ggml-cuda.h create mode 100644 ggml/include/ggml-hexagon.h create mode 100644 ggml/include/ggml-metal.h create mode 100644 ggml/include/ggml-opencl.h create mode 100644 ggml/include/ggml-openvino.h create mode 100644 ggml/include/ggml-opt.h create mode 100644 ggml/include/ggml-rpc.h create mode 100644 ggml/include/ggml-sycl.h create mode 100644 ggml/include/ggml-virtgpu.h create mode 100644 ggml/include/ggml-vulkan.h create mode 100644 ggml/include/ggml-webgpu.h create mode 100644 ggml/include/ggml-zdnn.h create mode 100644 ggml/include/ggml-zendnn.h create mode 100644 ggml/include/ggml.h create mode 100644 ggml/include/gguf.h create mode 100644 ggml/requirements.txt create mode 100755 ggml/scripts/gen-authors.sh create mode 100755 ggml/scripts/release.sh create mode 100755 ggml/scripts/sync-llama-am.sh create mode 100644 ggml/scripts/sync-llama.last create mode 100755 ggml/scripts/sync-llama.sh create mode 100755 ggml/scripts/sync-whisper-am.sh create mode 100644 ggml/scripts/sync-whisper.last create mode 100755 ggml/scripts/sync-whisper.sh create mode 100644 ggml/src/CMakeLists.txt create mode 100644 ggml/src/ggml-alloc.c create mode 100644 ggml/src/ggml-backend-dl.cpp create mode 100644 ggml/src/ggml-backend-dl.h create mode 100644 ggml/src/ggml-backend-impl.h create mode 100644 ggml/src/ggml-backend-meta.cpp create mode 100644 ggml/src/ggml-backend-reg.cpp create mode 100644 ggml/src/ggml-backend.cpp create mode 100644 ggml/src/ggml-blas/CMakeLists.txt create mode 100644 ggml/src/ggml-blas/ggml-blas.cpp create mode 100755 ggml/src/ggml-cann/CMakeLists.txt create mode 100644 ggml/src/ggml-cann/acl_tensor.cpp create mode 100644 ggml/src/ggml-cann/acl_tensor.h create mode 100644 ggml/src/ggml-cann/aclnn_ops.cpp create mode 100644 ggml/src/ggml-cann/aclnn_ops.h create mode 100644 ggml/src/ggml-cann/common.h create mode 100644 ggml/src/ggml-cann/ggml-cann.cpp create mode 100644 ggml/src/ggml-common.h create mode 100644 ggml/src/ggml-cpu/CMakeLists.txt create mode 100644 ggml/src/ggml-cpu/amx/amx.cpp create mode 100644 ggml/src/ggml-cpu/amx/amx.h create mode 100644 ggml/src/ggml-cpu/amx/common.h create mode 100644 ggml/src/ggml-cpu/amx/mmq.cpp create mode 100644 ggml/src/ggml-cpu/amx/mmq.h create mode 100644 ggml/src/ggml-cpu/arch-fallback.h create mode 100644 ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp create mode 100644 ggml/src/ggml-cpu/arch/arm/quants.c create mode 100644 ggml/src/ggml-cpu/arch/arm/repack.cpp create mode 100644 ggml/src/ggml-cpu/arch/loongarch/quants.c create mode 100644 ggml/src/ggml-cpu/arch/powerpc/cpu-feats.cpp create mode 100644 ggml/src/ggml-cpu/arch/powerpc/quants.c create mode 100644 ggml/src/ggml-cpu/arch/riscv/cpu-feats.cpp create mode 100644 ggml/src/ggml-cpu/arch/riscv/quants.c create mode 100644 ggml/src/ggml-cpu/arch/riscv/repack.cpp create mode 100644 ggml/src/ggml-cpu/arch/s390/cpu-feats.cpp create mode 100644 ggml/src/ggml-cpu/arch/s390/quants.c create mode 100644 ggml/src/ggml-cpu/arch/wasm/quants.c create mode 100644 ggml/src/ggml-cpu/arch/x86/cpu-feats.cpp create mode 100644 ggml/src/ggml-cpu/arch/x86/quants.c create mode 100644 ggml/src/ggml-cpu/arch/x86/repack.cpp create mode 100644 ggml/src/ggml-cpu/binary-ops.cpp create mode 100644 ggml/src/ggml-cpu/binary-ops.h create mode 100644 ggml/src/ggml-cpu/cmake/FindSIMD.cmake create mode 100644 ggml/src/ggml-cpu/cmake/FindSMTIME.cmake create mode 100644 ggml/src/ggml-cpu/common.h create mode 100644 ggml/src/ggml-cpu/ggml-cpu-impl.h create mode 100644 ggml/src/ggml-cpu/ggml-cpu.c create mode 100644 ggml/src/ggml-cpu/ggml-cpu.cpp create mode 100644 ggml/src/ggml-cpu/hbm.cpp create mode 100644 ggml/src/ggml-cpu/hbm.h create mode 100644 ggml/src/ggml-cpu/kleidiai/kernels.cpp create mode 100644 ggml/src/ggml-cpu/kleidiai/kernels.h create mode 100644 ggml/src/ggml-cpu/kleidiai/kleidiai.cpp create mode 100644 ggml/src/ggml-cpu/kleidiai/kleidiai.h create mode 100644 ggml/src/ggml-cpu/llamafile/sgemm.cpp create mode 100644 ggml/src/ggml-cpu/llamafile/sgemm.h create mode 100644 ggml/src/ggml-cpu/ops.cpp create mode 100644 ggml/src/ggml-cpu/ops.h create mode 100644 ggml/src/ggml-cpu/quants.c create mode 100644 ggml/src/ggml-cpu/quants.h create mode 100644 ggml/src/ggml-cpu/repack.cpp create mode 100644 ggml/src/ggml-cpu/repack.h create mode 100644 ggml/src/ggml-cpu/simd-gemm.h create mode 100644 ggml/src/ggml-cpu/simd-mappings.h create mode 100644 ggml/src/ggml-cpu/spacemit/ime.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/ime.h create mode 100644 ggml/src/ggml-cpu/spacemit/ime1_kernels.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/ime2_kernels.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/ime_env.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/ime_env.h create mode 100644 ggml/src/ggml-cpu/spacemit/ime_kernels.h create mode 100644 ggml/src/ggml-cpu/spacemit/repack.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/repack.h create mode 100644 ggml/src/ggml-cpu/spacemit/rvv_kernels.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/rvv_kernels.h create mode 100644 ggml/src/ggml-cpu/spacemit/spine_barrier.h create mode 100644 ggml/src/ggml-cpu/spacemit/spine_mem_pool.cpp create mode 100644 ggml/src/ggml-cpu/spacemit/spine_mem_pool.h create mode 100644 ggml/src/ggml-cpu/spacemit/spine_tcm.h create mode 100644 ggml/src/ggml-cpu/traits.cpp create mode 100644 ggml/src/ggml-cpu/traits.h create mode 100644 ggml/src/ggml-cpu/unary-ops.cpp create mode 100644 ggml/src/ggml-cpu/unary-ops.h create mode 100644 ggml/src/ggml-cpu/vec.cpp create mode 100644 ggml/src/ggml-cpu/vec.h create mode 100644 ggml/src/ggml-cuda/CMakeLists.txt create mode 100644 ggml/src/ggml-cuda/acc.cu create mode 100644 ggml/src/ggml-cuda/acc.cuh create mode 100644 ggml/src/ggml-cuda/add-id.cu create mode 100644 ggml/src/ggml-cuda/add-id.cuh create mode 100644 ggml/src/ggml-cuda/allreduce.cu create mode 100644 ggml/src/ggml-cuda/allreduce.cuh create mode 100644 ggml/src/ggml-cuda/arange.cu create mode 100644 ggml/src/ggml-cuda/arange.cuh create mode 100644 ggml/src/ggml-cuda/argmax.cu create mode 100644 ggml/src/ggml-cuda/argmax.cuh create mode 100644 ggml/src/ggml-cuda/argsort.cu create mode 100644 ggml/src/ggml-cuda/argsort.cuh create mode 100644 ggml/src/ggml-cuda/binbcast.cu create mode 100644 ggml/src/ggml-cuda/binbcast.cuh create mode 100644 ggml/src/ggml-cuda/clamp.cu create mode 100644 ggml/src/ggml-cuda/clamp.cuh create mode 100644 ggml/src/ggml-cuda/col2im-1d.cu create mode 100644 ggml/src/ggml-cuda/col2im-1d.cuh create mode 100644 ggml/src/ggml-cuda/common.cuh create mode 100644 ggml/src/ggml-cuda/concat.cu create mode 100644 ggml/src/ggml-cuda/concat.cuh create mode 100644 ggml/src/ggml-cuda/conv-transpose-1d.cu create mode 100644 ggml/src/ggml-cuda/conv-transpose-1d.cuh create mode 100644 ggml/src/ggml-cuda/conv2d-dw.cu create mode 100644 ggml/src/ggml-cuda/conv2d-dw.cuh create mode 100644 ggml/src/ggml-cuda/conv2d-transpose.cu create mode 100644 ggml/src/ggml-cuda/conv2d-transpose.cuh create mode 100644 ggml/src/ggml-cuda/conv2d.cu create mode 100644 ggml/src/ggml-cuda/conv2d.cuh create mode 100644 ggml/src/ggml-cuda/convert.cu create mode 100644 ggml/src/ggml-cuda/convert.cuh create mode 100644 ggml/src/ggml-cuda/count-equal.cu create mode 100644 ggml/src/ggml-cuda/count-equal.cuh create mode 100644 ggml/src/ggml-cuda/cp-async.cuh create mode 100644 ggml/src/ggml-cuda/cpy-utils.cuh create mode 100644 ggml/src/ggml-cuda/cpy.cu create mode 100644 ggml/src/ggml-cuda/cpy.cuh create mode 100644 ggml/src/ggml-cuda/cross-entropy-loss.cu create mode 100644 ggml/src/ggml-cuda/cross-entropy-loss.cuh create mode 100644 ggml/src/ggml-cuda/cumsum.cu create mode 100644 ggml/src/ggml-cuda/cumsum.cuh create mode 100644 ggml/src/ggml-cuda/dequantize.cuh create mode 100644 ggml/src/ggml-cuda/diag.cu create mode 100644 ggml/src/ggml-cuda/diag.cuh create mode 100644 ggml/src/ggml-cuda/diagmask.cu create mode 100644 ggml/src/ggml-cuda/diagmask.cuh create mode 100644 ggml/src/ggml-cuda/fattn-common.cuh create mode 100644 ggml/src/ggml-cuda/fattn-mma-f16.cuh create mode 100644 ggml/src/ggml-cuda/fattn-tile.cu create mode 100644 ggml/src/ggml-cuda/fattn-tile.cuh create mode 100644 ggml/src/ggml-cuda/fattn-vec.cuh create mode 100644 ggml/src/ggml-cuda/fattn-wmma-f16.cu create mode 100644 ggml/src/ggml-cuda/fattn-wmma-f16.cuh create mode 100644 ggml/src/ggml-cuda/fattn.cu create mode 100644 ggml/src/ggml-cuda/fattn.cuh create mode 100644 ggml/src/ggml-cuda/fill.cu create mode 100644 ggml/src/ggml-cuda/fill.cuh create mode 100644 ggml/src/ggml-cuda/fwht.cu create mode 100644 ggml/src/ggml-cuda/fwht.cuh create mode 100644 ggml/src/ggml-cuda/gated_delta_net.cu create mode 100644 ggml/src/ggml-cuda/gated_delta_net.cuh create mode 100644 ggml/src/ggml-cuda/getrows.cu create mode 100644 ggml/src/ggml-cuda/getrows.cuh create mode 100644 ggml/src/ggml-cuda/ggml-cuda.cu create mode 100644 ggml/src/ggml-cuda/gla.cu create mode 100644 ggml/src/ggml-cuda/gla.cuh create mode 100644 ggml/src/ggml-cuda/im2col.cu create mode 100644 ggml/src/ggml-cuda/im2col.cuh create mode 100644 ggml/src/ggml-cuda/mean.cu create mode 100644 ggml/src/ggml-cuda/mean.cuh create mode 100644 ggml/src/ggml-cuda/mma.cuh create mode 100644 ggml/src/ggml-cuda/mmf.cu create mode 100644 ggml/src/ggml-cuda/mmf.cuh create mode 100644 ggml/src/ggml-cuda/mmid.cu create mode 100644 ggml/src/ggml-cuda/mmid.cuh create mode 100644 ggml/src/ggml-cuda/mmq.cu create mode 100644 ggml/src/ggml-cuda/mmq.cuh create mode 100644 ggml/src/ggml-cuda/mmvf.cu create mode 100644 ggml/src/ggml-cuda/mmvf.cuh create mode 100644 ggml/src/ggml-cuda/mmvq.cu create mode 100644 ggml/src/ggml-cuda/mmvq.cuh create mode 100644 ggml/src/ggml-cuda/norm.cu create mode 100644 ggml/src/ggml-cuda/norm.cuh create mode 100644 ggml/src/ggml-cuda/opt-step-adamw.cu create mode 100644 ggml/src/ggml-cuda/opt-step-adamw.cuh create mode 100644 ggml/src/ggml-cuda/opt-step-sgd.cu create mode 100644 ggml/src/ggml-cuda/opt-step-sgd.cuh create mode 100644 ggml/src/ggml-cuda/out-prod.cu create mode 100644 ggml/src/ggml-cuda/out-prod.cuh create mode 100644 ggml/src/ggml-cuda/pad.cu create mode 100644 ggml/src/ggml-cuda/pad.cuh create mode 100644 ggml/src/ggml-cuda/pad_reflect_1d.cu create mode 100644 ggml/src/ggml-cuda/pad_reflect_1d.cuh create mode 100644 ggml/src/ggml-cuda/pool2d.cu create mode 100644 ggml/src/ggml-cuda/pool2d.cuh create mode 100644 ggml/src/ggml-cuda/quantize.cu create mode 100644 ggml/src/ggml-cuda/quantize.cuh create mode 100644 ggml/src/ggml-cuda/reduce_rows.cuh create mode 100644 ggml/src/ggml-cuda/roll.cu create mode 100644 ggml/src/ggml-cuda/roll.cuh create mode 100644 ggml/src/ggml-cuda/rope.cu create mode 100644 ggml/src/ggml-cuda/rope.cuh create mode 100644 ggml/src/ggml-cuda/scale.cu create mode 100644 ggml/src/ggml-cuda/scale.cuh create mode 100644 ggml/src/ggml-cuda/set-rows.cu create mode 100644 ggml/src/ggml-cuda/set-rows.cuh create mode 100644 ggml/src/ggml-cuda/set.cu create mode 100644 ggml/src/ggml-cuda/set.cuh create mode 100644 ggml/src/ggml-cuda/snake.cu create mode 100644 ggml/src/ggml-cuda/snake.cuh create mode 100644 ggml/src/ggml-cuda/softcap.cu create mode 100644 ggml/src/ggml-cuda/softcap.cuh create mode 100644 ggml/src/ggml-cuda/softmax.cu create mode 100644 ggml/src/ggml-cuda/softmax.cuh create mode 100644 ggml/src/ggml-cuda/solve_tri.cu create mode 100644 ggml/src/ggml-cuda/solve_tri.cuh create mode 100644 ggml/src/ggml-cuda/ssm-conv.cu create mode 100644 ggml/src/ggml-cuda/ssm-conv.cuh create mode 100644 ggml/src/ggml-cuda/ssm-scan.cu create mode 100644 ggml/src/ggml-cuda/ssm-scan.cuh create mode 100644 ggml/src/ggml-cuda/sum.cu create mode 100644 ggml/src/ggml-cuda/sum.cuh create mode 100644 ggml/src/ggml-cuda/sumrows.cu create mode 100644 ggml/src/ggml-cuda/sumrows.cuh create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_32.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_1-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_16-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_32.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_2-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_32-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_4-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_64-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_8-ncols2_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq112-dv112.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq128-dv128.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq192-dv128.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq256-dv256.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq320-dv256.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq40-dv40.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq512-dv512.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq576-dv512.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq64-dv64.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq72-dv72.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq80-dv80.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-tile-instance-dkq96-dv96.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-q8_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-bf16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu create mode 100755 ggml/src/ggml-cuda/template-instances/generate_cu_files.py create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_10.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_11.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_12.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_13.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_14.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_15.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_16.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_2.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_3.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_5.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_6.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_7.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_8.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmf-instance-ncols_9.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-mxfp4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-nvfp4.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q1_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.cu create mode 100644 ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.cu create mode 100644 ggml/src/ggml-cuda/top-k.cu create mode 100644 ggml/src/ggml-cuda/top-k.cuh create mode 100644 ggml/src/ggml-cuda/topk-moe.cu create mode 100644 ggml/src/ggml-cuda/topk-moe.cuh create mode 100644 ggml/src/ggml-cuda/tri.cu create mode 100644 ggml/src/ggml-cuda/tri.cuh create mode 100644 ggml/src/ggml-cuda/tsembd.cu create mode 100644 ggml/src/ggml-cuda/tsembd.cuh create mode 100644 ggml/src/ggml-cuda/unary.cu create mode 100644 ggml/src/ggml-cuda/unary.cuh create mode 100644 ggml/src/ggml-cuda/upscale.cu create mode 100644 ggml/src/ggml-cuda/upscale.cuh create mode 100644 ggml/src/ggml-cuda/vecdotq.cuh create mode 100644 ggml/src/ggml-cuda/vendors/cuda.h create mode 100644 ggml/src/ggml-cuda/vendors/hip.h create mode 100644 ggml/src/ggml-cuda/vendors/musa.h create mode 100644 ggml/src/ggml-cuda/wkv.cu create mode 100644 ggml/src/ggml-cuda/wkv.cuh create mode 100644 ggml/src/ggml-hexagon/CMakeLists.txt create mode 100644 ggml/src/ggml-hexagon/ggml-hexagon.cpp create mode 100644 ggml/src/ggml-hexagon/htp-drv.cpp create mode 100644 ggml/src/ggml-hexagon/htp-drv.h create mode 100644 ggml/src/ggml-hexagon/htp-opnode.h create mode 100644 ggml/src/ggml-hexagon/htp/CMakeLists.txt create mode 100644 ggml/src/ggml-hexagon/htp/act-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/argsort-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/binary-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake create mode 100644 ggml/src/ggml-hexagon/htp/concat-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/cpy-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/cumsum-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/diag-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/fill-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/flash-attn-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/get-rows-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/hex-dma.c create mode 100644 ggml/src/ggml-hexagon/htp/hex-dma.h create mode 100644 ggml/src/ggml-hexagon/htp/hex-dump.h create mode 100644 ggml/src/ggml-hexagon/htp/hex-fastdiv.h create mode 100644 ggml/src/ggml-hexagon/htp/hex-profile.h create mode 100644 ggml/src/ggml-hexagon/htp/hex-utils.h create mode 100644 ggml/src/ggml-hexagon/htp/hmx-flash-attn-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/hmx-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/hmx-ops.h create mode 100644 ggml/src/ggml-hexagon/htp/hmx-queue.c create mode 100644 ggml/src/ggml-hexagon/htp/hmx-queue.h create mode 100644 ggml/src/ggml-hexagon/htp/hmx-utils.h create mode 100644 ggml/src/ggml-hexagon/htp/htp-ctx.h create mode 100644 ggml/src/ggml-hexagon/htp/htp-ops.h create mode 100644 ggml/src/ggml-hexagon/htp/htp_iface.idl create mode 100644 ggml/src/ggml-hexagon/htp/hvx-arith.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-base.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-copy.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-div.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-dump.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-exp.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-flash-attn.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-floor.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-inverse.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-log.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-pow.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-reduce.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-repl.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-scale.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-sigmoid.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-sin-cos.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-sqrt.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-types.h create mode 100644 ggml/src/ggml-hexagon/htp/hvx-utils.h create mode 100644 ggml/src/ggml-hexagon/htp/main.c create mode 100644 ggml/src/ggml-hexagon/htp/matmul-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/pad-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/repeat-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/rope-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/set-rows-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/softmax-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/solve-tri-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/ssm-conv.c create mode 100644 ggml/src/ggml-hexagon/htp/sum-rows-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/unary-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/vtcm-utils.h create mode 100644 ggml/src/ggml-hexagon/htp/worker-pool.c create mode 100644 ggml/src/ggml-hexagon/htp/worker-pool.h create mode 100644 ggml/src/ggml-hexagon/libdl.h create mode 100644 ggml/src/ggml-hexagon/libggml-htp.inf create mode 100644 ggml/src/ggml-hip/CMakeLists.txt create mode 100644 ggml/src/ggml-impl.h create mode 100644 ggml/src/ggml-metal/CMakeLists.txt create mode 100644 ggml/src/ggml-metal/ggml-metal-common.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-common.h create mode 100644 ggml/src/ggml-metal/ggml-metal-context.h create mode 100644 ggml/src/ggml-metal/ggml-metal-context.m create mode 100644 ggml/src/ggml-metal/ggml-metal-device.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-device.h create mode 100644 ggml/src/ggml-metal/ggml-metal-device.m create mode 100644 ggml/src/ggml-metal/ggml-metal-impl.h create mode 100644 ggml/src/ggml-metal/ggml-metal-ops.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal-ops.h create mode 100644 ggml/src/ggml-metal/ggml-metal.cpp create mode 100644 ggml/src/ggml-metal/ggml-metal.metal create mode 100644 ggml/src/ggml-musa/CMakeLists.txt create mode 100644 ggml/src/ggml-musa/mudnn.cu create mode 100644 ggml/src/ggml-musa/mudnn.cuh create mode 100644 ggml/src/ggml-opencl/CMakeLists.txt create mode 100644 ggml/src/ggml-opencl/ggml-opencl.cpp create mode 100644 ggml/src/ggml-opencl/kernels/add.cl create mode 100644 ggml/src/ggml-opencl/kernels/add_id.cl create mode 100644 ggml/src/ggml-opencl/kernels/argsort.cl create mode 100644 ggml/src/ggml-opencl/kernels/clamp.cl create mode 100644 ggml/src/ggml-opencl/kernels/concat.cl create mode 100644 ggml/src/ggml-opencl/kernels/conv2d.cl create mode 100644 ggml/src/ggml-opencl/kernels/conv2d_f16_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/cpy.cl create mode 100644 ggml/src/ggml-opencl/kernels/cumsum.cl create mode 100644 ggml/src/ggml-opencl/kernels/cvt.cl create mode 100644 ggml/src/ggml-opencl/kernels/diag.cl create mode 100644 ggml/src/ggml-opencl/kernels/diag_mask_inf.cl create mode 100644 ggml/src/ggml-opencl/kernels/div.cl create mode 100644 ggml/src/ggml-opencl/kernels/embed_kernel.py create mode 100644 ggml/src/ggml-opencl/kernels/exp.cl create mode 100644 ggml/src/ggml-opencl/kernels/expm1.cl create mode 100644 ggml/src/ggml-opencl/kernels/fill.cl create mode 100644 ggml/src/ggml-opencl/kernels/flash_attn_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/flash_attn_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/gated_delta_net.cl create mode 100644 ggml/src/ggml-opencl/kernels/gelu.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_mxfp4_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q4_0_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q4_1_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q4_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q5_0_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q5_1_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q5_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_moe_q6_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_nl_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_xmem_f16_f32_os8.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_mxfp4_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q4_0_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q4_1_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q4_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q5_0_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q5_1_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q5_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_moe_q6_k_f32_ns.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/get_rows.cl create mode 100644 ggml/src/ggml-opencl/kernels/glu.cl create mode 100644 ggml/src/ggml-opencl/kernels/group_norm.cl create mode 100644 ggml/src/ggml-opencl/kernels/im2col_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/im2col_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/l2_norm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mean.cl create mode 100644 ggml/src/ggml-opencl/kernels/moe_reorder_b.cl create mode 100644 ggml/src/ggml-opencl/kernels/moe_sort_by_expert.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mat_f16_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_kq_kqv.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_f16_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_f32_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_iq4_nl_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q4_0_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q4_1_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q5_0_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q5_1_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q6_k_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_q8_0_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_id_mxfp4_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_id_q4_0_f32_8x_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_id_q8_0_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq4_nl_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_mxfp4_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_16x_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_1d_8x_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_8x_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_0_f32_v.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_1_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_0_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_1_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_q8_0_f32_flat.cl create mode 100644 ggml/src/ggml-opencl/kernels/neg.cl create mode 100644 ggml/src/ggml-opencl/kernels/norm.cl create mode 100644 ggml/src/ggml-opencl/kernels/pad.cl create mode 100644 ggml/src/ggml-opencl/kernels/relu.cl create mode 100644 ggml/src/ggml-opencl/kernels/repeat.cl create mode 100644 ggml/src/ggml-opencl/kernels/rms_norm.cl create mode 100644 ggml/src/ggml-opencl/kernels/rope.cl create mode 100644 ggml/src/ggml-opencl/kernels/scale.cl create mode 100644 ggml/src/ggml-opencl/kernels/set_rows.cl create mode 100644 ggml/src/ggml-opencl/kernels/sigmoid.cl create mode 100644 ggml/src/ggml-opencl/kernels/silu.cl create mode 100644 ggml/src/ggml-opencl/kernels/softmax_4_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/softmax_4_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/softmax_f16.cl create mode 100644 ggml/src/ggml-opencl/kernels/softmax_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/softplus.cl create mode 100644 ggml/src/ggml-opencl/kernels/solve_tri.cl create mode 100644 ggml/src/ggml-opencl/kernels/sqr.cl create mode 100644 ggml/src/ggml-opencl/kernels/sqrt.cl create mode 100644 ggml/src/ggml-opencl/kernels/ssm_conv.cl create mode 100644 ggml/src/ggml-opencl/kernels/sub.cl create mode 100644 ggml/src/ggml-opencl/kernels/sum_rows.cl create mode 100644 ggml/src/ggml-opencl/kernels/tanh.cl create mode 100644 ggml/src/ggml-opencl/kernels/transpose.cl create mode 100644 ggml/src/ggml-opencl/kernels/tri.cl create mode 100644 ggml/src/ggml-opencl/kernels/tsembd.cl create mode 100644 ggml/src/ggml-opencl/kernels/upscale.cl create mode 100644 ggml/src/ggml-openvino/.clang-format create mode 100644 ggml/src/ggml-openvino/CMakeLists.txt create mode 100644 ggml/src/ggml-openvino/ggml-decoder.cpp create mode 100644 ggml/src/ggml-openvino/ggml-decoder.h create mode 100644 ggml/src/ggml-openvino/ggml-openvino-extra.cpp create mode 100644 ggml/src/ggml-openvino/ggml-openvino-extra.h create mode 100644 ggml/src/ggml-openvino/ggml-openvino.cpp create mode 100644 ggml/src/ggml-openvino/ggml-quants.cpp create mode 100644 ggml/src/ggml-openvino/ggml-quants.h create mode 100644 ggml/src/ggml-openvino/openvino/decoder.h create mode 100644 ggml/src/ggml-openvino/openvino/frontend.cpp create mode 100644 ggml/src/ggml-openvino/openvino/frontend.h create mode 100644 ggml/src/ggml-openvino/openvino/input_model.cpp create mode 100644 ggml/src/ggml-openvino/openvino/input_model.h create mode 100644 ggml/src/ggml-openvino/openvino/node_context.h create mode 100644 ggml/src/ggml-openvino/openvino/op/add_id.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/argsort.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/clamp.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/concat.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/cont.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/cpy.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/div.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp create mode 100644 ggml/src/ggml-openvino/openvino/op/get_rows.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/im2col.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/l2_norm.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/mulmat.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/norm.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/pad.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/permute.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/repeat.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/reshape.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/rms_norm.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/rope.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/scale.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/set_rows.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/softmax.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/sum_rows.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/transpose.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/unary_silu.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/view.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op_table.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op_table.h create mode 100644 ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h create mode 100644 ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h create mode 100644 ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h create mode 100644 ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp create mode 100644 ggml/src/ggml-openvino/openvino/translate_session.cpp create mode 100644 ggml/src/ggml-openvino/openvino/translate_session.h create mode 100644 ggml/src/ggml-openvino/openvino/utils.cpp create mode 100644 ggml/src/ggml-openvino/openvino/utils.h create mode 100644 ggml/src/ggml-openvino/utils.cpp create mode 100644 ggml/src/ggml-openvino/utils.h create mode 100644 ggml/src/ggml-opt.cpp create mode 100644 ggml/src/ggml-quants.c create mode 100644 ggml/src/ggml-quants.h create mode 100644 ggml/src/ggml-rpc/CMakeLists.txt create mode 100644 ggml/src/ggml-rpc/ggml-rpc.cpp create mode 100644 ggml/src/ggml-rpc/transport.cpp create mode 100644 ggml/src/ggml-rpc/transport.h create mode 100644 ggml/src/ggml-sycl/CMakeLists.txt create mode 100644 ggml/src/ggml-sycl/add-id.cpp create mode 100644 ggml/src/ggml-sycl/add-id.hpp create mode 100644 ggml/src/ggml-sycl/backend.hpp create mode 100644 ggml/src/ggml-sycl/binbcast.cpp create mode 100644 ggml/src/ggml-sycl/binbcast.hpp create mode 100644 ggml/src/ggml-sycl/col2im-1d.cpp create mode 100644 ggml/src/ggml-sycl/col2im-1d.hpp create mode 100644 ggml/src/ggml-sycl/common.cpp create mode 100644 ggml/src/ggml-sycl/common.hpp create mode 100644 ggml/src/ggml-sycl/concat.cpp create mode 100644 ggml/src/ggml-sycl/concat.hpp create mode 100644 ggml/src/ggml-sycl/conv.cpp create mode 100644 ggml/src/ggml-sycl/conv.hpp create mode 100644 ggml/src/ggml-sycl/conv2d-dw.cpp create mode 100644 ggml/src/ggml-sycl/conv2d-dw.hpp create mode 100644 ggml/src/ggml-sycl/conv2d-transpose.cpp create mode 100644 ggml/src/ggml-sycl/conv2d-transpose.hpp create mode 100644 ggml/src/ggml-sycl/conv2d.cpp create mode 100644 ggml/src/ggml-sycl/conv2d.hpp create mode 100644 ggml/src/ggml-sycl/conv3d.cpp create mode 100644 ggml/src/ggml-sycl/conv3d.hpp create mode 100644 ggml/src/ggml-sycl/convert.cpp create mode 100644 ggml/src/ggml-sycl/convert.hpp create mode 100644 ggml/src/ggml-sycl/count-equal.cpp create mode 100644 ggml/src/ggml-sycl/count-equal.hpp create mode 100644 ggml/src/ggml-sycl/cpy.cpp create mode 100644 ggml/src/ggml-sycl/cpy.hpp create mode 100644 ggml/src/ggml-sycl/cumsum.cpp create mode 100644 ggml/src/ggml-sycl/cumsum.hpp create mode 100644 ggml/src/ggml-sycl/dequantize.hpp create mode 100644 ggml/src/ggml-sycl/diag.cpp create mode 100644 ggml/src/ggml-sycl/diag.hpp create mode 100644 ggml/src/ggml-sycl/dmmv.cpp create mode 100644 ggml/src/ggml-sycl/dmmv.hpp create mode 100644 ggml/src/ggml-sycl/dpct/helper.hpp create mode 100644 ggml/src/ggml-sycl/element_wise.cpp create mode 100644 ggml/src/ggml-sycl/element_wise.hpp create mode 100644 ggml/src/ggml-sycl/fattn-buffers.cpp create mode 100644 ggml/src/ggml-sycl/fattn-buffers.hpp create mode 100644 ggml/src/ggml-sycl/fattn-common.hpp create mode 100644 ggml/src/ggml-sycl/fattn-tile.cpp create mode 100644 ggml/src/ggml-sycl/fattn-tile.hpp create mode 100644 ggml/src/ggml-sycl/fattn-vec.hpp create mode 100644 ggml/src/ggml-sycl/fattn.cpp create mode 100644 ggml/src/ggml-sycl/fattn.hpp create mode 100644 ggml/src/ggml-sycl/fill.cpp create mode 100644 ggml/src/ggml-sycl/fill.hpp create mode 100644 ggml/src/ggml-sycl/gated_delta_net.cpp create mode 100644 ggml/src/ggml-sycl/gated_delta_net.hpp create mode 100644 ggml/src/ggml-sycl/gemm.hpp create mode 100644 ggml/src/ggml-sycl/getrows.cpp create mode 100644 ggml/src/ggml-sycl/getrows.hpp create mode 100644 ggml/src/ggml-sycl/ggml-sycl.cpp create mode 100644 ggml/src/ggml-sycl/gla.cpp create mode 100644 ggml/src/ggml-sycl/gla.hpp create mode 100644 ggml/src/ggml-sycl/im2col.cpp create mode 100644 ggml/src/ggml-sycl/im2col.hpp create mode 100644 ggml/src/ggml-sycl/mmq.cpp create mode 100644 ggml/src/ggml-sycl/mmq.hpp create mode 100644 ggml/src/ggml-sycl/mmvq.cpp create mode 100644 ggml/src/ggml-sycl/mmvq.hpp create mode 100644 ggml/src/ggml-sycl/norm.cpp create mode 100644 ggml/src/ggml-sycl/norm.hpp create mode 100644 ggml/src/ggml-sycl/outprod.cpp create mode 100644 ggml/src/ggml-sycl/outprod.hpp create mode 100644 ggml/src/ggml-sycl/pad.cpp create mode 100644 ggml/src/ggml-sycl/pad.hpp create mode 100644 ggml/src/ggml-sycl/pad_reflect_1d.cpp create mode 100644 ggml/src/ggml-sycl/pad_reflect_1d.hpp create mode 100644 ggml/src/ggml-sycl/pool.cpp create mode 100644 ggml/src/ggml-sycl/pool.hpp create mode 100644 ggml/src/ggml-sycl/presets.hpp create mode 100644 ggml/src/ggml-sycl/quantize.hpp create mode 100644 ggml/src/ggml-sycl/quants.hpp create mode 100644 ggml/src/ggml-sycl/repeat_back.cpp create mode 100644 ggml/src/ggml-sycl/repeat_back.hpp create mode 100644 ggml/src/ggml-sycl/roll.cpp create mode 100644 ggml/src/ggml-sycl/roll.hpp create mode 100644 ggml/src/ggml-sycl/rope.cpp create mode 100644 ggml/src/ggml-sycl/rope.hpp create mode 100644 ggml/src/ggml-sycl/set.cpp create mode 100644 ggml/src/ggml-sycl/set.hpp create mode 100644 ggml/src/ggml-sycl/set_rows.cpp create mode 100644 ggml/src/ggml-sycl/set_rows.hpp create mode 100644 ggml/src/ggml-sycl/softmax.cpp create mode 100644 ggml/src/ggml-sycl/softmax.hpp create mode 100644 ggml/src/ggml-sycl/solve_tri.cpp create mode 100644 ggml/src/ggml-sycl/solve_tri.hpp create mode 100644 ggml/src/ggml-sycl/ssm_conv.cpp create mode 100644 ggml/src/ggml-sycl/ssm_conv.hpp create mode 100644 ggml/src/ggml-sycl/ssm_scan.cpp create mode 100644 ggml/src/ggml-sycl/ssm_scan.hpp create mode 100644 ggml/src/ggml-sycl/sycl_hw.cpp create mode 100644 ggml/src/ggml-sycl/sycl_hw.hpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_1-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-f16.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q4_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_0.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q5_1.cpp create mode 100644 ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q8_0-q8_0.cpp create mode 100644 ggml/src/ggml-sycl/tsembd.cpp create mode 100644 ggml/src/ggml-sycl/tsembd.hpp create mode 100644 ggml/src/ggml-sycl/type.hpp create mode 100644 ggml/src/ggml-sycl/upscale.cpp create mode 100644 ggml/src/ggml-sycl/upscale.hpp create mode 100644 ggml/src/ggml-sycl/vecdotq.hpp create mode 100644 ggml/src/ggml-sycl/wkv.cpp create mode 100644 ggml/src/ggml-sycl/wkv.hpp create mode 100644 ggml/src/ggml-threading.cpp create mode 100644 ggml/src/ggml-threading.h create mode 100644 ggml/src/ggml-virtgpu/CMakeLists.txt create mode 100644 ggml/src/ggml-virtgpu/apir_cs_ggml-rpc-front.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/CMakeLists.txt create mode 100644 ggml/src/ggml-virtgpu/backend/apir_cs_ggml-rpc-back.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-convert.h create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched-backend.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer-type.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched-buffer.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched.gen.h create mode 100644 ggml/src/ggml-virtgpu/backend/backend-dispatched.h create mode 100644 ggml/src/ggml-virtgpu/backend/backend-virgl-apir.h create mode 100644 ggml/src/ggml-virtgpu/backend/backend.cpp create mode 100644 ggml/src/ggml-virtgpu/backend/shared/api_remoting.h create mode 100644 ggml/src/ggml-virtgpu/backend/shared/apir_backend.gen.h create mode 100644 ggml/src/ggml-virtgpu/backend/shared/apir_backend.h create mode 100644 ggml/src/ggml-virtgpu/backend/shared/apir_cs.h create mode 100644 ggml/src/ggml-virtgpu/backend/shared/apir_cs_ggml.h create mode 100644 ggml/src/ggml-virtgpu/backend/shared/apir_cs_rpc.h create mode 100644 ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp create mode 100644 ggml/src/ggml-virtgpu/ggml-backend-buffer.cpp create mode 100644 ggml/src/ggml-virtgpu/ggml-backend-device.cpp create mode 100644 ggml/src/ggml-virtgpu/ggml-backend-reg.cpp create mode 100644 ggml/src/ggml-virtgpu/ggml-backend.cpp create mode 100644 ggml/src/ggml-virtgpu/ggml-remoting.h create mode 100644 ggml/src/ggml-virtgpu/ggmlremoting_functions.yaml create mode 100644 ggml/src/ggml-virtgpu/include/apir_hw.h create mode 100755 ggml/src/ggml-virtgpu/regenerate_remoting.py create mode 100644 ggml/src/ggml-virtgpu/virtgpu-apir.h create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward-backend.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward-buffer-type.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward-buffer.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward-impl.h create mode 100644 ggml/src/ggml-virtgpu/virtgpu-forward.gen.h create mode 100644 ggml/src/ggml-virtgpu/virtgpu-shm.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-shm.h create mode 100644 ggml/src/ggml-virtgpu/virtgpu-utils.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu-utils.h create mode 100644 ggml/src/ggml-virtgpu/virtgpu.cpp create mode 100644 ggml/src/ggml-virtgpu/virtgpu.h create mode 100644 ggml/src/ggml-vulkan/CMakeLists.txt create mode 100644 ggml/src/ggml-vulkan/cmake/host-toolchain.cmake.in create mode 100644 ggml/src/ggml-vulkan/ggml-vulkan.cpp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/CMakeLists.txt create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/acc.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/add.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/add1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/add_id.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/arange.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/argmax.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/argsort.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/argsort_large.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/clamp.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/col2im_1d.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/concat.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/contig_copy.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/conv2d_dw.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/conv2d_mm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/conv_transpose_1d.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/copy.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/copy_from_quant.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/copy_transpose.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/cos.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/count_equal.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/cumsum.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/cumsum_multipass2.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_f32.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_m.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq1_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_nl.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq4_xs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_mxfp4.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_nvfp4.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q1_0.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q2_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q3_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_0.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q4_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_0.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q5_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q6_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_q8_0.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/diag.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/diag_mask_inf.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/div.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dot_product_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/bfloat16.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/coopmat2_decode_vector.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/feature-tests/integer_dot.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/fill.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mask_opt.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_mmq_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_split_k_reduce.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/fwht.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/gated_delta_net.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/geglu.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/geglu_erf.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/geglu_quick.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/generic_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/generic_unary_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/get_rows.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/get_rows_quant.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/group_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/im2col.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/leaky_relu.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/log.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/pad.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/roll.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_funcs.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_head.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_neox.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_norm.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_params.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/rope_vision.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/scale.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/silu_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sin.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/snake.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_back.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large1.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large2.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large3.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/soft_max_large_common.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/solve_tri.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sqrt.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/square.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/ssm_conv.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/ssm_scan.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sub.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/sum_rows.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/swiglu.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/swiglu_oai.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/timestep_embedding.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/topk_argsort.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/topk_moe.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/topk_nary_search.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/tri.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/types.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/unary.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/upscale.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/utils.glsl create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/wkv6.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/wkv7.comp create mode 100644 ggml/src/ggml-webgpu/CMakeLists.txt create mode 100644 ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp create mode 100644 ggml/src/ggml-webgpu/ggml-webgpu.cpp create mode 100644 ggml/src/ggml-webgpu/pre_wgsl.hpp create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/add_id.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/argmax.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/argsort.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/argsort_merge.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/binary.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/concat.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/conv2d.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/cpy.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/cumsum.wgsl create mode 100755 ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_blk.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_reduce.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/gated_delta_net.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/get_rows.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/im2col.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_gather.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_id_vec.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_reg_tile.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_subgroup_matrix.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_acc.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_vec_q_acc.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/pad.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/quantize_q8.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/repeat.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/rms_norm_mul.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/rope.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/row_norm.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/scale.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/set.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/set_rows.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/set_rows_quant.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/soft_max.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/solve_tri.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/ssm_conv.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/sum_rows.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/unary.wgsl create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/upscale.wgsl create mode 100644 ggml/src/ggml-zdnn/.gitignore create mode 100644 ggml/src/ggml-zdnn/CMakeLists.txt create mode 100644 ggml/src/ggml-zdnn/common.hpp create mode 100644 ggml/src/ggml-zdnn/ggml-zdnn.cpp create mode 100644 ggml/src/ggml-zdnn/mmf.cpp create mode 100644 ggml/src/ggml-zdnn/mmf.hpp create mode 100644 ggml/src/ggml-zdnn/utils.cpp create mode 100644 ggml/src/ggml-zdnn/utils.hpp create mode 100644 ggml/src/ggml-zendnn/CMakeLists.txt create mode 100644 ggml/src/ggml-zendnn/ggml-zendnn.cpp create mode 100644 ggml/src/ggml.c create mode 100644 ggml/src/ggml.cpp create mode 100644 ggml/src/gguf.cpp create mode 100644 ggml/tests/CMakeLists.txt create mode 100644 ggml/tests/test-arange.cpp create mode 100644 ggml/tests/test-backend-ops.cpp create mode 100644 ggml/tests/test-cont.c create mode 100644 ggml/tests/test-conv-transpose-1d.cpp create mode 100644 ggml/tests/test-conv-transpose.c create mode 100644 ggml/tests/test-conv1d-dw-c1.cpp create mode 100644 ggml/tests/test-conv1d-dw-c2.cpp create mode 100644 ggml/tests/test-conv1d.cpp create mode 100644 ggml/tests/test-conv2d-dw.cpp create mode 100644 ggml/tests/test-conv2d.cpp create mode 100644 ggml/tests/test-customop.c create mode 100644 ggml/tests/test-dup.c create mode 100644 ggml/tests/test-interpolate.cpp create mode 100644 ggml/tests/test-opt.cpp create mode 100644 ggml/tests/test-pad-reflect-1d.cpp create mode 100644 ggml/tests/test-pool.c create mode 100644 ggml/tests/test-quantize-fns.cpp create mode 100644 ggml/tests/test-quantize-perf.cpp create mode 100644 ggml/tests/test-rel-pos.c create mode 100644 ggml/tests/test-roll.cpp create mode 100644 ggml/tests/test-timestep_embedding.cpp create mode 100755 models.sh create mode 100755 quantize.sh create mode 100644 src/audio-io.h create mode 100644 src/audio-postproc-stream.h create mode 100644 src/audio-postproc.h create mode 100644 src/audio-resample.h create mode 100644 src/backend.h create mode 100644 src/bpe.h create mode 100644 src/dac-decoder.h create mode 100644 src/dac-encoder.h create mode 100644 src/debug.h create mode 100644 src/duration-estimator.h create mode 100644 src/gguf-weights.h create mode 100644 src/hubert-enc.h create mode 100644 src/lang-map.h create mode 100644 src/maskgit-tts.h create mode 100644 src/omnivoice-llm.h create mode 100644 src/omnivoice.cpp create mode 100644 src/omnivoice.h create mode 100644 src/ov-error.h create mode 100644 src/philox.h create mode 100644 src/pipeline-codec.cpp create mode 100644 src/pipeline-codec.h create mode 100644 src/pipeline-tts.cpp create mode 100644 src/pipeline-tts.h create mode 100644 src/prompt-tts.h create mode 100644 src/qwen3-enc.h create mode 100644 src/rvq-codec.h create mode 100644 src/rvq-file.h create mode 100644 src/semantic-enc.h create mode 100644 src/srt.h create mode 100644 src/text-chunker-stream.h create mode 100644 src/text-chunker.h create mode 100644 src/timer.h create mode 100644 src/tts-server.h create mode 100644 src/utf8.h create mode 100644 src/voice-design.h create mode 100644 src/wav.h create mode 100644 src/weight-ctx.h create mode 100644 tests/abi-c.c create mode 100644 tests/clone-CPU-BF16.log create mode 100644 tests/clone-CPU-F32.log create mode 100644 tests/clone-CPU-Q4_K_M.log create mode 100644 tests/clone-CPU-Q8_0.log create mode 100644 tests/clone-CUDA0-BF16.log create mode 100644 tests/clone-CUDA0-F32.log create mode 100644 tests/clone-CUDA0-Q4_K_M.log create mode 100644 tests/clone-CUDA0-Q8_0.log create mode 100644 tests/clone-Vulkan0-BF16.log create mode 100644 tests/clone-Vulkan0-F32.log create mode 100644 tests/clone-Vulkan0-Q4_K_M.log create mode 100644 tests/clone-Vulkan0-Q8_0.log create mode 100755 tests/cross-decode.py create mode 100755 tests/debug-clone-cossim.py create mode 100755 tests/debug-clone-cossim.sh create mode 100755 tests/debug-tts-cossim.py create mode 100755 tests/debug-tts-cossim.sh create mode 100644 tests/tts-CPU-BF16.log create mode 100644 tests/tts-CPU-F32.log create mode 100644 tests/tts-CPU-Q4_K_M.log create mode 100644 tests/tts-CPU-Q8_0.log create mode 100644 tests/tts-CUDA0-BF16.log create mode 100644 tests/tts-CUDA0-F32.log create mode 100644 tests/tts-CUDA0-Q4_K_M.log create mode 100644 tests/tts-CUDA0-Q8_0.log create mode 100644 tests/tts-Vulkan0-BF16.log create mode 100644 tests/tts-Vulkan0-F32.log create mode 100644 tests/tts-Vulkan0-Q4_K_M.log create mode 100644 tests/tts-Vulkan0-Q8_0.log create mode 100644 tools/omnivoice-codec.cpp create mode 100644 tools/omnivoice-tts.cpp create mode 100644 tools/quantize.cpp create mode 100644 tools/tts-server.cpp create mode 100644 tools/version.cmake create mode 100644 update.cmd create mode 100755 update.sh create mode 100644 vendor/cpp-httplib/CMakeLists.txt create mode 100644 vendor/cpp-httplib/LICENSE create mode 100644 vendor/cpp-httplib/httplib.cpp create mode 100644 vendor/cpp-httplib/httplib.h create mode 100644 vendor/yyjson/yyjson.c create mode 100644 vendor/yyjson/yyjson.h diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..e6a4eb9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,200 @@ +cmake_minimum_required(VERSION 3.14) +project(omnivoice-ggml LANGUAGES C CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# version.h: embed git commit hash into all binaries. +# runs on every build, only rewrites if the hash changed. +set(VERSION_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/version.h") +add_custom_target(version ALL + COMMAND "${CMAKE_COMMAND}" "-DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DOUTPUT=${VERSION_OUTPUT}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tools/version.cmake" + BYPRODUCTS "${VERSION_OUTPUT}" + COMMENT "Checking git version" +) + +# pthread: required explicitly on older glibc (< 2.34) where libpthread +# is not merged into libc. Modern distros link it implicitly but aarch64 +# and older x86_64 toolchains need the explicit dependency. +find_package(Threads REQUIRED) + +# Suppress MSVC fopen/sprintf deprecation warnings, force UTF-8 source and +# execution charsets so non-ASCII string literals (CJK in voice-design.h, +# lang-map.h, text-chunker.h) survive the compile without a BOM. /utf-8 is +# restricted to C and C++ since nvcc treats a bare /utf-8 as an input +# filename and aborts with "A single input file is required". CUDA sources +# do not carry CJK literals so they do not need this flag. +if(MSVC) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) + add_compile_options($<$:/utf-8>) +endif() + +# Put executables and backend .so in the same directory (build root). +# Without this, ggml defaults to bin/ for .so but executables stay in root, +# and ggml_backend_load_all() can't find the backends at runtime. +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) + +# Audio tokenizer tensor names can exceed default GGML_MAX_NAME of 64 +add_compile_definitions(GGML_MAX_NAME=128) + +# Harden: mark fread/fwrite/etc with warn_unused_result on all platforms +# SYCL excluded: _FORTIFY_SOURCE swaps memcpy for __memcpy_chk, unresolved in device code +if(NOT MSVC AND NOT GGML_SYCL) + add_compile_definitions(_FORTIFY_SOURCE=2) +endif() + +# CUDA architectures: cover Turing to Blackwell for distributed binaries. +# Users can override with -DCMAKE_CUDA_ARCHITECTURES=native for local builds. +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + find_package(CUDAToolkit QUIET) + if(CUDAToolkit_FOUND AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8") + set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real;120a-real;121a-real") + else() + set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real") + endif() +endif() + +# ggml as subdirectory, inherits GGML_CUDA, GGML_METAL, etc. from cmake flags +# CUDA graphs default on: standalone ggml ships them off. Overridable with +# -DGGML_CUDA_GRAPHS=OFF or at runtime with GGML_CUDA_DISABLE_GRAPHS=1. +if(NOT DEFINED GGML_CUDA_GRAPHS) + set(GGML_CUDA_GRAPHS_DEFAULT ON) +endif() +add_subdirectory(ggml) + +# cpp-httplib (HTTP server library, no SSL, behind reverse proxy in prod). +# Used by tts-server. +add_subdirectory(vendor/cpp-httplib) + +# yyjson (MIT, fast JSON parser/writer). Used by tts-server. +add_library(yyjson STATIC vendor/yyjson/yyjson.c) +target_include_directories(yyjson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/yyjson) +if(MSVC) + target_compile_options(yyjson PRIVATE /W0) +else() + target_compile_options(yyjson PRIVATE -w) +endif() + +# Shared compile options and ggml linkage +macro(link_ggml_backends target) + target_include_directories(${target} PRIVATE + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_SOURCE_DIR} + ${CMAKE_BINARY_DIR} + ) + target_include_directories(${target} SYSTEM PRIVATE + ${CMAKE_SOURCE_DIR}/ggml/include + ) + if(MSVC) + target_compile_options(${target} PRIVATE /W4 /wd4100 /wd4505) + else() + target_compile_options(${target} PRIVATE -Wall -Wextra -Wshadow -Wconversion + -Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion) + endif() + target_link_libraries(${target} PRIVATE ggml Threads::Threads) + if(TARGET ggml-base) + target_link_libraries(${target} PRIVATE ggml-base) + endif() + foreach(backend cpu blas cuda metal vulkan sycl) + if(TARGET ggml-${backend}) + get_target_property(CURRENT_BACKEND_TYPE ggml-${backend} TYPE) + if (CURRENT_BACKEND_TYPE STREQUAL "MODULE_LIBRARY") + # DL mode: backend is loaded at runtime via dlopen, + # skip all link-time deps. + continue() + endif() + target_link_libraries(${target} PRIVATE ggml-${backend}) + endif() + endforeach() + + # SYCL links its runtime PRIVATE, consumers need -fsycl to resolve libsycl.so + if(TARGET ggml-sycl AND GGML_SYCL) + target_link_options(${target} PRIVATE -fsycl) + endif() + add_dependencies(${target} version) +endmacro() + +# Core library always STATIC : the bundled CLI tools include +# pipeline-tts.h / pipeline-codec.h / backend.h directly for the debug +# paths (--llm-test, --maskgit-test) and need every pipeline_* / backend_* +# symbol resolved without going through the public ABI. OMNIVOICE_STATIC +# is propagated PUBLIC : the lib's own .cpp files see it (so OV_API +# resolves to empty when compiling omnivoice.cpp on Windows), and every +# consumer that links omnivoice-core inherits it too (same effect on +# their side, no spurious dllimport on a static archive). The shared +# library for ABI consumers is a separate, opt-in target below. +add_library(omnivoice-core STATIC + src/omnivoice.cpp + src/pipeline-codec.cpp + src/pipeline-tts.cpp +) +target_compile_definitions(omnivoice-core PUBLIC OMNIVOICE_STATIC) +link_ggml_backends(omnivoice-core) + +# Public shared library for ABI consumers (Python ctypes, Rust bindgen, +# Go cgo). Opt-in : -DOMNIVOICE_SHARED=ON at configure time. Exports +# only the OV_API-marked symbols ; every internal pipeline_* / backend_* +# stays hidden inside the .so. Intentionally a different target name +# from omnivoice-core so the static path used by the tools is never +# affected. +option(OMNIVOICE_SHARED "Build the shared omnivoice library for ABI consumers" OFF) +if(OMNIVOICE_SHARED) + add_library(omnivoice SHARED + src/omnivoice.cpp + src/pipeline-codec.cpp + src/pipeline-tts.cpp + ) + target_compile_definitions(omnivoice PRIVATE OMNIVOICE_BUILD) + set_target_properties(omnivoice PROPERTIES + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + ) + link_ggml_backends(omnivoice) +endif() + +# quantize: GGUF requantizer (BF16 -> K-quants), shared with acestep.cpp policy +add_executable(quantize tools/quantize.cpp) +link_ggml_backends(quantize) + +# omnivoice-codec : standalone codec CLI (codes <-> WAV) +add_executable(omnivoice-codec tools/omnivoice-codec.cpp) +target_link_libraries(omnivoice-codec PRIVATE omnivoice-core) +link_ggml_backends(omnivoice-codec) + +# omnivoice-tts : full TTS pipeline. Grows phase by phase, currently load-only. +add_executable(omnivoice-tts tools/omnivoice-tts.cpp) +target_link_libraries(omnivoice-tts PRIVATE omnivoice-core) +link_ggml_backends(omnivoice-tts) + +# tts-server : OpenAI-compatible HTTP server over the TTS pipeline. +# Always built : it is part of the project, not an optional add-on. +add_executable(tts-server tools/tts-server.cpp) +target_link_libraries(tts-server PRIVATE omnivoice-core httplib yyjson) +link_ggml_backends(tts-server) + +# test-abi-c : pure C99 smoke test that locks in the public ABI contract. +# Compiles omnivoice.h with a C compiler under -Wall -Werror -pedantic and +# links against the static lib. The test never loads a model ; failure +# means the public API regressed. Built by default so a regression breaks +# the main build, not just an opt-in target. +add_executable(test-abi-c tests/abi-c.c) +set_target_properties(test-abi-c PROPERTIES + C_STANDARD 99 + C_STANDARD_REQUIRED ON + C_EXTENSIONS OFF +) +if(MSVC) + target_compile_options(test-abi-c PRIVATE /W4 /WX) +else() + target_compile_options(test-abi-c PRIVATE -Wall -Werror -pedantic) +endif() +target_include_directories(test-abi-c PRIVATE + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR} +) +target_link_libraries(test-abi-c PRIVATE omnivoice-core) +link_ggml_backends(test-abi-c) + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9a958fb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# Используем базовый образ Debian Trixie (13) +FROM debian:trixie-slim + +# Отключаем интерактивные запросы при установке пакетов +ENV DEBIAN_FRONTEND=noninteractive + +# Обновляем список пакетов и устанавливаем необходимые зависимости +RUN apt-get update && apt-get install -y \ + # Базовые утилиты + wget \ + curl \ + # вообще нахуй не надо, но пусть будет + gnupg \ + lsb-release \ + # Vulkan Loader и инструменты + libvulkan1 \ + vulkan-tools \ + # Драйверы Mesa (LLVMpipe для программного рендеринга) + mesa-vulkan-drivers \ + # Заголовочные файлы для разработки (опционально) + libvulkan-dev \ + # Дополнительные утилиты + mesa-utils \ + # Библиотека без которой ничего не работает + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# Устанавливаем переменную окружения для Vulkan Loader +ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/radeon_icd.json +#ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json + +# Опционально: копируем ваше приложение в контейнер +COPY ./app/tts-server /app/ +COPY ./app/*so* /usr/local/lib/ + +RUN ldconfig && \ + echo "/usr/local/lib" > /etc/ld.so.conf.d/custom-libs.conf && \ + ldconfig +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +WORKDIR /app +# Команда по умолчанию (замените на свою) +CMD ["/app/tts-server"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c90faa8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-2026 The omnivoice.cpp authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f4302be --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# omnivoice.cpp + +Local AI text-to-speech with voice cloning and voice design, powered +by GGML. C++17 port of OmniVoice (k2-fsa/OmniVoice). 646 languages, +24 kHz mono output, runs on CPU, CUDA, ROCm, Metal, Vulkan. + +## Features + +- Voice cloning from a reference WAV plus its transcript +- Voice design via attribute keywords (gender, age, pitch, style, + volume, emotion) +- Auto voice with consistent speaker identity across long inputs +- Long-form synthesis with punctuation-aware text chunking, voice + prompt promotion, cross-fade and pydub-strict silence removal +- Bit deterministic generation in greedy mode, seedable Philox PRNG + for stochastic sampling +- Q8_0 quantisation of the 612 M parameter Qwen3 backbone +- Two CLI tools : `omnivoice-tts` (text -> WAV) and `omnivoice-codec` + (WAV <-> RVQ codes) + +## Build + +``` +git clone --recurse-submodules https://github.com/ServeurpersoCom/omnivoice.cpp.git +cd omnivoice.cpp +./buildcuda.sh # NVIDIA GPU +./buildvulkan.sh # AMD/Intel GPU (Vulkan) +./buildcpu.sh # CPU only +./buildall.sh # all backends, runtime DL loading +NVCC_CCBIN=g++-13 ./buildcuda.sh # rolling release distros (Arch w/ GCC 16, etc.) +``` + +## Model conversion + +Pre-converted GGUFs are available on Hugging Face : + + https://huggingface.co/Serveurperso/OmniVoice-GGUF + +Drop them in `models/` and skip to the quick start. To convert from +the original checkpoint : + +``` +./checkpoints.sh # hf download k2-fsa/OmniVoice -> checkpoints/ +./convert.py # 2 GGUFs in BF16 -> models/ +./quantize.sh # base LM Q8_0 (tokenizer stays at native dtype) +``` + +## Quick start + +``` +echo "Hello world." | ./build/omnivoice-tts \ + --model models/omnivoice-base-Q8_0.gguf \ + --codec models/omnivoice-tokenizer-F32.gguf \ + --lang English -o hello.wav +``` + +Voice cloning : + +``` +./build/omnivoice-tts \ + --model models/omnivoice-base-Q8_0.gguf \ + --codec models/omnivoice-tokenizer-F32.gguf \ + --ref-wav ref.wav --ref-text ref.txt \ + --lang English -o out.wav < prompt.txt +``` + +Pre-encoded reference (`clone.sh`): `omnivoice-codec` encodes a reference WAV +into a compact `.rvq` latent (it applies the exact TTS reference +preprocessing, so the codes are bit-identical to the `--ref-wav` path). +Passing it via `--ref-rvq` skips the codec encode on every synthesis: + +``` +build/omnivoice-codec --model models/omnivoice-tokenizer-F32.gguf -i ref.wav +build/omnivoice-tts \ + --model models/omnivoice-base-Q8_0.gguf \ + --codec models/omnivoice-tokenizer-F32.gguf \ + --ref-rvq ref.rvq --ref-text ref.txt \ + --lang English -o out.wav < prompt.txt +``` + +## Embedding the library + +The CLI tools are thin wrappers over a public ABI. Single-header, +single-name-prefix, plain C linkage so that C, C++, Python ctypes, +Rust bindgen and Go cgo all consume it the same way. + +```c +#include "omnivoice.h" + +struct ov_init_params iparams; +ov_init_default_params(&iparams); +iparams.model_path = "models/omnivoice-base-Q8_0.gguf"; +iparams.codec_path = "models/omnivoice-tokenizer-F32.gguf"; + +struct ov_context * ov = ov_init(&iparams); + +struct ov_tts_params params; +ov_tts_default_params(¶ms); +params.text = "Hello world."; +params.lang = "English"; + +struct ov_audio audio = { 0 }; +ov_synthesize(ov, ¶ms, &audio); +/* audio.samples, audio.n_samples, audio.sample_rate, audio.channels */ +ov_audio_free(&audio); +ov_free(ov); +``` + +`tests/abi-c.c` is built with `-std=c99 -Wall -Werror -pedantic` on +every build, so any regression that breaks plain C consumability fails +the build, not just an opt-in target. + +For a binding-friendly shared library (libomnivoice.so / .dll / .dylib), +configure with `cmake -DOMNIVOICE_SHARED=ON ...`. The shared target +exports only the `ov_*` symbols ; every internal `pipeline_*` and +`backend_*` stays hidden inside the .so. + +See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the model, the +GGUF layout, the inference pipeline, every CLI flag, the public API +reference and the validation results. + +## License + +MIT. See [LICENSE](LICENSE). + +Upstream model : OmniVoice by Xiaomi / k2-fsa, Apache 2.0. +Audio codec : Higgs Audio v2 (`bosonai/higgs-audio-v2-tokenizer`), +Apache 2.0. diff --git a/buildall.cmd b/buildall.cmd new file mode 100644 index 0000000..092979c --- /dev/null +++ b/buildall.cmd @@ -0,0 +1,12 @@ +@echo off + +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + +rem rd /s /q build 2>nul +mkdir build 2>nul +cd build + +cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON +cmake --build . --config Release -j %NUMBER_OF_PROCESSORS% + +cd .. diff --git a/buildall.sh b/buildall.sh new file mode 100755 index 0000000..8ce58a6 --- /dev/null +++ b/buildall.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +rm -rf build +mkdir build +cd build + +export PATH=/usr/local/cuda/bin:$PATH + +cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON +cmake --build . --config Release -j "$(nproc)" diff --git a/buildcpu.sh b/buildcpu.sh new file mode 100755 index 0000000..bfdf14e --- /dev/null +++ b/buildcpu.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf build +mkdir build +cd build + +cmake .. -DGGML_BLAS=ON +cmake --build . --config Release -j "$(nproc)" diff --git a/buildcuda.cmd b/buildcuda.cmd new file mode 100644 index 0000000..e64cbac --- /dev/null +++ b/buildcuda.cmd @@ -0,0 +1,12 @@ +@echo off + +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + +rem rd /s /q build 2>nul +mkdir build 2>nul +cd build + +cmake .. -DGGML_CUDA=ON +cmake --build . --config Release -j %NUMBER_OF_PROCESSORS% + +cd .. diff --git a/buildcuda.sh b/buildcuda.sh new file mode 100755 index 0000000..67f711f --- /dev/null +++ b/buildcuda.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf build +mkdir build +cd build + +cmake .. -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc +cmake --build . --config Release -j "$(nproc)" diff --git a/buildsycl.sh b/buildsycl.sh new file mode 100755 index 0000000..0fe3818 --- /dev/null +++ b/buildsycl.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf build +mkdir build +cd build + +cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx +cmake --build . --config Release -j "$(nproc)" diff --git a/buildtermux.sh b/buildtermux.sh new file mode 100755 index 0000000..711d9e7 --- /dev/null +++ b/buildtermux.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf build +mkdir build +cd build + +cmake .. -DGGML_BLAS=ON -DBLAS_INCLUDE_DIRS=$PREFIX/include/openblas +cmake --build . --config Release -j "$(nproc)" diff --git a/buildvulkan.cmd b/buildvulkan.cmd new file mode 100644 index 0000000..354d000 --- /dev/null +++ b/buildvulkan.cmd @@ -0,0 +1,12 @@ +@echo off + +call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + +rem rd /s /q build 2>nul +mkdir build 2>nul +cd build + +cmake .. -DGGML_VULKAN=ON +cmake --build . --config Release -j %NUMBER_OF_PROCESSORS% + +cd .. diff --git a/buildvulkan.sh b/buildvulkan.sh new file mode 100755 index 0000000..44aee3c --- /dev/null +++ b/buildvulkan.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf app +mkdir app +cd app + +cmake .. -DGGML_VULKAN=ON +cmake --build . --config Release -j "$(nproc)" diff --git a/checkpoints.sh b/checkpoints.sh new file mode 100755 index 0000000..ad35223 --- /dev/null +++ b/checkpoints.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Download OmniVoice checkpoint from HuggingFace. +# Usage: ./checkpoints.sh + +set -eu + +DIR="checkpoints" +mkdir -p "$DIR" + +HF="hf download --quiet" + +dl_repo() { + local name="$1" repo="$2" + local target="$DIR/$name" + if [ -d "$target" ] && [ "$(ls "$target"/*.safetensors 2>/dev/null | wc -l)" -gt 0 ]; then + echo "[OK] $name" + return + fi + echo "[Download] $name <- $repo" + $HF "$repo" --local-dir "$target" +} + +# k2-fsa/OmniVoice : LLM core (Qwen3 0.6B + audio_emb + audio_heads) +# audio_tokenizer/ subdir (HuBERT + DAC + RVQ + fc/fc2) +# tokenizer.json + chat_template.jinja +dl_repo "OmniVoice" "k2-fsa/OmniVoice" diff --git a/convert.py b/convert.py new file mode 100755 index 0000000..20c855a --- /dev/null +++ b/convert.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +# convert.py: safetensors to GGUF for OmniVoice (LM + audio tokenizer). +# Reads from checkpoints/OmniVoice/, writes 2 byte-perfect F32 GGUFs to models/. +# The source checkpoint is 100% F32 (k2-fsa/OmniVoice on Hugging Face), +# so this converter never downcasts : every tensor is written in its native +# source dtype and quantize.sh derives BF16 / Q8_0 from these F32 GGUFs. +# +# omnivoice-base-F32.gguf Qwen3 0.6B + audio_embeddings + audio_heads + tokenizer +# omnivoice-tokenizer-F32.gguf HuBERT + DAC encoder/decoder + RVQ + fc/fc2 + +import json +import os +import struct +import sys + +import numpy as np +import gguf + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CHECKPOINT_DIR = os.path.join(SCRIPT_DIR, "checkpoints", "OmniVoice") +OUTPUT_DIR = os.path.join(SCRIPT_DIR, "models") + +def log(tag, msg): + print("[%s] %s" % (tag, msg), file=sys.stderr, flush=True) + +# safetensors header reader +def read_sf_header(path): + with open(path, "rb") as f: + n = struct.unpack("", + "<|lang_start|>", "<|lang_end|>", + "<|instruct_start|>", "<|instruct_end|>", + "<|text_start|>", "<|text_end|>", + ]: + if tag_name in special_map: + key = "omnivoice.special." + tag_name.strip("<|>").replace("|", "_") + w.add_uint32(key, special_map[tag_name]) + + # tokenizer + add_bpe_tokenizer(w, tok_path, tag) + + # tensors + meta, hdr_size = read_sf_header(sf_path) + n_tensors, n_bytes = 0, 0 + n_skip = 0 + for name in sorted(meta.keys()): + if is_skip_base_tensor(name): + n_skip += 1 + continue + nb = add_tensor_passthrough(w, name, sf_path, hdr_size, meta[name]) + n_tensors += 1 + n_bytes += nb + log(tag, "total: %d tensors, %.1f MB (%d skipped)" % + (n_tensors, n_bytes / (1 << 20), n_skip)) + + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file(progress=True) + w.close() + + out_mb = os.path.getsize(output_path) / (1 << 20) + log(tag, "wrote %.0f MB -> %s" % (out_mb, output_path)) + +# Tensors to skip in audio_tokenizer (training-only, not used at inference) +def is_skip_tokenizer_tensor(name): + # decoder_semantic + fc1: auxiliary HuBERT-feature reconstruction loss + if name.startswith("decoder_semantic.") or name.startswith("fc1."): + return True + # RVQ codebook EMA buffers (cluster_size, embed_avg, inited): training only + if "quantizer.quantizers." in name: + if name.endswith(".cluster_size") or name.endswith(".embed_avg") or name.endswith(".inited"): + return True + # parametrizations.weight.original{0,1}: replaced by folded weight_norm below + if "parametrizations.weight.original" in name: + return True + return False + +# Fold HuBERT pos_conv weight_norm: weight = v * g / ||v||_{dim=(0,1)} +# Convention: torch.nn.utils.weight_norm(conv, dim=2) for grouped conv1d. +# Verified bit-identical to torch._weight_norm(v, g, dim=2) up to FP32 noise. +def fold_pos_conv_weight_norm(meta, hdr_size, sf_path): + g_name = "semantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original0" + v_name = "semantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original1" + target = "semantic_model.encoder.pos_conv_embed.conv.weight" + + g = read_sf_tensor(sf_path, hdr_size, meta[g_name]) # (1, 1, 128) + v = read_sf_tensor(sf_path, hdr_size, meta[v_name]) # (768, 48, 128) + norm = np.sqrt(np.sum(v * v, axis=(0, 1), keepdims=True)) + weight = v * g / norm + return target, weight.astype(np.float32, copy=False) + +# omnivoice-tokenizer: HuBERT + DAC + RVQ + fc/fc2 +def convert_tokenizer(ckpt_dir, output_path): + tag = "TOK" + audio_dir = os.path.join(ckpt_dir, "audio_tokenizer") + cfg_path = os.path.join(audio_dir, "config.json") + sf_path = os.path.join(audio_dir, "model.safetensors") + + with open(cfg_path, "r", encoding="utf-8") as f: + cfg = json.load(f) + + log(tag, "writing %s" % os.path.basename(output_path)) + w = gguf.GGUFWriter(output_path, "omnivoice-tokenizer", use_temp_file=True) + w.add_name("OmniVoice-tokenizer") + + # global audio config + w.add_uint32("omnivoice.sample_rate", cfg["sample_rate"]) + w.add_uint32("omnivoice.semantic_sample_rate", cfg["semantic_sample_rate"]) + w.add_uint32("omnivoice.downsample_factor", cfg["downsample_factor"]) + w.add_uint32("omnivoice.codebook_size", cfg["codebook_size"]) + w.add_uint32("omnivoice.codebook_dim", cfg["codebook_dim"]) + + # acoustic (DAC) config + ac = cfg["acoustic_model_config"] + w.add_uint32("omnivoice.acoustic.encoder_hidden_size", ac["encoder_hidden_size"]) + w.add_uint32("omnivoice.acoustic.decoder_hidden_size", ac["decoder_hidden_size"]) + w.add_uint32("omnivoice.acoustic.hidden_size", ac["hidden_size"]) + w.add_uint32("omnivoice.acoustic.n_codebooks", ac["n_codebooks"]) + w.add_uint32("omnivoice.acoustic.hop_length", ac["hop_length"]) + w.add_array("omnivoice.acoustic.upsampling_ratios", + [int(x) for x in ac["upsampling_ratios"]]) + w.add_array("omnivoice.acoustic.downsampling_ratios", + [int(x) for x in ac["downsampling_ratios"]]) + + # semantic (HuBERT) config + sm = cfg["semantic_model_config"] + w.add_uint32("omnivoice.semantic.hidden_size", sm["hidden_size"]) + w.add_uint32("omnivoice.semantic.intermediate_size", sm["intermediate_size"]) + w.add_uint32("omnivoice.semantic.num_attention_heads", sm["num_attention_heads"]) + w.add_uint32("omnivoice.semantic.num_hidden_layers", sm["num_hidden_layers"]) + w.add_uint32("omnivoice.semantic.num_feat_extract_layers", sm["num_feat_extract_layers"]) + w.add_array("omnivoice.semantic.conv_dim", [int(x) for x in sm["conv_dim"]]) + w.add_array("omnivoice.semantic.conv_kernel", [int(x) for x in sm["conv_kernel"]]) + w.add_array("omnivoice.semantic.conv_stride", [int(x) for x in sm["conv_stride"]]) + w.add_uint32("omnivoice.semantic.num_conv_pos_embeddings", sm["num_conv_pos_embeddings"]) + w.add_uint32("omnivoice.semantic.num_conv_pos_embedding_groups", sm["num_conv_pos_embedding_groups"]) + w.add_float32("omnivoice.semantic.layer_norm_eps", float(sm["layer_norm_eps"])) + + # tensors + meta, hdr_size = read_sf_header(sf_path) + + # fold pos_conv weight_norm before iterating, so the folded weight replaces + # both parametrizations entries in the output GGUF + folded_name, folded_arr = fold_pos_conv_weight_norm(meta, hdr_size, sf_path) + + n_tensors, n_bytes = 0, 0 + n_skip = 0 + for name in sorted(meta.keys()): + if is_skip_tokenizer_tensor(name): + n_skip += 1 + continue + nb = add_tensor_passthrough(w, name, sf_path, hdr_size, meta[name]) + n_tensors += 1 + n_bytes += nb + + # folded pos_conv weight is computed F32 from F32 source params, write F32 + w.add_tensor(folded_name, folded_arr) + n_tensors += 1 + n_bytes += folded_arr.nbytes + + log(tag, "total: %d tensors, %.1f MB (%d skipped: training-only)" % + (n_tensors, n_bytes / (1 << 20), n_skip)) + + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file(progress=True) + w.close() + + out_mb = os.path.getsize(output_path) / (1 << 20) + log(tag, "wrote %.0f MB -> %s" % (out_mb, output_path)) + +def main(): + if not os.path.isdir(CHECKPOINT_DIR): + log("GGUF", "checkpoints/OmniVoice not found, run checkpoints.sh first") + sys.exit(1) + + os.makedirs(OUTPUT_DIR, exist_ok=True) + + base_path = os.path.join(OUTPUT_DIR, "omnivoice-base-F32.gguf") + tok_path = os.path.join(OUTPUT_DIR, "omnivoice-tokenizer-F32.gguf") + + if os.path.exists(base_path): + log("BASE", "skip: %s exists" % os.path.basename(base_path)) + else: + convert_base(CHECKPOINT_DIR, base_path) + + if os.path.exists(tok_path): + log("TOK", "skip: %s exists" % os.path.basename(tok_path)) + else: + convert_tokenizer(CHECKPOINT_DIR, tok_path) + + log("GGUF", "done -> %s" % OUTPUT_DIR) + +if __name__ == "__main__": + main() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..41ea872 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +services: + omnivoice-server: + build: . + security_opt: + - apparmor:flatpak + ports: + - "8000:8000" + volumes: + - ./models:/models + devices: + - "/dev/kfd:/dev/kfd" + - "/dev/dri:/dev/dri" + group_add: + - video + ipc: host + cap_add: + - SYS_PTRACE + environment: + - HSA_OVERRIDE_GFX_VERSION=11.0.0 + command: > + /app/tts-server + --model /models/omnivoice-base-Q4_K_M.gguf + --codec /models/omnivoice-tokenizer-Q4_K_M.gguf + --host 0.0.0.0 + --port 8000 +# command: > +# vulkaninfo | grep "deviceName" + logging: + options: + max-size: "1m" + max-file: "1" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..c842e44 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,900 @@ +# Architecture + +Technical reference for omnivoice.cpp, the GGML port of OmniVoice +(k2-fsa/OmniVoice). This document covers the model, the conversion to +GGUF, the inference pipeline, the GGML graph conventions, and the CLI +tools. + +## Upstream model + +OmniVoice (Xiaomi / k2-fsa, Apache 2.0) is a multilingual zero-shot +text-to-speech system covering 646 languages. It targets three modes : + + voice cloning reference audio plus reference transcript drive the + target speaker identity + voice design six attribute categories (gender, age, pitch, style, + volume, emotion) drive a synthesised speaker + auto voice no reference, the model picks a coherent speaker per + utterance + +The system is a non autoregressive, mask-predict (MaskGIT) generative +model running on top of a Qwen3 backbone with custom audio +input/output and a separate audio tokenizer. The audio tokenizer is +the Higgs Audio v2 codec (`bosonai/higgs-audio-v2-tokenizer`, +Apache 2.0), which combines a HuBERT semantic stream, a DAC acoustic +stream, and an 8-codebook residual vector quantiser at 25 frames per +second over 24 kHz mono audio. + +Single public checkpoint : `k2-fsa/OmniVoice` (3.1 GB). + + Backbone Qwen3 0.6B (28 layers, hidden 1024, GQA 16/8) + Audio codebooks 8 residual, 1024 entries each plus 1 mask token + Audio framerate 25 Hz + Hop length 960 samples + Sample rate 24 kHz mono + Semantic SR 16 kHz (HuBERT input) + MaskGIT steps 32 default, configurable + +## Build + +``` +git clone --recurse-submodules https://github.com/ServeurpersoCom/omnivoice.cpp.git +cd omnivoice.cpp +./buildcuda.sh # NVIDIA GPU +./buildvulkan.sh # AMD/Intel GPU (Vulkan) +./buildcpu.sh # CPU only +./buildall.sh # all backends, runtime DL loading +``` + +The GGML submodule lives at `https://github.com/ServeurpersoCom/ggml.git` +and provides two custom ops required by the codec : +`GGML_OP_SNAKE` and `GGML_OP_COL2IM_1D`. Both have CPU, CUDA, Metal, +and Vulkan kernels. + +## Model conversion + +``` +./checkpoints.sh # hf download k2-fsa/OmniVoice -> checkpoints/OmniVoice/ +./convert.py # 2 GGUFs in BF16 -> models/ +./quantize.sh # base LM Q8_0 (tokenizer stays at native dtype) +``` + +Outputs : + +``` +models/omnivoice-base-BF16.gguf 1.2 GB LLM + audio_emb + audio_heads + tokenizer +models/omnivoice-base-Q8_0.gguf 626 MB quantized base, 1.9x smaller +models/omnivoice-tokenizer-F32.gguf 702 MB HuBERT + DAC + RVQ + fc/fc2 (native F32) +``` + +The audio tokenizer GGUF preserves the source dtype 1:1. The reference +checkpoint stores the codec at F32, so the GGUF stays F32 to avoid +truncation noise across the 8-stage RVQ residual chain. Late codebooks +fall below 50 percent codebook match against the reference if any +intermediate weight is rounded to BF16. + +Quantisation policy : Q8_0 only on the base LM. The 612 M parameter +backbone is small enough that lower quants degrade quality without +meaningful size gains. + +## GGUF layout + +`omnivoice-base-{quant}.gguf` (arch `omnivoice-lm`) : + +``` +metadata + general.architecture omnivoice-lm + block_count 28 + embedding_length 1024 + feed_forward_length 3072 + head_count 16 + head_count_kv 8 (GQA 2:1) + key_length 128 + vocab_size 151676 + context_length 40960 + layer_norm_rms_eps 1e-6 + rope_freq_base 1e6 + omnivoice.tie_word_embeddings true + omnivoice.num_audio_codebook 8 + omnivoice.audio_vocab_size 1025 + omnivoice.audio_mask_id 1024 + omnivoice.audio_codebook_weights [8, 8, 6, 6, 4, 4, 2, 2] + omnivoice.special.denoise 151669 + omnivoice.special.lang_start 151670 + omnivoice.special.lang_end 151671 + omnivoice.special.instruct_start 151672 + omnivoice.special.instruct_end 151673 + omnivoice.special.text_start 151674 + omnivoice.special.text_end 151675 + tokenizer (Qwen2 BPE, 151676 vocab, 151387 merges, 33 added_tokens) + +tensors (312) + llm.embed_tokens.weight (151676, 1024) + llm.norm.weight (1024,) + llm.layers.0..27.{q,k,v,o}_proj.weight GQA, no bias + llm.layers.0..27.self_attn.{q_norm, k_norm}.weight per-head RMSNorm (128,) + llm.layers.0..27.{input,post_attention}_layernorm.weight RMSNorm + llm.layers.0..27.mlp.{gate,up,down}_proj.weight SwiGLU, no bias + audio_embeddings.weight (8200, 1024) 8 codebooks * 1025 vocab + audio_heads.weight (8200, 1024) audio output, no bias +``` + +`omnivoice-tokenizer-{quant}.gguf` (arch `omnivoice-tokenizer`) : + +``` +metadata + omnivoice.sample_rate 24000 + omnivoice.semantic_sample_rate 16000 + omnivoice.downsample_factor 320 + omnivoice.codebook_size 1024 + omnivoice.codebook_dim 64 + omnivoice.acoustic.encoder_hidden_size 64 + omnivoice.acoustic.decoder_hidden_size 1024 + omnivoice.acoustic.hidden_size 256 + omnivoice.acoustic.n_codebooks 9 (only 8 used) + omnivoice.acoustic.hop_length 960 + omnivoice.acoustic.upsampling_ratios [8, 5, 4, 2, 3] + omnivoice.acoustic.downsampling_ratios [8, 5, 4, 2, 3] + omnivoice.semantic.hidden_size 768 (HuBERT base) + omnivoice.semantic.intermediate_size 3072 + omnivoice.semantic.num_attention_heads 12 + omnivoice.semantic.num_hidden_layers 12 + omnivoice.semantic.num_feat_extract_layers 7 + omnivoice.semantic.conv_dim [512]*7 + omnivoice.semantic.conv_kernel [10, 3, 3, 3, 3, 2, 2] + omnivoice.semantic.conv_stride [5, 2, 2, 2, 2, 2, 2] + omnivoice.semantic.num_conv_pos_embeddings 128 + omnivoice.semantic.num_conv_pos_embedding_groups 16 + omnivoice.semantic.layer_norm_eps 1e-5 + +tensors (486) + acoustic_encoder.* DAC encoder, 5 blocks, downsamples 8 5 4 2 3 + acoustic_decoder.* DAC decoder, 5 blocks, upsamples 8 5 4 2 3 + encoder_semantic.* semantic conv blocks + semantic_model.* HuBERT base, weight_norm folded + quantizer.quantizers.0..7.{codebook.embed, project_in.{w,b}, project_out.{w,b}} + fc.{weight, bias} 1024 -> 1024 (after concat acoustic + semantic) + fc2.{weight, bias} 1024 -> 256 (before DAC decoder) +``` + +Single weight_norm fold at convert time : +`semantic_model.encoder.pos_conv_embed.conv.weight`, formula +`weight = v * g / ||v||_{dim=(0,1)}` matching +`torch._weight_norm(v, g, dim=2)`. Validated bit-perfect, max abs diff +3.9e-7 against the PyTorch reference. + +## Component architecture + +### Qwen3 0.6B backbone with custom IO + +Standard Qwen3 modulo two changes : + + input embed hybrid text plus audio, weighted sum across 8 + codebooks gated by `audio_mask` + output head custom `audio_heads` Linear (8200, 1024), no text + `lm_head` + +``` +input_ids [B, 8, S] int (text on row 0, audio codes on rows 1..7) +audio_mask [B, S] bool + +text_emb = embed_tokens(input_ids[:, 0, :]) (B, S, 1024) +shifted = input_ids * audio_mask + offsets[None, :, None] + offsets = arange(8) * 1025 +audio_emb = audio_embeddings(shifted).sum(dim=1) (B, S, 1024) +inputs = where(audio_mask, audio_emb, text_emb) (B, S, 1024) + +x = qwen3_forward(inputs, attention_mask, position_ids) (B, S, 1024) + +logits_flat = x @ audio_heads.weight.T (B, S, 8200) +logits = reshape (B, 8, S, 1025) +``` + +Qwen3 specifics already in llama.cpp : + + 28 layers, hidden 1024, intermediate 3072 + 16 query heads + 8 KV heads (GQA 2:1), head_dim 128 + per-head RMSNorm on Q and K (q_norm, k_norm shape (128,)) before RoPE + no bias on Q/K/V/O/MLP + RoPE theta = 1e6 + SwiGLU MLP + tie_word_embeddings = true (`lm_head` absent, output goes through audio_heads) + +### MaskGIT decoder + +Iterative non autoregressive decoder, no KV cache. Each step is a full +prefill of the LLM on the current input. + +Prompt (per item, broadcast across 8 codebooks) : + +``` +[<|denoise|>]? +<|lang_start|> {iso_code or "None"} <|lang_end|> +<|instruct_start|> {style or "None"} <|instruct_end|> +<|text_start|> {ref_text + " " + text} <|text_end|> +{ref_audio_codes}? +{MASK x num_target_tokens} +``` + +Unconditional prompt for CFG = the trailing `num_target_tokens` mask +tokens only. Batched (cond + uncond) doubles the batch dim. + +``` +for step in 0..num_step-1 : + forward(input_ids, audio_mask, attention_mask) (2B, 8, S, 1025) + log_probs = log_softmax(c + cfg_scale * (c - u)) + log_probs[..., MASK_ID] = -inf + if class_temp > 0 : + keep_top_k_ratio(log_probs, 0.1) + gumbel_sample(temp = class_temp) + pred = argmax(log_probs) + score = log_probs.max - layer_idx * layer_penalty (5.0) + if pos_temp > 0 : + score += gumbel * pos_temp + score[already_unmasked] = -inf + topk_idx = topk(score.flatten(), schedule[step]) + tokens[topk_idx] = pred[topk_idx] + update batch_input_ids cond and uncond +``` + +Schedule of newly unmasked positions per step is computed from +`_get_time_steps(t_start=0, t_end=1, num_step, t_shift=0.1)` then +`ceil(N_total * (t[step+1] - t[step]))`. 32 steps default. + +KV cache is not usable across MaskGIT steps. The attention is fully +bidirectional, so the prefix hidden states depend on the current +target state through every layer. As tokens get progressively +unmasked the K and V tensors of the prefix at every layer above the +embeddings drift, which forbids the standard prefix-cache trick that +works for causal LMs. Each step is therefore a full prefill of the +LLM at cost `2 * forward_full(B, S)` (the 2 accounts for the cond + +uncond CFG rows). + +Determinism. With `class_temperature = 0` and `position_temperature = 0` +the decoder is bit deterministic. Higher temperatures rely on a +seedable Philox4x32-10 PRNG. The pipeline threads the Philox counter +across MaskGIT calls so that chunked inference matches the global RNG +drift of the PyTorch reference. + +#### Inner-loop optimisations + +The num_step iterations of one chunk run on a fixed shape (`S`, `K`, `B'`) so +the per-step overhead can be cut without touching the math. + +`pipeline_tts_llm_forward_batched` accepts a `T_audio` parameter that +narrows the GPU output to the audio window only. The MaskGIT decoder +reads cond logits at `[S - T, S)` on row 0 and uncond logits at +`[0, T)` on row 1, so the full `[B', V, K, S]` tensor is wasteful. +With `T_audio > 0` the function builds two `ggml_view_4d` over those +ranges, makes them contiguous via `ggml_cont`, and only those two +sub-tensors are flagged as graph outputs. The GPU to CPU copy shrinks +from `B' * V * K * S` floats to `2 * V * K * T_audio` floats, around +5.6x less on the typical voice cloning shape (S ~ 1880, T ~ 350). +When `T_audio == 0` the function falls back to the full output, used +by the debug dump path that needs every position. + +`MaskgitBatchedCtx` holds the inputs that stay constant across the +steps : the F32 audio mask and its complement, the RoPE position +vector, and the F16 attention bias. The bias is the heaviest piece, +`B' * S * S` F16 conversions per step (about 7 M ops on the typical +shape). `pipeline_tts_llm_batched_ctx_init` precomputes the bias once +per chunk, with a single `ggml_fp32_to_fp16` call for each of the two +distinct values (1.0 and 0.0) hoisted out of the conversion loop. The +context also keeps the original int32 pointers so the debug loop path +can hand them down to the single forward unchanged. + +Both optimisations preserve the math exactly, the only side effect is +a slight reordering of the GPU FP32 reductions when the scheduler +fuses the new output nodes differently, which moves the audio cosine +similarity by a few times 1e-6. Token-level results stay 100 percent +exact against the PyTorch reference. + +### Audio tokenizer pipeline + +Encode (voice cloning reference path) : + +``` +ref_audio @ 24 kHz (1, 1, T_samples) + -> resample 16 kHz (kaiser polyphase) + -> pad 160 each side + -> HuBERT.feature_extractor (320x downsample) + -> HuBERT.feature_projection (LayerNorm + Linear) + -> + pos_conv_embed (folded) + -> 12 transformer layers + -> mean over 13 hidden states (1, 768, T_sem) + -> downsample by 2 (semantic_downsample_factor) + -> SemanticEncoder (conv blocks) (1, 768, T_frames) + -> e_acoustic (DAC encoder, 5 down-blocks) (1, 256, T_frames) + -> concat dim=1 (1, 1024, T_frames) + -> fc Linear (1024 -> 1024) (1, 1024, T_frames) + -> RVQ encode (8 codebooks residual) (1, 8, T_frames) int @ 25 fps +``` + +Decode (TTS path) : + +``` +codes [B, 8, T] int + -> RVQ decode : + for k in 0..7 : + e_k = codebook[k].embed[codes[k, :]] (B, T, 64) + p_k = e_k @ project_out[k].W.T + bias[k] (B, T, 1024) + out += p_k + -> transpose (B, 1024, T) + -> fc2 Linear (1024 -> 256) (B, 256, T) + -> acoustic_decoder DAC : + conv1 (256 -> 1024, k=7, pad=3) + for block in 0..4, ratios [8, 5, 4, 2, 3] : + snake1 (alpha) + conv_t1 (IC -> OC, k=2*r, stride=r, + padding=ceil(r/2), output_padding=r%2) + for res_unit in 0..2, dilations [1, 3, 9] : + snake1 (alpha) + conv1 (OC, OC, k=7, dil=d, pad=3*d) + snake2 (alpha) + conv2 (OC, OC, k=1) + residual add + snake1 (alpha) final + conv2 (32 -> 1, k=7, pad=3) + -> audio (B, 1, 960*T) +``` + +960x upsample = 8 * 5 * 4 * 2 * 3. T_in @ 25 fps -> T_out @ 24 kHz exact. + +### DAC decoder block channels + +``` +block 0 : IC=1024 OC=512 stride=8 K=16 pad=4 output_pad=0 +block 1 : IC=512 OC=256 stride=5 K=10 pad=3 output_pad=1 +block 2 : IC=256 OC=128 stride=4 K=8 pad=2 output_pad=0 +block 3 : IC=128 OC=64 stride=2 K=4 pad=1 output_pad=0 +block 4 : IC=64 OC=32 stride=3 K=6 pad=2 output_pad=1 +final : 32 -> 1 +``` + +PyTorch ConvTranspose1d formula : +`T_out = (T_in - 1)*stride - 2*padding + dilation*(kernel - 1) + output_padding + 1` + +With our parameters (d=1, k=2*s, p=ceil(s/2), op=s%2) the formula +collapses to `T_out = stride * T_in` exactly for all five blocks. + +### Snake activation + +DAC reference formula (Hugging Face `Snake1d.forward`) : +`y = x + (alpha + 1e-9).reciprocal() * sin(alpha * x)^2` + +`ggml_snake(x, a, inv_b)` computes `y = x + sin^2(a * x) * inv_b`. +Mapping : + + a = alpha (loaded direct, BF16 to F32) + inv_b = 1/(alpha + 1e-9) (precomputed CPU side at load, F32) + +Both stored as F32 `[1, C]` tensors. `alpha` shape in checkpoint : +`(1, C, 1)`, ggml ne = (1, C, 1). C lives on ne[1]. Loader reads C from +`mt->ne[1]`. + +### ConvTranspose1d via GEMM + col2im_1d + +PyTorch `nn.ConvTranspose1d(IC, OC, kernel=K, stride=s, padding=p)` with +weight shape `(IC, OC, K)`. GGML decomposition : + +``` +1. Permute weight (IC, OC, K) PyTorch -> (IC, K*OC) ggml at load time. + Layout : dst[(oc*K + k) * IC + ic] = src[ic*OC*K + oc*K + k] + This makes k vary faster than oc inside the K*OC axis, matching + what ggml_compute_forward_col2im_1d_impl expects : + col_data[(oc * K + k) + t_in * K_OC] + +2. Build runtime graph : + xt = ggml_cont(ctx, ggml_transpose(ctx, x)) # [IC, T_in] + col = ggml_mul_mat(ctx, w, xt) # [K*OC, T_in] + y = ggml_col2im_1d(ctx, col, stride, OC, padding) # [T_no_op, OC] + if (output_pad > 0) + y = ggml_pad(ctx, y, output_pad, 0, 0, 0) # right-pad zeros + if (bias) + y = ggml_add(ctx, y, bias_2d) +``` + +Validated math : `T_no_op = (T_in - 1)*stride + K - 2*pad`. Adding +`output_pad` right-pad gives the PyTorch output size exactly. + +### RVQ codec + +Per-codebook tensors (k = 0..7) : + +``` +codebook.embed (1024, 64) PyTorch -> ggml ne=(64, 1024) +project_in.weight (64, 1024) PyTorch -> ggml ne=(1024, 64) encode-only +project_in.bias (64,) encode-only +project_out.weight (1024, 64) PyTorch -> ggml ne=(64, 1024) +project_out.bias (1024,) +``` + +Decode graph (per codebook k, accumulated) : + +``` +codes_k = ggml_view_1d(codes, T, k * stride) # [T] i32 +e_k = ggml_get_rows(embed[k], codes_k) # [64, T] +p_k = ggml_mul_mat(project_out_w[k], e_k) # [1024, T] +p_k = ggml_add(p_k, project_out_b[k]) +acc += p_k +``` + +Encode (residual loop) : + +``` +residual = embeddings_in +for k in 0..7 : + e_k = project_in[k](residual) + codes_k = argmin_i ||e_k - codebook[k].embed[i]||^2 + quantized = project_out[k](codebook[k].embed[codes_k]) + residual -= quantized +``` + +### HuBERT semantic encoder + +12 transformer layers Pre-LN, GELU FFN, MHA 12 heads * 64 dim, biases +on all QKVO. Pre-conv feature extractor : 7 Conv1D layers, kernels +`[10, 3, 3, 3, 3, 2, 2]`, strides `[5, 2, 2, 2, 2, 2, 2]`, GroupNorm on +the first only, GELU between. Feature projection LayerNorm + Linear +(512 -> 768). Positional embedding via grouped Conv1D (128 kernel, +16 groups), `weight_norm` folded at convert time. Final LayerNorm. + +Output computation : + +``` +mean(stack(all_13_hidden_states, dim=1), dim=1) # (B, T_sem, 768) +``` + +This is unusual : the encoder averages across the initial input plus +the 12 transformer layer outputs, not just the last hidden state. + +## Long-form TTS pipeline + +`pipeline_tts_synthesize_long` orchestrates inputs longer than the +chunking threshold. It mirrors `_generate_chunked` in +`omnivoice/models/omnivoice.py`. + +``` +1. Estimate total target tokens via duration_estimate_tokens. +2. If T_total <= chunk_threshold_sec * frame_rate, run a single shot + pipeline_tts_synthesize and skip chunking. +3. Otherwise split text on punctuation with chunk_text_punctuation, + targeting chunk_duration_sec seconds per chunk. +4. Generate chunks sequentially : + - chunk 0 with no reference (auto voice / voice design path) or + with the external reference (cloning path) + - in the auto voice case, the audio tokens of chunk 0 become the + voice prompt for chunks 1..N, locking in the speaker identity +5. Cross-fade decoded chunks with cross_fade_chunks(rate, 0.3 s). +6. Apply post-processing on the merged waveform. +``` + +A shared Philox counter `ctr_lo` is threaded across MaskGIT calls so +PRNG state advances continuously between chunks, matching the global +`torch.cuda.manual_seed` behaviour on the reference side. + +### Text chunking + +`chunk_text_punctuation(text, chunk_len, min_chunk_len)` splits text on +sentence-ending punctuation (skipping abbreviation periods), then +merges sentences into chunks of at most `chunk_len` UTF-8 codepoints. +Undersized chunks (< `min_chunk_len`) are merged into a neighbour. +The function operates on UTF-8 strings and treats length as codepoints, +matching Python `len(str)` semantics. Per chunk character budget : + +``` +n_chars = utf8_codepoint_count(full_text) +avg_tokens_per_char = T_total / n_chars +chunk_len = (int)(chunk_duration_sec * frame_rate / avg_tokens_per_char) +``` + +`add_punctuation(text)` appends a terminal `.` (Latin) or its CJK +equivalent when missing. Used on the reference transcript when +`preprocess_prompt` is on. + +### Audio post-processing + +`audio-postproc.h` is a strict math port of `omnivoice/utils/audio.py` +plus the relevant `pydub.silence` routines. All public functions take +and return float32 mono PCM in [-1, 1] at the pipeline rate (24 kHz). +Silence detection runs on int16 samples to match pydub bit-for-bit. + +``` +remove_silence(buf, min_silence_ms, keep_silence_ms, + seek_step_ms, threshold_dbfs) + + Splits buf on contiguous silent regions where every + seek_step_ms-long frame stays below threshold_dbfs (RMS, S16, + default -50 dBFS), keeps keep_silence_ms of leading and trailing + silence around each retained segment, and concatenates the result. + +cross_fade_chunks(chunks, rate, fade_seconds) + + Concatenates audio chunks with a linear cross-fade of fade_seconds + at each junction. + +peak_normalize_half(buf) + + Scales buf so peak |x| equals 0.5. Used in pure auto voice when no + reference RMS is available. + +fade_and_pad(buf, rate, fade_seconds, pad_seconds) + + Applies a linear fade in / fade out and pads silence at the start + and end. Default fade 0.05 s, pad 0.05 s. Mirrors final reference + post-step. +``` + +`ref_rms` is plumbed end to end and decides the volume branch : + + ref_rms < 0 pure auto voice, peak_normalize_half on the + cross-faded waveform + ref_rms < 0.1 quiet reference, rescale by ref_rms / 0.1 + otherwise no-op + +When a reference WAV is provided, the CLI computes its RMS on the F32 +samples after optional silence trimming and passes it down. The same +quantity is used on the PyTorch side. + +### Voice modes + +``` +auto voice no ref-wav. Chunk 0 generates with no reference, + subsequent chunks reuse chunk 0 audio tokens as the + voice prompt. peak_normalize_half on output. + +voice design no ref-wav, --instruct provides one or more attribute + markers (gender, age, pitch, style, volume, emotion) + resolved by voice_design.h to the EN/ZH instruct + string the reference uses. Chunking behaves like + auto voice. + +voice cloning --ref-wav and --ref-text provided. The reference is + resampled to 16 kHz, run through the audio tokenizer + encoder, and the resulting RVQ codes are reused as + the voice prompt for every chunk. The reference RMS + sets the target loudness. +``` + +## Public API + +Two layers, picked by use case. + +### Top-level public ABI : src/omnivoice.h + +Single-header, plain C99, linkage `extern "C"`. The opaque `ov_context` +handle aggregates the GGML backend pair, the LM pipeline, the audio +tokenizer codec, the BPE tokenizer and the voice-design vocabulary. +One init, one free, one synthesize call covers the full TTS path. +Same names, same struct layout, same calling convention from C, C++, +Python ctypes, Rust bindgen, Go cgo or any other binding generator. + +```c +#include "omnivoice.h" + +struct ov_init_params iparams; +ov_init_default_params(&iparams); +iparams.model_path = "models/omnivoice-base-Q8_0.gguf"; +iparams.codec_path = "models/omnivoice-tokenizer-F32.gguf"; + +struct ov_context * ov = ov_init(&iparams); + +struct ov_tts_params params; +ov_tts_default_params(¶ms); +params.text = "Hello world."; +params.lang = "English"; + +struct ov_audio audio = { 0 }; +enum ov_status rc = ov_synthesize(ov, ¶ms, &audio); +if (rc == OV_STATUS_OK) { + /* audio.samples is a malloc'd buffer of audio.n_samples floats + at audio.sample_rate Hz, audio.channels = 1 (mono) */ +} +ov_audio_free(&audio); +ov_free(ov); +``` + +Status codes : + +``` +OV_STATUS_OK 0 +OV_STATUS_INVALID_PARAMS -1 (mutually exclusive ref inputs etc.) +OV_STATUS_INSTRUCT_INVALID -2 (instruct rejected by VoiceDesign) +OV_STATUS_GENERATE_FAILED -3 (any internal generate / decode fail) +OV_STATUS_OOM -4 (output samples allocation failed) +OV_STATUS_CANCELLED -5 (cancel callback returned true) +``` + +`ov_tts_params` exposes `cancel` and `cancel_user_data`. The pipeline +polls between chunks of long-form output, so cancel granularity is +roughly `chunk_duration_sec` (15 s by default). + +The MaskGIT sampler config is flattened directly into `ov_tts_params` +as seven `mg_*` fields ; `ov_tts_default_params` initialises them to +the reference defaults (`num_step=32, guidance_scale=2.0, t_shift=0.1, +layer_penalty_factor=5.0, position_temperature=5.0, +class_temperature=0.0, seed=42`). + +`ov_version()` returns a static string of the form +`"MAJOR.MINOR.PATCH (git-hash, date)"`. The macros `OV_VERSION_MAJOR`, +`OV_VERSION_MINOR`, `OV_VERSION_PATCH` are also available at +compile time for feature-detection. + +### ABI guarantee + +`tests/abi-c.c` is built on every build with +`-std=c99 -Wall -Werror -pedantic`. It includes the public header, +calls every entry through its early-return path, and is wired into +the default build target. Any regression that breaks plain C +consumability fails the main build, not an opt-in step. + +The static library `libomnivoice-core.a` is the default build +artefact. For binding consumers, configure with +`-DOMNIVOICE_SHARED=ON` to add a `libomnivoice.so` (or `.dll` / +`.dylib`) shared target that exports only the `ov_*` symbols ; +every internal `pipeline_*` and `backend_*` stays hidden behind +`-fvisibility=hidden`. Install rules follow `GNUInstallDirs`. + +### Low-level API : src/pipeline-tts.h, src/pipeline-codec.h + +Direct access to the LM forward (`pipeline_tts_llm_forward`, +`pipeline_tts_llm_forward_batched`), the MaskGIT-only path +(`pipeline_tts_generate`), the codec encode / decode +(`pipeline_codec_encode`, `pipeline_codec_decode`), the instruct +resolver (`pipeline_tts_resolve_instruct`) and the manual init / free +(`pipeline_tts_load`, `pipeline_codec_load`, `backend_init`). + +Used by `--llm-test` and `--maskgit-test` in `omnivoice-tts`, by +`omnivoice-codec` for the standalone codec roundtrip, and by the +Python cossim harness through dump files. + +This layer is intentionally not part of the public ABI (C++ types in +the signatures, no visibility export). It exists for the in-tree +debug paths and stays available as long as the bundled CLI tools +need it. The handle layer above is the recommended entry for +everything else. + +## CLI tools + +### omnivoice-tts + +End-to-end synthesis : text on stdin, WAV file on disk. Verbatim +`--help` (the binary also prints an `omnivoice.cpp ()` +banner line first) : + +``` +Usage: omnivoice-tts --model --codec [options] -o < text.txt + +Required: + --model LLM GGUF (F32 / BF16 / Q8_0) + --codec Codec GGUF (omnivoice-tokenizer-*.gguf) + -o Output WAV (24 kHz mono). '-' streams to stdout (pipe friendly). + +Input: + stdin Target text to synthesise. With -o '-', stdin is read + incrementally and synthesis starts as soon as the first + sentence boundary is reached. With -o file.wav, stdin is + read fully then synthesised in one shot. + --srt Dub an SRT: synth each cue into its time slot, write one + timeline WAV ready to mux. Pairs with --ref-wav / --ref-rvq + for a cloned voice. Per cue duration comes from the SRT. + +Optional: + --format WAV output format: wav16, wav24, wav32 (default: wav16) + --lang Language label (default 'None') + --instruct Style instruction (default 'None') + --duration Output duration in seconds (default: estimate from text) + --no-denoise Omit the <|denoise|> prefix + --ref-wav Reference WAV for voice cloning + --ref-text Transcript file for the reference (required with --ref-wav / --ref-rvq) + --ref-rvq Pre-encoded reference codes from omnivoice-codec (replaces --ref-wav) + --seed Sampling seed (default: -1 for random) + --steps MaskGIT decode steps (default: 32, fewer is faster) + --no-preprocess-prompt Skip ref-wav silence trim and ref-text terminal punctuation + --chunk-duration Long-form chunk duration (default: 15.0, <= 0 disables chunking) + --chunk-threshold Activate chunking above this estimated duration (default: 30.0) + --stream-by-line Flush synthesis at each newline, one WAV header per line (-o '-') + +Debug: + --no-fa Disable flash attention + --clamp-fp16 Clamp hidden states to FP16 range + --dump Dump intermediate tensors (f32) to + --llm-test Full LLM forward, dump audio_logits + --maskgit-test Greedy MaskGIT decoder, dump audio_tokens [K, T] + (no codec decode, reads target text from stdin) +``` + +### omnivoice-codec + +Audio tokenizer round-trip : WAV to RVQ codes, RVQ codes to WAV. +Verbatim `--help` : + +``` +Usage: omnivoice-codec --model -i + +Required: + --model Codec GGUF (omnivoice-tokenizer-*.gguf) + -i Input. WAV -> encode, .rvq -> decode + +Optional: + --format WAV output format: wav16, wav24, wav32 (default: wav16) + +Output is auto-named next to input : clip.wav -> clip.rvq, clip.rvq -> clip.wav. +Encode applies the TTS reference preprocessing (RMS auto-gain, silence trim, +hop truncation); the resulting .rvq feeds omnivoice-tts --ref-rvq directly. +``` + +The `.rvq` file is a small binary container with shape `[8, T]` int32 +codes plus a header carrying the sample rate and frame rate. + +## Module map + +``` +src/ + backend.h GGML backend init, scheduler factory, env override + weight-ctx.h Generic weight context for GGUF loaders + gguf-weights.h mmap GGUF, gf_load_tensor, gf_get_* + audio-io.h WAV read, mono write (S16 / S24 / F32) + audio-resample.h Kaiser polyphase 24 kHz <-> 16 kHz + audio-postproc.h remove_silence, peak_normalize_half, fade_and_pad, + cross_fade_chunks. Strict pydub / utils.audio port. + wav.h WAV header reader (PCM16/24/F32, mono/stereo) + philox.h Philox4x32-10 counter-based PRNG, PyTorch CUDA aligned + debug.h Tensor dumper for cossim tests + + bpe.h Qwen2 / GPT-2 byte-level BPE tokenizer, GGUF loader + lang-map.h Language name to ISO 639-3 ID resolution + (auto-generated from omnivoice/utils/lang_map.py) + voice-design.h Speaker attribute validation and EN / ZH instruct + resolution (mirrors voice_design.py) + text-chunker.h chunk_text_punctuation, add_punctuation, END_PUNCTUATION + duration-estimator.h RuleDurationEstimator port (per-script weights, + Unicode category fallback) + + rvq-codec.h Residual VQ encode + decode (8 codebooks) + dac-decoder.h DAC acoustic decoder (5 blocks, ratios 8 5 4 2 3) + dac-encoder.h DAC acoustic encoder (mirror of decoder) + semantic-enc.h SemanticEncoder convs (768 -> 768) + hubert-enc.h HuBERT base (feature extractor + pos_conv + + 12 transformer layers + final LN) + + qwen3-enc.h Qwen3 transformer building blocks + omnivoice-llm.h OmniVoice TTS LLM weights and graph helpers + prompt-tts.h Prompt builder (denoise + lang + instruct + text + + ref + mask) and CFG batch stacking + maskgit-tts.h Iterative non autoregressive decoder, configurable + step count (32 default), CFG, layer penalty, gumbel + sampling, deterministic in greedy mode + + pipeline-codec.{h,cpp} Audio tokenizer end-to-end (encode and decode) + pipeline-tts.{h,cpp} Full TTS orchestration, single shot and chunked, + plus low-level entries kept available for the + debug paths and the cossim test harness + omnivoice.{h,cpp} Public ABI : opaque ov_context handle, plain C99 + header in extern "C", consumable from C, C++, + Python ctypes, Rust bindgen, Go cgo + +tools/ + omnivoice-tts.cpp CLI : text to WAV (auto / design / clone) + omnivoice-codec.cpp CLI : codes <-> WAV + quantize.cpp GGUF requantizer + version.cmake Embeds the git short hash into the binary + +tests/ + debug-tts-cossim.py Byte-level comparison of every pipeline stage + against the PyTorch reference, voice design path + debug-clone-cossim.py Same, voice cloning path + cross-decode.py Cross check : decode C++ tokens through PyTorch + codec and vice versa + prompt.txt Long-form English TTS sample + ref-audio.wav Voice cloning reference clip + ref-text.txt Transcript matching ref-audio.wav + abi-c.c Plain C99 smoke test for the public ABI ; built + with -Wall -Werror -pedantic on every build, + locks in C consumability and symbol linkage +``` + +## GGML conventions + +### Tensor shape and layout + +PyTorch shape `(out, in)` for a Linear weight stores as ggml +`ne[0]=in, ne[1]=out`. The GGUF tensor-shape array is reversed, so +reading `reversed(t.shape)` from gguf-py yields the PyTorch shape +directly. + +For PyTorch `Conv1d` weight `(OC, IC, K)`, ggml ne is `(K, IC, OC)`. +The kernel axis is innermost (contiguous in memory). + +For PyTorch `ConvTranspose1d` weight `(IC, OC, K)`, the convert-time +permutation to ggml `(IC, K*OC)` rearranges +`(oc*K + k) * IC + ic` so that `ggml_col2im_1d` receives the correct +column matrix. + +`ggml_mul_mat(A, B)` : with A.ne[0] = K (must match B.ne[0]), +A.ne[1] = M, B.ne[1] = N, output has ne = (N, M). In PyTorch terms, +A is `(M, K)`, B is `(N, K)`, output is `(M, N)`, which equals +`A @ B^T`. + +### Custom GGML ops + +Provided by the `ServeurpersoCom/ggml` fork : + +`ggml_snake(ctx, x, a, inv_b)` : `y = x + sin^2(a * x) * inv_b`. +F32 / F16 / BF16 input/output. CPU + CUDA + Metal + Vulkan. + +`ggml_col2im_1d(ctx, a, s0, oc, p0)` : scatter-add `[K*OC, T_in]` +columns into `[T_out, OC]` signal where +`T_out = (T_in - 1)*s0 + K - 2*p0`. Layout requires k to vary faster +than oc inside the K*OC axis. F32 / F16 / BF16. All backends. The +fork also folds the padding crop into this op via the `p0` parameter, +removing a follow-up `ggml_view` for the typical ConvTranspose1d use +case. + +### Backend lifecycle + +`backend_init("MOD")` then `backend_sched_new(bp, max_nodes)`. Backend +handles are shared across modules in the same binary, refcounted. The +GPU backend is the default, the CPU backend is kept as a scheduler +fallback. + +## Validation + +The reference comparison harness is `tests/debug-tts-cossim.py` and +`tests/debug-clone-cossim.py`. They run the same input through the +PyTorch reference (with TF32 disabled, eager attention) and through the +C++ binary, dump each pipeline stage to disk, and report cosine +similarity per stage. Latest run, chunked path, English long-form +prompt : + +``` +TTS chunked: Logits cos=1.000000 max 3.5e-04 + Step0 pred_tokens 99.93% (2 FP flips) + Tokens 1.000000 exact 100.00% + Audio 0.999991 + +Clone chunked: Lf hidden cos=1.000000 max 1.9e-03 + Logits cos=1.000000 + Step0 pred_tokens 100.00% + Tokens 1.000000 exact 100.00% + Audio 0.999989 +``` + +The few Step0 token flips are argmax ties at the FP epsilon (~2e-5 +between top1 and top2 at those positions), inherent to the mixed cuBLAS +vs GGML kernel arithmetic. They resorb over the 32 MaskGIT steps so +the final tokens match bit for bit and decoded audio cosine is +> 0.9999. + +## Glossary + + RVQ Residual Vector Quantisation. Stack of codebooks where + each one quantises the residual from the previous + codebook reconstruction. + + DAC Descript Audio Codec. Convolutional encoder/decoder over + residual VQ codes. + + HuBERT Hidden-Unit BERT. Transformer encoder pretrained with + masked acoustic unit prediction. Used here to extract + semantic embeddings from raw audio. + + Snake Periodic activation introduced in BigVGAN, + `y = x + (1/alpha) * sin^2(alpha * x)`. Replaces + LeakyReLU in the DAC encoder/decoder. + + CFG Classifier-Free Guidance. The model is run twice + (conditional and unconditional) and the outputs combined + as `c + scale * (c - u)` to amplify the conditional + signal. + + MaskGIT Masked Generative Image Transformer (Chang et al., + arXiv:2202.04200). Iterative non autoregressive decoder + where masked tokens are progressively unmasked over a + fixed number of steps, prioritising high-confidence + positions per step. Originally introduced for image + generation, adapted here to audio codes. + + Philox Counter-based PRNG used by PyTorch CUDA. Thread safe and + skip-ahead friendly, well suited to deterministic + chunked inference. diff --git a/examples/client.sh b/examples/client.sh new file mode 100755 index 0000000..1bcb7c1 --- /dev/null +++ b/examples/client.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Call the omnivoice OpenAI-compatible TTS server. +# Default response_format is pcm : audio streams chunked as it is generated. + +host="${1:-127.0.0.1}" +port="${2:-8080}" + +# Streaming pcm, piped straight into a player as it arrives. +# s16le mono 24 kHz. ffplay reads the raw stream from stdin. +curl -s -X POST "http://${host}:${port}/v1/audio/speech" \ + -H "Content-Type: application/json" \ + -d '{"input":"The quick brown fox jumps over the lazy dog."}' \ + | ffplay -f s16le -ar 24000 -ch_layout mono -nodisp -autoexit -i - + +# One-shot wav written to a file. +curl -s -X POST "http://${host}:${port}/v1/audio/speech" \ + -H "Content-Type: application/json" \ + -d '{"input":"This one is written to a file.","response_format":"wav"}' \ + --output out.wav +echo "wrote out.wav" diff --git a/examples/clone.cmd b/examples/clone.cmd new file mode 100644 index 0000000..ec12fdd --- /dev/null +++ b/examples/clone.cmd @@ -0,0 +1,13 @@ +@echo off + +set PATH=%~dp0..\build\Release;%PATH% + +omnivoice-tts.exe ^ + --model ..\models\omnivoice-base-Q8_0.gguf ^ + --codec ..\models\omnivoice-tokenizer-Q8_0.gguf ^ + --ref-rvq freeman.rvq ^ + --ref-text freeman.txt ^ + --lang English ^ + -o clone.wav < prompt.txt + +pause diff --git a/examples/clone.sh b/examples/clone.sh new file mode 100755 index 0000000..d844820 --- /dev/null +++ b/examples/clone.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eu + +../build/omnivoice-tts \ + --model ../models/omnivoice-base-Q8_0.gguf \ + --codec ../models/omnivoice-tokenizer-Q8_0.gguf \ + --ref-rvq freeman.rvq \ + --ref-text freeman.txt \ + --lang English \ + -o clone.wav < prompt.txt diff --git a/examples/freeman.rvq b/examples/freeman.rvq new file mode 100644 index 0000000000000000000000000000000000000000..7d299a743f8b08dc32bd619ecd112bedf99305be GIT binary patch literal 4741 zcmV;05_;{n(#J-6r#9&|t^;I|m+YvLs`kz9O1+cYCV=vr*B~oQoY9cM%E7m9)g@&5 zTGlWWB`+=6MjxL^(YzET3}F-}MO*7u&*Zi`S!iim%~H>}l*5pV$*pBa(-j)P(SczJ zVcp_d>Y88QY11`{KOTz+g&P}v7m8b=V80Di7xDOid2N@J3wJl zyv#7JKm6$wfGY)`vPB3dB*7IOyBNNr6)uguiEBD1h0W+JKPG)wml1$5d-}z3S9u_)$ z?POlDz}L)Z00UdWKwk1vpKTtH5i9YnTi}tGqME@6c}PK5@=}19E+#M1^x<64FrCqi z#;Q%d4}70H7h1~qoN?)cCnf5Kmk9JE94+Ec?J63*QdB7F6^eR!%9Exv6`%4=rhIt< zN>c>aA)};eTITys-u^_fKbeRn9hwAL(txtq`3OEXQ3F!6K(&8-;&9)o2EM|ik8%jg zZ~Re&e3Wvnu#jpEE?dL1#`PU|EDIw=X;d?og$7eVKqq9W7Jq&xFM{zW7PYD_zc(Jr`&&x|Y))H0j_QgR6@Q6*(N znb-9c5V_i=WEpTGUs*u&patRP2|n6VG>Au#v_%GcYjT~`t{@jQ#7+=4Biv@vK)l63 z2XnB&Tv?=1u%KWKR`?;tpgL8E0VSksxA7Yb{58M%MP?Iys|4Ky6_}~*EE1$R$juy) zj^*v^51)yQKfJ>i*jS_!YN5$rgCqRqtXx6u08*oz zg}l?CryPoSGhx*l3b%%l9F0&!z?4-$7BGeRk2P!1-nz=Mi`!vFYEmN#({L9s?tr9r zuxf&c5>c!*6s$5+tIkd$cQbTAs6lQ!h7aKM6jCH@XlfEjKh6=bn_0yX{*aFv>;j;f zbT2KPYL1tdg*{_UZCw>u*CfuiiSTsb8{t{PVrsPv!YpG6H1ZK&(m{*+5C&Y6(w^bm z*0)mep&a-rnLhBgJ*lD1WrOnwugdlc%k`u`sqjuZI#emYDTpI5I#^%|W{$Hh1x^ra z#;L&6E;CdHeF$6MXDZ+w9r)vBxv&tXvf#I!HRx=g>093<*FRdcDG}Ri(%z~-jHxZ= zAW(~rvnKH);b=t)B?&_l76P$NNX8Mc>PN*eB`2=2CqoUokK3;2hz!XqGLaFBw6?UI zF=-DUCwtei;WCB(+U>UkY-={WIId(W=Ulpjpd?YuzRe@kjV@gNg zDhHQPXZ>9jBZ_TEx7v!4q#Q5lH5jWR9H^s`X{pMp-XrJ z_yXM^;Q)QafhU8ZTHCCF8{?7Tf4JF|o(8k3sYPQH1c{z!2m+>xxrcZ>;?fqLR5rBf z2OfY+);CmRFHS|HR|!a+HBe$4OTPaW@Me>FPCu{~^o)uc(Rlga-iqc||tSi{1ceAOwfq?+P^_U4LlS6`%Ah7^zWC zJ0QiQ?1k@d^a6@pCXpwx@Cje&fKE$@*Ngjohfr?8&3}sZj$LH04nZ=`G2X(nL)2mp z!fVUob_XBdQ2-^~(@XnWg&~q9iU22)PutvLja5yI2G(E=E5b*fdz^-9e~X?)PE-Kj z8EHUPTif))VzX66K_1c~N5__fl?~8?NjX`;cn%W~YM~?`wJV7NSd|lSl|*0E zL^cE&f@~A(0ouqR8lhDpLB`@4iu2%Day>O!JUG)e;FFx9C};UOPoLn5(JUXUd96TBQFAdgxRpF9)` z4Qy*A_^Uy+45Wh9P$zmcI8YcArXryQWD56#$K`r(fWY-<3WIVBa)h=Tk@c!^<6{@+ z?2`}`2+0b#o6xR+At5hVrbH$39y|abr72NuG}K8^U#dkU8_??s{u+gCMiY?eaY{(4 zAlQT2@|%8$Vg_M=k%9DNAeSKuBpBF)EZCR1UC_e^5(r)3$hQ?1y^czPaof8N!K;+0 zU>g40u*s{BoRzB*O%oV1OG&z=N^1L?j@Q(Y@Sq zlUCV5FtJ$9l5Uf$M&NEr9WzSUaAFP9xNH#b8lPbHg^0myLV2f4NmOE^yBS!AIp%YR z&ER$p-`oH&Suoe!a8xihoSWW^^D2m|{ZpEc* zfC`H$Hr1PJk?9jDnH5PufFDsc1Oy;a-!no7g2ZWvQtGQ4NI=3S?m1>tGqA#2+GdBh zT#Hl9a-1TfvMF7B%4S_i42*UF6V^=4CzTtBFiHY4TiF6g@oC7Q_`#|=vEp9)fz=3> zGc0!a!!B7$$)7TikSaZZYzEqtn<}T9SVTY%CSqLkk};8s_=ig3=mm*3LAG`bEM<^E zgp4*}gH2#dac;`O0@|?~wM9h*gwhm0N;I+mSVmgCO2)q6#hjFwZXu1q)&>MouEWHp z6H)m}C_)FHjCD;R7qQz78ut>+1g|7Z=%0L^6@{4WNeT>@O7C)aF_EN-CCyo0ap(qx zD2=W~PAZe*TI8v1p~yh8Bhe@tx2-Tm4J0IDPS|8Jt$nFr7l>LETTfogObp-C>Dc}iD<4NAQD+GFm76zZ?-lwcSQg`^@z==6rn!TO~+A4 z3dNeN=bNFyP6p+g#FW-IGz74S0YXwoY*5h&FDxl3V1N&NA{U>&5R5laT1DN&<`N+- z;VagFknG4swnF$xQI4__G77;v()^`hdYRf$a_|=5O^a=7I8@1kQ2^#l>2Lhn4Ft%8 zmF@&#LT-W;PJ*|EOJt7^l*tsHPy~vDOfE@>dD?77;-yT~Z63RWO=W}wukLJSHr^Ty zKH|_cE0Br>2ssC4j)N-!73geG@f&SSLAi3RjS2n`oaL&j!; zk))P*<48SV%)VmMFLXu+>TQjV}wb3&%=kFKu3ch8O}i~GTv7%=B1^DWfEi4*1-zV9`hv(PD}~TdtAYp zpNXb*5Q(0rS`#WIWNsJ9`vcs(LK6^mie^d6nuN{>S1}bDfA+l&Kq2BtZZkf#iSBdW$5+#*~@W zV`>}e%|?Vmy=2wOc5^$37Q|#A=dflj-D;J#YLkDcLpf{* z5AU!fV+dHv#Cl^Mw#;S{NYg-$&QOR)z@{8<(OBYMAV2waXBv5<6@hjXwSO(@c-}D& zthnN|G$jEd)ZmBb>XHs9z=S65V_p&_);>Yi=no$G9LrK+7#R>j+y$w`y{Qtj0m?%-al9ndizMwchx9@)6yW6OdMIKQ2QE~J@gOC2F7gN@M0Ob? zN#z%fVjMhR(-BbQjaboIgL(>Jk_S~wMCO6Z@gCA2$Mj_~Z5rGyFcvuV$w@B6;m~3- zQIlF^f@K*)#RhI<4ija`I=o^*Y}}QJI8|w7;vv)FxIvPhe255em<|L|fJ z4giN-I(QGVGLs1xW#Tu3SXRO!=QLbQj!Otgfud&CqAcZ)Te}g7+@fV1rfg14txF4K zAml5aEg%;)ID^Ux#Un!*suq+A#u9ko8NdvnXWxqyi)``<<2d69ubGOA@NyMzAi@|l z37OpxC#oL_%^nAF6vRT8mi!efLpwr1ZO&J%T?j223bIrRdIhq+%>!xuYt#;cU?lGW zB|d+--$4-9Jd1gwHT*cAZA2t77%i$-40F=&(qg5K6^1veDh11&G`+YKVMXc?Pr3Y* zBC3oIR>TrkIW+Pk*gb4U#reloU<8Fw@WXf>%L}!Ju`p%?s%u4%%xBb~FUrsk9D$RO zU@k!i-CRRql(NwF&QJvLyrV2``2`v>C zkfSLKg=;&w4cKnADaj2hY4dxTzw87e|KQ7HAIRA?(Dg9v4aIg1+D3~?R~`>Qh!dY- z**0k86FNO%B6x5|Iu_%IYNe?(JTX~MP5_GWpbroQh#PQXrh!aV=2>8ngQK3sIhwid zX&w`gaL{zBiM)bNK{A5HUPLR=BK?j(Di$TVJ_^6l#ZxiCoWW)%i)) z2r&RsNMvb|SjjDxCXeH&ZUt7e-xHMK0*v7bCYFJaj0*9ps+dSJ`0^MS3`LL(m8u}W z%aCyDx0u`@1b=rahPi5GB(h_A`BqoH^kY`s2Gr>F73_&pxBp>9#%9#5koBdg2l9MhY>MQ4_Pr=1=qqu zKS}^CMjk)~e=>)MU}Yk9CV3uFY6Y|xtcw;>!k_&Bw;!n}j7lyk&Cd)GDwr`sR!G6a zmvC0LMf^uzD~m;j0^+%CX#jqYt6u(eAe~_OW&!;zS?%FX$cpj`!qn+4 zovNGE=GC0OWzKy2D9X09k}w1a;SkL@2l4uGj~mR0Nj6f|!1%L2C;e(F>7ddRMByKE z&;%nJVbh*=moMIIODP`Fk^H7$AHgVuV_h1I=^S9FWyM8)$*I~ZUcs!(AV6PpKu^5d zQ!5^I#uP}~!^eumy$3i%YLJpwW~!l}+?j$#ED+gcs<${DjK@=cs*p?4Kq;8vBW<8k zivHX-uX{+#T!y*KHNs<<`JjXz3(y|)N{0(4HO>;-|4Ot#DE>QGnEg`$sjF?laB4uPTZ$rq~ zD6y2K$qQMDIM}jczzVj#;#xqXM<3b~qBu0I4DQL1E4)w>?$yByQ9}WIdU7~`&1+6y z>542egq+A}!C+*OQ8=g&k+kBZ Tg`w@n6u`SvzCaZwR9$06IdpT! literal 0 HcmV?d00001 diff --git a/examples/freeman.txt b/examples/freeman.txt new file mode 100644 index 0000000..2d1a80e --- /dev/null +++ b/examples/freeman.txt @@ -0,0 +1 @@ +If you go into different cultures, they have different concepts of creation. They have their own creation story and what an afterlife is. Where you go, what you do, who you're going to be with. People say, well... \ No newline at end of file diff --git a/examples/prompt.txt b/examples/prompt.txt new file mode 100644 index 0000000..f18fa4e --- /dev/null +++ b/examples/prompt.txt @@ -0,0 +1 @@ +omnivoice.cpp is a minimal C++17 and GGML port of OmniVoice for zero shot multilingual text to speech. diff --git a/examples/server.sh b/examples/server.sh new file mode 100755 index 0000000..d3daf34 --- /dev/null +++ b/examples/server.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Start the omnivoice OpenAI-compatible TTS server. + +./build/tts-server \ + --model models/omnivoice-base-Q8_0.gguf \ + --codec models/omnivoice-tokenizer-F32.gguf \ + --host 127.0.0.1 --port 8080 --lang None diff --git a/examples/tts.cmd b/examples/tts.cmd new file mode 100644 index 0000000..d79c508 --- /dev/null +++ b/examples/tts.cmd @@ -0,0 +1,12 @@ +@echo off + +set PATH=%~dp0..\build\Release;%PATH% + +omnivoice-tts.exe ^ + --model ..\models\omnivoice-base-Q8_0.gguf ^ + --codec ..\models\omnivoice-tokenizer-Q8_0.gguf ^ + --instruct "male, young adult, moderate pitch" ^ + --lang English ^ + -o tts.wav < prompt.txt + +pause diff --git a/examples/tts.sh b/examples/tts.sh new file mode 100755 index 0000000..935228f --- /dev/null +++ b/examples/tts.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eu + +../build/omnivoice-tts \ + --model ../models/omnivoice-base-Q8_0.gguf \ + --codec ../models/omnivoice-tokenizer-Q8_0.gguf \ + --instruct "male, young adult, moderate pitch" \ + --lang English \ + -o tts.wav < prompt.txt diff --git a/format.sh b/format.sh new file mode 100755 index 0000000..c3b160a --- /dev/null +++ b/format.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +find . -name "*.cpp" -o -name "*.h" | grep -v -e build/ -e ggml/ -e vendor/ | xargs clang-format -i diff --git a/ggml/.editorconfig b/ggml/.editorconfig new file mode 100644 index 0000000..6f987c0 --- /dev/null +++ b/ggml/.editorconfig @@ -0,0 +1,22 @@ +# https://EditorConfig.org + +# Top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file, utf-8 charset +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 +indent_style = space +indent_size = 4 + +[*.md] +indent_size = 2 + +[Makefile] +indent_style = tab + +[prompts/*.txt] +insert_final_newline = unset diff --git a/ggml/.github/pull_request_template.md b/ggml/.github/pull_request_template.md new file mode 100644 index 0000000..aa8bd95 --- /dev/null +++ b/ggml/.github/pull_request_template.md @@ -0,0 +1 @@ +*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.* diff --git a/ggml/.github/workflows/ci.yml b/ggml/.github/workflows/ci.yml new file mode 100644 index 0000000..a500687 --- /dev/null +++ b/ggml/.github/workflows/ci.yml @@ -0,0 +1,272 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + libraries: [shared, static] + + runs-on: ${{ matrix.os }} + + steps: + - name: Clone + uses: actions/checkout@v6 + + - name: Dependencies for Ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install llvm + + - name: Add msbuild to PATH + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v2 + + - name: Create Build Environment + run: mkdir build + + - name: Configure CMake + working-directory: ./build + run: cmake .. + ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }} + ${{ matrix.libraries == 'static' && '-DBUILD_SHARED_LIBS=OFF' || '-DBUILD_SHARED_LIBS=ON' }} + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed + -DGGML_METAL=OFF + + - name: Build + working-directory: ./build + run: cmake --build . ${{ contains(matrix.os, 'windows') && '--config Release' || '' }} + + - name: Test + working-directory: ./build + run: ctest --verbose --timeout 900 ${{ contains(matrix.os, 'windows') && '--build-config Release' || '' }} + + - name: Install + working-directory: ./build + run: cmake --build . --target install ${{ contains(matrix.os, 'windows') && '--config Release' || '' }} + + - name: Test CMake config + run: | + mkdir test-cmake + cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }} + cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }} + +# TODO: simplify the following workflows using a matrix + ggml-ci-x64-cpu-low-perf: + runs-on: ubuntu-22.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.16 + with: + key: ggml-ci-x64-cpu-low-perf + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libcurl4-openssl-dev + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt + + ggml-ci-arm64-cpu-low-perf: + runs-on: ubuntu-22.04-arm + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.16 + with: + key: ggml-ci-arm64-cpu-low-perf + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libcurl4-openssl-dev + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt + + ggml-ci-x64-cpu-high-perf: + runs-on: ubuntu-22.04 + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.16 + with: + key: ggml-ci-x64-cpu-high-perf + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libcurl4-openssl-dev + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt + + ggml-ci-arm64-cpu-high-perf: + runs-on: ubuntu-22.04-arm + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.16 + with: + key: ggml-ci-arm64-cpu-high-perf + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libcurl4-openssl-dev + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt + + ggml-ci-arm64-cpu-high-perf-sve: + runs-on: ubuntu-22.04-arm + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.16 + with: + key: ggml-ci-arm64-cpu-high-perf-sve + evict-old-files: 1d + + - name: Dependencies + id: depends + run: | + sudo apt-get update + sudo apt-get install build-essential libcurl4-openssl-dev + + - name: Test + id: ggml-ci + run: | + LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt + + ggml-ci-x64-nvidia-cuda: + runs-on: [self-hosted, Linux, X64, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + nvidia-smi + GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml + + ggml-ci-x64-nvidia-vulkan-cm: + runs-on: [self-hosted, Linux, X64, NVIDIA] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml + + ggml-ci-x64-nvidia-vulkan-cm2: + runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml + + # TODO: provision AMX-compatible machine + #ggml-ci-x64-cpu-amx: + # runs-on: [self-hosted, Linux, X64, CPU, AMX] + + # steps: + # - name: Clone + # id: checkout + # uses: actions/checkout@v6 + + # - name: Test + # id: ggml-ci + # run: | + # bash ./ci/run.sh ~/results/ggml /mnt/ggml + + ggml-ci-mac-metal: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml + + ggml-ci-mac-vulkan: + runs-on: [self-hosted, macOS, ARM64] + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Test + id: ggml-ci + run: | + vulkaninfo --summary + GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml diff --git a/ggml/.github/workflows/release.yml b/ggml/.github/workflows/release.yml new file mode 100644 index 0000000..c443a7b --- /dev/null +++ b/ggml/.github/workflows/release.yml @@ -0,0 +1,27 @@ +name: Release + +on: + push: + tags: + - 'v*' + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Create Release + id: create_release + uses: ggml-org/action-create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref_name }} + release_name: ${{ github.ref }} + draft: false + prerelease: false diff --git a/ggml/.gitignore b/ggml/.gitignore new file mode 100644 index 0000000..47921f7 --- /dev/null +++ b/ggml/.gitignore @@ -0,0 +1,39 @@ +build/ +build-*/ +out/ +tmp/ +models/ +models-mnt + +compile_commands.json +CMakeSettings.json +.vs/ +.vscode/ +.idea/ +.clangd + +.venv/ +ggml_env/ +.exrc +.cache +.DS_Store +.stablelm +.gpt-2 + +src/arm_neon.h +tests/arm_neon.h + +zig-out/ +zig-cache/ + +*.o +*.d +*.dot + +*.sw? + +__pycache__/ + +# Model files +ggml-model-f16.bin +*.bat diff --git a/ggml/.gitmodules b/ggml/.gitmodules new file mode 100644 index 0000000..e69de29 diff --git a/ggml/AUTHORS b/ggml/AUTHORS new file mode 100644 index 0000000..d130ab3 --- /dev/null +++ b/ggml/AUTHORS @@ -0,0 +1,335 @@ +# date: Tue Feb 4 13:03:51 EET 2025 +# this file is auto-generated by scripts/gen-authors.sh + +0cc4m +65a <10104049+65a@users.noreply.github.com> +AT +Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> +Adam Tazi <52357206+ad1tazi@users.noreply.github.com> +Adrien Gallouët +Adrien Gallouët +Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com> +AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com> +AidanBeltonS +Akarshan Biswas +Akarshan Biswas +Albert Jin +Alberto Cabrera Pérez +Alberto Cabrera Pérez +Alex Azarov +Alex O'Connell <35843486+acon96@users.noreply.github.com> +Alex von Gluck IV +AmbientL <107641468+AmbientL@users.noreply.github.com> +AmirAli Mirian <37371367+amiralimi@users.noreply.github.com> +Ananta Bastola +Andreas (Andi) Kunar +Andreas Kieslinger <47689530+aendk@users.noreply.github.com> +Andrei +Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com> +Andrii Ryzhkov +Arjun +Ashraful Islam +Astariul <43774355+astariul@users.noreply.github.com> +AsukaMinato +Avi Lumelsky +Bart Pelle <3662930+Velocity-@users.noreply.github.com> +Ben Ashbaugh +Bernhard M. Wiedemann +Borislav Stanimirov +Brad Ito +Brad Murray <59848399+bradmurray-dt@users.noreply.github.com> +Brian +Bryan Lozano +Carolinabanana <140120812+Carolinabanana@users.noreply.github.com> +CarterLi999 <664681047@qq.com> +Cebtenzzre +Changyeon Kim +Charles Xu <63788048+chaxu01@users.noreply.github.com> +Charles Xu +Chen Xi +Chen Xi +Chenguang Li <87689256+noemotiovon@users.noreply.github.com> +Chris Elrod +Christian Kastner +Clint Herron +Conrad Kramer +Cordeiro <1471463+ocordeiro@users.noreply.github.com> +Cristiano Calcagno +DAN™ +Dan Forbes +Dan Johansson <164997844+eddnjjn@users.noreply.github.com> +Dan Johansson +Daniel Bevenius +Daniel Ziegenberg +Daniele <57776841+daniandtheweb@users.noreply.github.com> +Daulet Zhanguzin +Dave +Dave Airlie +Dave Airlie +David Miller +DavidKorczynski +Davidson Francis +Dibakar Gope +Didzis Gosko +Diego Devesa +Diogo +Djip007 <3705339+Djip007@users.noreply.github.com> +Djip007 +Dou Xinpeng <15529241576@163.com> +Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com> +Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com> +Ebey Abraham +Eldar Yusupov +Emmanuel Durand +Engininja2 <139037756+Engininja2@users.noreply.github.com> +Eric Zhang <34133756+EZForever@users.noreply.github.com> +Erik Scholz +Ettore Di Giacinto +Eve <139727413+netrunnereve@users.noreply.github.com> +F1L1P <78918286+F1L1Pv2@users.noreply.github.com> +Faisal Zaghloul +FantasyGmm <16450052+FantasyGmm@users.noreply.github.com> +Felix +Finn Voorhees +FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com> +Frankie Robertson +GainLee +George Hindle +Georgi Gerganov +Gilad S <7817232+giladgd@users.noreply.github.com> +Gilad S +Gilad S. <7817232+giladgd@users.noreply.github.com> +Guillaume Wenzek +Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com> +Haus1 +Herman Semenov +HimariO +Hirochika Matsumoto +Hong Bo PENG +Hugo Rosenkranz-Costa +Hyunsung Lee +IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com> +Ian Bull +Ihar Hrachyshka +Ikko Eltociear Ashimine +Ivan +Ivan Filipov <159561759+vanaka11@users.noreply.github.com> +Ivan Stepanov +Ivan Zdane +Jack Mousseau +Jack Vial +JacobLinCool +Jakob Frick +Jan Ploski +Jared Van Bortel +Jeff Bolz +Jeffrey Quesnelle +Jeroen Mostert +Jiahao Li +JidongZhang-THU <1119708529@qq.com> +Jiří Podivín <66251151+jpodivin@users.noreply.github.com> +Jo Liss +Joe Todd +Johannes Gäßler +John Balis +Josh Bleecher Snyder +Judd +Jun Hee Yoo +Junil Kim +Justina Cho +Justine Tunney +Justine Tunney +Karol Kontny <82021046+kkontny@users.noreply.github.com> +Kawrakow <48489457+ikawrakow@users.noreply.github.com> +Kevin Gibbons +Konstantin Zhuravlyov +Kylin <56434533+KyL0N@users.noreply.github.com> +LoganDark +LoganDark +LostRuins <39025047+LostRuins@users.noreply.github.com> +Lukas Möller +M Refi D.A <24388107+refinism@users.noreply.github.com> +M. Yusuf Sarıgöz +Ma Mingfei +Mahesh Madhav <67384846+heshpdx@users.noreply.github.com> +MaiHD +Mark Zhuang +Markus Tavenrath +Masaya, Kato <62578291+msy-kato@users.noreply.github.com> +Mathieu Baudier +Mathijs de Bruin +Matt Stephenson +Max Krasnyansky +Max Krasnyansky +Mayank Kumar Pal +Meng, Hengyu +Mengqing Cao +Metal Whale <45712559+metalwhale@users.noreply.github.com> +Michael Klimenko +Michael Podvitskiy +Michael Verrilli +Molly Sophia +Natsu +Neo Zhang <14088817+arthw@users.noreply.github.com> +Neo Zhang Jianyu +Neuman Vong +Nevin +Nicholai Tukanov +Nico Bosshard +Nicolò Scipione +Nikita Sarychev <42014488+sARY77@users.noreply.github.com> +Nouamane Tazi +Olivier Chafik +Olivier Chafik +Ondřej Čertík +Ouadie EL FAROUKI +PAB +Paul Tsochantaris +Peter +Philpax +Pierre Alexandre SCHEMBRI +Plamen Minev +Playdev +Prashant Vithule <119530321+Vithulep@users.noreply.github.com> +Przemysław Pawełczyk +R0CKSTAR +R0CKSTAR +Radoslav Gerganov +Radosław Gryta +Ravindra Marella +Ray Cromwell +Reinforce-II +Rémy Oudompheng +Reza Rezvan +Rick G <26732651+TheFlipbook@users.noreply.github.com> +RiverZhou +Robert Ormandi <52251610+ormandi@users.noreply.github.com> +Romain Biessy +Ronsor +Rotem Dan +Ryan Hitchman +SRHMorris <69468379+SRHMorris@users.noreply.github.com> +SXX +Salvatore Mesoraca +Sam Spilsbury +Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com> +Santtu Keskinen +Sergio López +Sergio López +Shanshan Shen <467638484@qq.com> +Shijie <821898965@qq.com> +Shupei Fan +Siddharth Ramakrishnan +Sigbjørn Skjæret +Skyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com> +Slava Primenko +Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com> +Steward Garcia <57494570+FSSRepo@users.noreply.github.com> +Supreet Sethi +Takuya Takeuchi +Tamotsu Takahashi +Tanmay +Tanmay Sachan +Timothy Cronin <40186632+4imothy@users.noreply.github.com> +Tom Bailey +Tom Jobbins <784313+TheBloke@users.noreply.github.com> +Tony Wasserka <4840017+neobrain@users.noreply.github.com> +Tristan Druyen +Tyé singwa <92231658+tye-singwa@users.noreply.github.com> +UEXTM.com <84163508+uextm@users.noreply.github.com> +WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com> +William Tambellini +William Tambellini +XiaotaoChen +Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com> +Xuan Son Nguyen +Yavor Ivanov +YavorGIvanov +Yilong Guo +Yilong Guo +Yuri Khrustalev +Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com> +Zhiyuan Li +Zhiyuan Li +a3sh <38979186+A3shTnT@users.noreply.github.com> +ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com> +agray3 +amd-dwang +amritahs-ibm +apcameron <37645737+apcameron@users.noreply.github.com> +appvoid <78444142+appvoid@users.noreply.github.com> +ariez-xyz <41232910+ariez-xyz@users.noreply.github.com> +automaticcat +bandoti <141645996+bandoti@users.noreply.github.com> +bmwl +bobqianic <129547291+bobqianic@users.noreply.github.com> +bssrdf +chengchi +compilade <113953597+compilade@users.noreply.github.com> +compilade +ddpasa <112642920+ddpasa@users.noreply.github.com> +denersc +dscripka +fitzsim +fj-y-saito <85871716+fj-y-saito@users.noreply.github.com> +fraxy-v <65565042+fraxy-v@users.noreply.github.com> +gn64 +goerch +goldwaving <77494627+goldwaving@users.noreply.github.com> +haopeng <657407891@qq.com> +hidenorly +hipudding +hydai +issixx <46835150+issixx@users.noreply.github.com> +jaeminSon +jdomke <28772296+jdomke@users.noreply.github.com> +jiez <373447296@qq.com> +johnson442 <56517414+johnson442@users.noreply.github.com> +junchao-loongson <68935141+junchao-loongson@users.noreply.github.com> +k.h.lai +katsu560 <118887472+katsu560@users.noreply.github.com> +klosax <131523366+klosax@users.noreply.github.com> +kunnis +l3utterfly +le.chang +leejet <31925346+leejet@users.noreply.github.com> +leejet +leo-pony +lhez +liuwei-git <14815172+liuwei-git@users.noreply.github.com> +luoyu-intel +magicse +mahorozte <41834471+mahorozte@users.noreply.github.com> +mashizora <30516315+mashizora@users.noreply.github.com> +matt23654 +matteo +ochafik +otaGran +pengxin99 +pikalover6 <49179590+pikalover6@users.noreply.github.com> +postmasters +sjinzh +skirodev <57715494+skirodev@users.noreply.github.com> +slaren +snadampal <87143774+snadampal@users.noreply.github.com> +someone13574 <81528246+someone13574@users.noreply.github.com> +stduhpf +taher <8665427+nullhook@users.noreply.github.com> +texmex76 <40733439+texmex76@users.noreply.github.com> +the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com> +thewh1teagle <61390950+thewh1teagle@users.noreply.github.com> +ucag.li +ulatekh +uvos +uvos +wangshuai09 <391746016@qq.com> +woachk <24752637+woachk@users.noreply.github.com> +xctan +yangyaofei +yuri@FreeBSD +zhentaoyu +zhouwg <6889919+zhouwg@users.noreply.github.com> +zhouwg +谢乃闻 +布客飞龙 <562826179@qq.com> +旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com> diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt new file mode 100644 index 0000000..0406978 --- /dev/null +++ b/ggml/CMakeLists.txt @@ -0,0 +1,505 @@ +cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories. + +project("ggml" C CXX ASM) + +### GGML Version +set(GGML_VERSION_MAJOR 0) +set(GGML_VERSION_MINOR 15) +set(GGML_VERSION_PATCH 2) +set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/") + +find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH) +if(GIT_EXE) + # Get current git commit hash + execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GGML_BUILD_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + + # Check if the working directory is dirty (i.e., has uncommitted changes) + execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- . + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GGML_GIT_DIRTY + ERROR_QUIET + ) +endif() + +set(GGML_VERSION "${GGML_VERSION_BASE}") + +if(NOT GGML_BUILD_COMMIT) + set(GGML_BUILD_COMMIT "unknown") +endif() + +# Build the commit string with optional dirty flag +if(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1) + set(GGML_BUILD_COMMIT "${GGML_BUILD_COMMIT}-dirty") +endif() + +include(CheckIncludeFileCXX) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +endif() + +if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(GGML_STANDALONE ON) + + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + + # configure project version + # TODO +else() + set(GGML_STANDALONE OFF) + + if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + endif() +endif() + +if (EMSCRIPTEN) + set(BUILD_SHARED_LIBS_DEFAULT OFF) + + option(GGML_WASM_SINGLE_FILE "ggml: embed WASM inside the generated ggml.js" ON) +else() + if (MINGW) + set(BUILD_SHARED_LIBS_DEFAULT OFF) + else() + set(BUILD_SHARED_LIBS_DEFAULT ON) + endif() +endif() + +# remove the lib prefix on win32 mingw +if (WIN32) + set(CMAKE_STATIC_LIBRARY_PREFIX "") + set(CMAKE_SHARED_LIBRARY_PREFIX "") + set(CMAKE_SHARED_MODULE_PREFIX "") +endif() + +option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT}) +option(GGML_BACKEND_DL "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF) +set(GGML_BACKEND_DIR "" CACHE PATH "ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL") + +# +# option list +# + +# TODO: mark all options as advanced when not GGML_STANDALONE + +if (APPLE) + set(GGML_METAL_DEFAULT ON) + set(GGML_BLAS_DEFAULT ON) + set(GGML_BLAS_VENDOR_DEFAULT "Apple") +else() + set(GGML_METAL_DEFAULT OFF) + set(GGML_BLAS_DEFAULT OFF) + set(GGML_BLAS_VENDOR_DEFAULT "Generic") +endif() + +if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH}) + message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF") + set(GGML_NATIVE_DEFAULT OFF) +else() + set(GGML_NATIVE_DEFAULT ON) +endif() + +# defaults +if (NOT GGML_LLAMAFILE_DEFAULT) + set(GGML_LLAMAFILE_DEFAULT OFF) +endif() + +if (NOT GGML_CUDA_GRAPHS_DEFAULT) + set(GGML_CUDA_GRAPHS_DEFAULT OFF) +endif() + +# general +option(GGML_STATIC "ggml: static link libraries" OFF) +option(GGML_NATIVE "ggml: optimize the build for the current system" ${GGML_NATIVE_DEFAULT}) +option(GGML_LTO "ggml: enable link time optimization" OFF) +option(GGML_CCACHE "ggml: use ccache if available" ON) + +# debug +option(GGML_ALL_WARNINGS "ggml: enable all compiler warnings" ON) +option(GGML_ALL_WARNINGS_3RD_PARTY "ggml: enable all compiler warnings in 3rd party libs" OFF) +option(GGML_GPROF "ggml: enable gprof" OFF) + +# build +option(GGML_FATAL_WARNINGS "ggml: enable -Werror flag" OFF) + +# sanitizers +option(GGML_SANITIZE_THREAD "ggml: enable thread sanitizer" OFF) +option(GGML_SANITIZE_ADDRESS "ggml: enable address sanitizer" OFF) +option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF) + +# instruction set specific +if (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT) + set(INS_ENB OFF) +else() + set(INS_ENB ON) +endif() + +message(DEBUG "GGML_NATIVE : ${GGML_NATIVE}") +message(DEBUG "GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}") +message(DEBUG "INS_ENB : ${INS_ENB}") + +option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF) +option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON) +option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF) +option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB}) +option(GGML_AVX "ggml: enable AVX" ${INS_ENB}) +option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF) +option(GGML_AVX2 "ggml: enable AVX2" ${INS_ENB}) +option(GGML_BMI2 "ggml: enable BMI2" ${INS_ENB}) +option(GGML_AVX512 "ggml: enable AVX512F" OFF) +option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI" OFF) +option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI" OFF) +option(GGML_AVX512_BF16 "ggml: enable AVX512-BF16" OFF) +if (NOT MSVC) + # in MSVC F16C and FMA is implied with AVX2/AVX512 + option(GGML_FMA "ggml: enable FMA" ${INS_ENB}) + option(GGML_F16C "ggml: enable F16C" ${INS_ENB}) + # MSVC does not seem to support AMX + option(GGML_AMX_TILE "ggml: enable AMX-TILE" OFF) + option(GGML_AMX_INT8 "ggml: enable AMX-INT8" OFF) + option(GGML_AMX_BF16 "ggml: enable AMX-BF16" OFF) +endif() +option(GGML_LASX "ggml: enable lasx" ON) +option(GGML_LSX "ggml: enable lsx" ON) +option(GGML_RVV "ggml: enable rvv" ON) +option(GGML_RV_ZFH "ggml: enable riscv zfh" ON) +option(GGML_RV_ZVFH "ggml: enable riscv zvfh" ON) +option(GGML_RV_ZICBOP "ggml: enable riscv zicbop" ON) +option(GGML_RV_ZIHINTPAUSE "ggml: enable riscv zihintpause" ON) +option(GGML_RV_ZVFBFWMA "ggml: enable riscv zvfbfwma" OFF) +option(GGML_XTHEADVECTOR "ggml: enable xtheadvector" OFF) +option(GGML_VXE "ggml: enable vxe" ${GGML_NATIVE}) + +option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF) +set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM") +set(GGML_CPU_POWERPC_CPUTYPE "" CACHE STRING "ggml: CPU type for PowerPC") + +# ggml core +set(GGML_SCHED_MAX_COPIES "4" CACHE STRING "ggml: max input copies for pipeline parallelism") +option(GGML_CPU "ggml: enable CPU backend" ON) +option(GGML_SCHED_NO_REALLOC "ggml: disallow reallocations in ggml-alloc (for debugging)" OFF) + +# 3rd party libs / backends +option(GGML_ACCELERATE "ggml: enable Accelerate framework" ON) +option(GGML_BLAS "ggml: use BLAS" ${GGML_BLAS_DEFAULT}) +set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING + "ggml: BLAS library vendor") +option(GGML_LLAMAFILE "ggml: use LLAMAFILE" ${GGML_LLAMAFILE_DEFAULT}) + +option(GGML_CUDA "ggml: use CUDA" OFF) +option(GGML_MUSA "ggml: use MUSA" OFF) +option(GGML_CUDA_FORCE_MMQ "ggml: use mmq kernels instead of cuBLAS" OFF) +option(GGML_CUDA_FORCE_CUBLAS "ggml: always use cuBLAS instead of mmq kernels" OFF) +set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING + "ggml: max. batch size for using peer access") +option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF) +option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF) +option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON) +option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF) +option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT}) +option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON) +set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING + "ggml: cuda link binary compression mode; requires cuda 12.8+") +set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size") + +option(GGML_HIP "ggml: use HIP" OFF) +option(GGML_HIP_GRAPHS "ggml: use HIP graph" ON) +option(GGML_HIP_RCCL "ggml: use ROCm Collective Comm. Library" OFF) +option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON) +option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF) +option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON) +option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF) +option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF) +option(GGML_MUSA_MUDNN_COPY "ggml: enable muDNN for accelerated copy" OFF) +option(GGML_VULKAN "ggml: use Vulkan" OFF) +option(GGML_VULKAN_CHECK_RESULTS "ggml: run Vulkan op checks" OFF) +option(GGML_VULKAN_DEBUG "ggml: enable Vulkan debug output" OFF) +option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug output" OFF) +option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF) +option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF) +option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF) +option(GGML_WEBGPU "ggml: use WebGPU" OFF) +option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF) +option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF) +option(GGML_WEBGPU_GPU_PROFILE "ggml: enable WebGPU profiling (GPU)" OFF) +option(GGML_WEBGPU_JSPI "ggml: use JSPI for WebGPU" ON) +option(GGML_ZDNN "ggml: use zDNN" OFF) +option(GGML_VIRTGPU "ggml: use the VirtGPU/Virglrenderer API Remoting frontend" OFF) +option(GGML_VIRTGPU_BACKEND "ggml: build the VirtGPU/Virglrenderer API Remoting backend" OFF) +option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT}) +option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF) +option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF) +option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" ${GGML_METAL}) +set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING + "ggml: metal minimum macOS version") +set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)") +option(GGML_OPENMP "ggml: use OpenMP" ON) +option(GGML_RPC "ggml: use RPC" OFF) +option(GGML_SYCL "ggml: use SYCL" OFF) +option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) +option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) +option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON) +option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON) +option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON) +set (GGML_SYCL_TARGET "INTEL" CACHE STRING + "ggml: sycl target device") +set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING + "ggml: sycl device architecture") + +option(GGML_OPENVINO "ggml: use OPENVINO" OFF) + +option(GGML_OPENCL "ggml: use OpenCL" OFF) +option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF) +option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON) +option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON) +set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING + "ggml: OpenCL API version to target") + +option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF) +set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)") + +# toolchain for vulkan-shaders-gen +set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen") + +option(GGML_ZENDNN "ggml: use ZenDNN" OFF) +option(ZENDNN_ROOT "ggml: path to ZenDNN installation" "") + +# extra artifacts +option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE}) +option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE}) + +# +# dependencies +# + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED true) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED true) + +set(THREADS_PREFER_PTHREAD_FLAG ON) + +find_package(Threads REQUIRED) + +include(GNUInstallDirs) + +# +# build the library +# + +add_subdirectory(src) + +# +# tests and examples +# + +if (GGML_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif () + +if (GGML_BUILD_EXAMPLES) + add_subdirectory(examples) +endif () + +# +# install +# + +include(CMakePackageConfigHelpers) + +# all public headers +set(GGML_PUBLIC_HEADERS + include/ggml.h + include/ggml-cpu.h + include/ggml-alloc.h + include/ggml-backend.h + include/ggml-blas.h + include/ggml-cann.h + include/ggml-cpp.h + include/ggml-cuda.h + include/ggml-opt.h + include/ggml-metal.h + include/ggml-rpc.h + include/ggml-virtgpu.h + include/ggml-sycl.h + include/ggml-vulkan.h + include/ggml-webgpu.h + include/ggml-zendnn.h + include/ggml-openvino.h + include/gguf.h) + +set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}") +#if (GGML_METAL) +# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal") +#endif() +install(TARGETS ggml LIBRARY PUBLIC_HEADER) +install(TARGETS ggml-base LIBRARY) + +if (GGML_STANDALONE) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in + ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc + @ONLY) + + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +endif() + +# +# Create CMake package +# + + + +# Capture variables prefixed with GGML_. + +set(variable_set_statements +" +####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() ####### +####### Any changes to this file will be overwritten by the next CMake run ####### + +") + +set(GGML_SHARED_LIB ${BUILD_SHARED_LIBS}) + +get_cmake_property(all_variables VARIABLES) +foreach(variable_name IN LISTS all_variables) + if(variable_name MATCHES "^GGML_") + string(REPLACE ";" "\\;" + variable_value "${${variable_name}}") + + set(variable_set_statements + "${variable_set_statements}set(${variable_name} \"${variable_value}\")\n") + endif() +endforeach() + +set(GGML_VARIABLES_EXPANDED ${variable_set_statements}) + +# Create the CMake package and set install location. + +set(GGML_INSTALL_VERSION ${GGML_VERSION}) +set(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files") +set(GGML_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files") +set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files") + +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml + PATH_VARS GGML_INCLUDE_INSTALL_DIR + GGML_LIB_INSTALL_DIR + GGML_BIN_INSTALL_DIR) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake + VERSION ${GGML_INSTALL_VERSION} + COMPATIBILITY SameMajorVersion) + +target_compile_definitions(ggml-base PRIVATE + GGML_VERSION="${GGML_INSTALL_VERSION}" + GGML_COMMIT="${GGML_BUILD_COMMIT}" +) +message(STATUS "ggml version: ${GGML_INSTALL_VERSION}") +message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}") + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml) + +if (MSVC) + set(MSVC_WARNING_FLAGS + /wd4005 # Macro redefinition + /wd4244 # Conversion from one type to another type, possible loss of data + /wd4267 # Conversion from 'size_t' to a smaller type, possible loss of data + /wd4305 # Conversion from 'type1' to 'type2', possible loss of data + /wd4566 # Conversion from 'char' to 'wchar_t', possible loss of data + /wd4996 # Disable POSIX deprecation warnings + /wd4702 # Unreachable code warnings + ) + set(MSVC_COMPILE_OPTIONS + "$<$:/utf-8>" + "$<$:/utf-8>" + ) + function(configure_msvc_target target_name) + if(TARGET ${target_name}) + target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS}) + target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS}) + endif() + endfunction() + + configure_msvc_target(ggml-base) + configure_msvc_target(ggml) + configure_msvc_target(ggml-cpu) + configure_msvc_target(ggml-cpu-x64) + configure_msvc_target(ggml-cpu-sse42) + configure_msvc_target(ggml-cpu-sandybridge) + # __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512 + # skipping ggml-cpu-ivybridge + # skipping ggml-cpu-piledriver + configure_msvc_target(ggml-cpu-haswell) + configure_msvc_target(ggml-cpu-skylakex) + configure_msvc_target(ggml-cpu-cannonlake) + configure_msvc_target(ggml-cpu-cascadelake) + configure_msvc_target(ggml-cpu-icelake) + # MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?! + # https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170 + # https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170 + # skipping ggml-cpu-cooperlake + # skipping ggml-cpu-zen4 + configure_msvc_target(ggml-cpu-alderlake) + # MSVC doesn't support AMX + # skipping ggml-cpu-sapphirerapids + + if (GGML_BUILD_EXAMPLES) + configure_msvc_target(common-ggml) + configure_msvc_target(common) + + configure_msvc_target(mnist-common) + configure_msvc_target(mnist-eval) + configure_msvc_target(mnist-train) + + configure_msvc_target(gpt-2-ctx) + configure_msvc_target(gpt-2-alloc) + configure_msvc_target(gpt-2-backend) + configure_msvc_target(gpt-2-sched) + configure_msvc_target(gpt-2-quantize) + configure_msvc_target(gpt-2-batched) + + configure_msvc_target(gpt-j) + configure_msvc_target(gpt-j-quantize) + + configure_msvc_target(magika) + configure_msvc_target(yolov3-tiny) + configure_msvc_target(sam) + + configure_msvc_target(simple-ctx) + configure_msvc_target(simple-backend) + endif() + + if (GGML_BUILD_TESTS) + configure_msvc_target(test-mul-mat) + configure_msvc_target(test-arange) + configure_msvc_target(test-backend-ops) + configure_msvc_target(test-cont) + configure_msvc_target(test-conv-transpose) + configure_msvc_target(test-conv-transpose-1d) + configure_msvc_target(test-conv1d) + configure_msvc_target(test-conv2d) + configure_msvc_target(test-conv2d-dw) + configure_msvc_target(test-customop) + configure_msvc_target(test-dup) + configure_msvc_target(test-opt) + configure_msvc_target(test-pool) + endif () +endif() diff --git a/ggml/CONTRIBUTING.md b/ggml/CONTRIBUTING.md new file mode 100644 index 0000000..8a645a0 --- /dev/null +++ b/ggml/CONTRIBUTING.md @@ -0,0 +1,3 @@ +Please use [llama.cpp's contribution guidelines](https://github.com/ggml-org/llama.cpp/blob/master/CONTRIBUTING.md) for this project. + +*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.* diff --git a/ggml/LICENSE b/ggml/LICENSE new file mode 100644 index 0000000..e7dca55 --- /dev/null +++ b/ggml/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-2026 The ggml authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ggml/README.md b/ggml/README.md new file mode 100644 index 0000000..455c812 --- /dev/null +++ b/ggml/README.md @@ -0,0 +1,50 @@ +# ggml + +[Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205) + +Tensor library for machine learning + +***Note that this project is under active development. \ +Some of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos*** + +## Features + +- Low-level cross-platform implementation +- Integer quantization support +- Broad hardware support +- Automatic differentiation +- ADAM and L-BFGS optimizers +- No third-party dependencies +- Zero memory allocations during runtime + +## Build + +```bash +git clone https://github.com/ggml-org/ggml +cd ggml + +# install python dependencies in a virtual environment +python3.10 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +# build the examples +mkdir build && cd build +cmake .. +cmake --build . --config Release -j 8 +``` + +## GPT inference (example) + +```bash +# run the GPT-2 small 117M model +../examples/gpt-2/download-ggml-model.sh 117M +./bin/gpt-2-backend -m models/gpt-2-117M/ggml-model.bin -p "This is an example" +``` + +For more information, checkout the corresponding programs in the [examples](examples) folder. + +## Resources + +- [Introduction to ggml](https://huggingface.co/blog/introduction-to-ggml) +- [The GGUF file format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md) diff --git a/ggml/ci/run.sh b/ggml/ci/run.sh new file mode 100644 index 0000000..0285bed --- /dev/null +++ b/ggml/ci/run.sh @@ -0,0 +1,395 @@ +#/bin/bash +# +# sample usage: +# +# mkdir tmp +# +# # CPU-only build +# bash ./ci/run.sh ./tmp/results ./tmp/mnt +# +# # with CUDA support +# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt +# +# # With SYCL support +# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt +# + +if [ -z "$2" ]; then + echo "usage: $0 " + exit 1 +fi + +mkdir -p "$1" +mkdir -p "$2" + +OUT=$(realpath "$1") +MNT=$(realpath "$2") + +rm -v $OUT/*.log +rm -v $OUT/*.exit +rm -v $OUT/*.md + +sd=`dirname $0` +cd $sd/../ +SRC=`pwd` + +CMAKE_EXTRA="" +CTEST_EXTRA="" + +if [ ! -z ${GG_BUILD_METAL} ]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON" +fi + +if [ ! -z ${GG_BUILD_CUDA} ]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON" + + if command -v nvidia-smi >/dev/null 2>&1; then + CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.') + if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}" + else + echo "Warning: Using fallback CUDA architectures" + CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89" + fi + else + echo "Error: nvidia-smi not found, cannot build with CUDA" + exit 1 + fi +fi + +if [ ! -z ${GG_BUILD_ROCM} ]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON" + if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then + echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)" + exit 1 + fi + + CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}" +fi + +if [ ! -z ${GG_BUILD_SYCL} ]; then + if [ -z ${ONEAPI_ROOT} ]; then + echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:" + echo "source /opt/intel/oneapi/setvars.sh" + exit 1 + fi + # Use only main GPU + export ONEAPI_DEVICE_SELECTOR="level_zero:0" + # Enable sysman for correct memory reporting + export ZES_ENABLE_SYSMAN=1 + # to circumvent precision issues on CPY operations + export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt" + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON" +fi + +if [ ! -z ${GG_BUILD_VULKAN} ]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1" + + # if on Mac, disable METAL + if [[ "$OSTYPE" == "darwin"* ]]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF" + fi + +fi + +if [ ! -z ${GG_BUILD_WEBGPU} ]; then + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1" +fi + +if [ ! -z ${GG_BUILD_MUSA} ]; then + # Use qy1 by default (MTT S80) + MUSA_ARCH=${MUSA_ARCH:-21} + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}" +fi + +if [ ! -z ${GG_BUILD_NO_SVE} ]; then + # arm 9 and newer enables sve by default, adjust these flags depending on the cpu used + CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm" +fi + +## helpers + +# download a file if it does not exist or if it is outdated +function gg_wget { + local out=$1 + local url=$2 + + local cwd=`pwd` + + mkdir -p $out + cd $out + + # should not re-download if file is the same + wget -nv -N $url + + cd $cwd +} + +function gg_printf { + printf -- "$@" >> $OUT/README.md +} + +function gg_run { + ci=$1 + + set -o pipefail + set -x + + gg_run_$ci | tee $OUT/$ci.log + cur=$? + echo "$cur" > $OUT/$ci.exit + + set +x + set +o pipefail + + gg_sum_$ci + + ret=$((ret | cur)) +} + +## ci + +# ctest_debug + +function gg_run_ctest_debug { + cd ${SRC} + + rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug + + set -e + + (time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log + (time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log + + (time ctest ${CTEST_EXTRA} --output-on-failure -E "test-opt|test-backend-ops" ) 2>&1 | tee -a $OUT/${ci}-ctest.log + + set +e +} + +function gg_sum_ctest_debug { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Runs ctest in debug mode\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" + gg_printf '```\n' + gg_printf '\n' +} + +# ctest_release + +function gg_run_ctest_release { + cd ${SRC} + + rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release + + set -e + + (time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log + (time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log + + if [ -z $GG_BUILD_LOW_PERF ]; then + (time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log + else + (time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log + fi + + set +e +} + +function gg_sum_ctest_release { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Runs ctest in release mode\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)" + gg_printf '```\n' +} + +# gpt_2 + +function gg_run_gpt_2 { + cd ${SRC} + + gg_wget models-mnt/gpt-2 https://huggingface.co/ggerganov/ggml/resolve/main/ggml-model-gpt-2-117M.bin + + cd build-ci-release + + set -e + + model="../models-mnt/gpt-2/ggml-model-gpt-2-117M.bin" + prompts="../examples/prompts/gpt-2.txt" + + (time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -tt ${prompts} ) 2>&1 | tee -a $OUT/${ci}-tg.log + (time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log + (time ./bin/gpt-2-sched --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log + + (time ./bin/gpt-2-batched --model ${model} -s 1234 -n 64 -np 8 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log + + set +e +} + +function gg_sum_gpt_2 { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Runs short GPT-2 text generation\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}-tg.log)" + gg_printf '```\n' +} + +# TODO: update +## mnist +# +#function gg_run_mnist { +# cd ${SRC} +# +# cd build-ci-release +# +# set -e +# +# mkdir -p models/mnist +# python3 ../examples/mnist/convert-h5-to-ggml.py ../examples/mnist/models/mnist/mnist_model.state_dict +# +# model_f32="./models/mnist/ggml-model-f32.bin" +# samples="../examples/mnist/models/mnist/t10k-images.idx3-ubyte" +# +# # first command runs and exports "mnist.ggml", the second command runs the exported model +# +# (time ./bin/mnist ${model_f32} ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log +# (time ./bin/mnist-cpu ./mnist.ggml ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log +# +# set +e +#} +# +#function gg_sum_mnist { +# gg_printf '### %s\n\n' "${ci}" +# +# gg_printf 'MNIST\n' +# gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" +# gg_printf '```\n' +# gg_printf '%s\n' "$(cat $OUT/${ci}-mnist.log)" +# gg_printf '```\n' +#} + +# sam + +function gg_run_sam { + cd ${SRC} + + gg_wget models-mnt/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth + gg_wget models-mnt/sam/ https://raw.githubusercontent.com/YavorGIvanov/sam.cpp/ceafb7467bff7ec98e0c4f952e58a9eb8fd0238b/img.jpg + + cd build-ci-release + + set -e + + path_models="../models-mnt/sam/" + model_f16="${path_models}/ggml-model-f16.bin" + img_0="${path_models}/img.jpg" + + python3 ../examples/sam/convert-pth-to-ggml.py ${path_models}/sam_vit_b_01ec64.pth ${path_models}/ 1 + + # Test default parameters + (time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 ) 2>&1 | tee -a $OUT/${ci}-main.log + grep -q "point prompt" $OUT/${ci}-main.log + grep -q "bbox (371, 436), (144, 168)" $OUT/${ci}-main.log || + grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log + + # Test box prompt and single mask output + (time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 -b 368,144,441,173 -sm) 2>&1 | tee -a $OUT/${ci}-main.log + grep -q "box prompt" $OUT/${ci}-main.log + grep -q "bbox (370, 439), (144, 169)" $OUT/${ci}-main.log || + grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log + + set +e +} + +function gg_sum_sam { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Run SAM\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)" + gg_printf '```\n' +} + +# yolo + +function gg_run_yolo { + cd ${SRC} + + gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/yolov3-tiny.weights + gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/dog.jpg + + cd build-ci-release + cp -r ../examples/yolo/data . + + set -e + + path_models="../models-mnt/yolo/" + + python3 ../examples/yolo/convert-yolov3-tiny.py ${path_models}/yolov3-tiny.weights + + (time ./bin/yolov3-tiny -m yolov3-tiny.gguf -i ${path_models}/dog.jpg ) 2>&1 | tee -a $OUT/${ci}-main.log + + grep -qE "dog: (55|56|57|58|59)%" $OUT/${ci}-main.log + grep -qE "car: (50|51|52|53|54)%" $OUT/${ci}-main.log + grep -qE "truck: (54|55|56|57|58)%" $OUT/${ci}-main.log + grep -qE "bicycle: (57|58|59|60|61)%" $OUT/${ci}-main.log + + set +e +} + +function gg_sum_yolo { + gg_printf '### %s\n\n' "${ci}" + + gg_printf 'Run YOLO\n' + gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)" + gg_printf '```\n' + gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)" + gg_printf '```\n' +} + +## main + +if true ; then + # Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt + rm -rf ${SRC}/models-mnt + mnt_models=${MNT}/models + mkdir -p ${mnt_models} + ln -sfn ${mnt_models} ${SRC}/models-mnt + + # Create a fresh python3 venv and enter it + if ! python3 -m venv "$MNT/venv"; then + echo "Error: Failed to create Python virtual environment at $MNT/venv." + exit 1 + fi + source "$MNT/venv/bin/activate" + + pip install -r ${SRC}/requirements.txt --disable-pip-version-check +fi + + +ret=0 + +test $ret -eq 0 && gg_run ctest_debug +test $ret -eq 0 && gg_run ctest_release + +test $ret -eq 0 && gg_run gpt_2 +#test $ret -eq 0 && gg_run mnist +test $ret -eq 0 && gg_run sam +test $ret -eq 0 && gg_run yolo + +if [ -z $GG_BUILD_LOW_PERF ]; then + # run tests meant for low-perf runners + date +fi + +cat $OUT/README.md + +exit $ret diff --git a/ggml/cmake/FindNCCL.cmake b/ggml/cmake/FindNCCL.cmake new file mode 100644 index 0000000..67511e2 --- /dev/null +++ b/ggml/cmake/FindNCCL.cmake @@ -0,0 +1,36 @@ +# cmake/FindNCCL.cmake + +# NVIDIA does not distribute CMake files with NCCl, therefore use this file to find it instead. + +find_path(NCCL_INCLUDE_DIR + NAMES nccl.h + HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda + PATH_SUFFIXES include +) + +find_library(NCCL_LIBRARY + NAMES nccl + HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda + PATH_SUFFIXES lib lib64 +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(NCCL + DEFAULT_MSG + NCCL_LIBRARY NCCL_INCLUDE_DIR +) + +if(NCCL_FOUND) + set(NCCL_LIBRARIES ${NCCL_LIBRARY}) + set(NCCL_INCLUDE_DIRS ${NCCL_INCLUDE_DIR}) + + if(NOT TARGET NCCL::NCCL) + add_library(NCCL::NCCL UNKNOWN IMPORTED) + set_target_properties(NCCL::NCCL PROPERTIES + IMPORTED_LOCATION "${NCCL_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}" + ) + endif() +endif() + +mark_as_advanced(NCCL_INCLUDE_DIR NCCL_LIBRARY) diff --git a/ggml/cmake/GitVars.cmake b/ggml/cmake/GitVars.cmake new file mode 100644 index 0000000..1a4c24e --- /dev/null +++ b/ggml/cmake/GitVars.cmake @@ -0,0 +1,22 @@ +find_package(Git) + +# the commit's SHA1 +execute_process(COMMAND + "${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8 + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_SHA1 + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + +# the date of the commit +execute_process(COMMAND + "${GIT_EXECUTABLE}" log -1 --format=%ad --date=local + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_DATE + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + +# the subject of the commit +execute_process(COMMAND + "${GIT_EXECUTABLE}" log -1 --format=%s + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_COMMIT_SUBJECT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake new file mode 100644 index 0000000..cb66388 --- /dev/null +++ b/ggml/cmake/common.cmake @@ -0,0 +1,50 @@ +function(ggml_get_flags CCID CCVER) + set(C_FLAGS "") + set(CXX_FLAGS "") + + if (CCID MATCHES "Clang") + set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return) + set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi) + + if ( + (CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR + (CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0) + ) + list(APPEND C_FLAGS -Wdouble-promotion) + endif() + elseif (CCID STREQUAL "GNU") + set(C_FLAGS -Wdouble-promotion) + set(CXX_FLAGS -Wno-array-bounds) + + if (CCVER VERSION_GREATER_EQUAL 8.1.0) + list(APPEND CXX_FLAGS -Wextra-semi) + endif() + endif() + + set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE) + set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE) +endfunction() + +function(ggml_get_system_arch) + if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR + CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR + (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND + CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) + set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE) + elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR + CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR + (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND + CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$")) + set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE) + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power") + set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE) + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64") + set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE) + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") + set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE) + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") + set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE) + else() + set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE) + endif() +endfunction() diff --git a/ggml/cmake/ggml-config.cmake.in b/ggml/cmake/ggml-config.cmake.in new file mode 100644 index 0000000..23a3066 --- /dev/null +++ b/ggml/cmake/ggml-config.cmake.in @@ -0,0 +1,201 @@ +@PACKAGE_INIT@ + +@GGML_VARIABLES_EXPANDED@ + +# Find all dependencies before creating any target. +include(CMakeFindDependencyMacro) +find_dependency(Threads) +if (NOT GGML_SHARED_LIB) + set(GGML_BASE_INTERFACE_LINK_LIBRARIES "") + set(GGML_CPU_INTERFACE_LINK_LIBRARIES "") + set(GGML_CPU_INTERFACE_LINK_OPTIONS "") + + if (APPLE AND GGML_ACCELERATE) + find_library(ACCELERATE_FRAMEWORK Accelerate) + if(NOT ACCELERATE_FRAMEWORK) + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0) + return() + endif() + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK}) + endif() + + if (GGML_OPENMP_ENABLED) + find_dependency(OpenMP) + set(GGML_OPENMP_INTERFACE_LINK_LIBRARIES "") + if (TARGET OpenMP::OpenMP_C) + list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C) + endif() + if (TARGET OpenMP::OpenMP_CXX) + list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_CXX) + endif() + list(APPEND GGML_BASE_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES}) + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES}) + endif() + + if (GGML_CPU_HBM) + find_library(memkind memkind) + if(NOT memkind) + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0) + return() + endif() + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind) + endif() + + if (GGML_BLAS) + find_dependency(BLAS) + list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES}) + list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS ${BLAS_LINKER_FLAGS}) + endif() + + if (GGML_CUDA) + set(GGML_CUDA_INTERFACE_LINK_LIBRARIES "") + find_dependency(CUDAToolkit) + if (GGML_STATIC) + list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $) + if (WIN32) + list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $ $) + else() + list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $ $) + endif() + endif() + if (NOT GGML_CUDA_NO_VMM) + list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $) + endif() + endif() + + if (GGML_METAL) + find_library(FOUNDATION_LIBRARY Foundation) + find_library(METAL_FRAMEWORK Metal) + find_library(METALKIT_FRAMEWORK MetalKit) + if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK) + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0) + return() + endif() + set(GGML_METAL_INTERFACE_LINK_LIBRARIES + ${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK}) + endif() + + if (GGML_OPENCL) + find_dependency(OpenCL) + set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $) + endif() + + if (GGML_VULKAN) + find_dependency(Vulkan) + set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $) + endif() + + if (GGML_HIP) + find_dependency(hip) + find_dependency(hipblas) + find_dependency(rocblas) + set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas) + endif() + + if (GGML_SYCL) + set(GGML_SYCL_INTERFACE_LINK_LIBRARIES "") + find_package(DNNL) + if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL") + list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl) + endif() + if (WIN32) + find_dependency(IntelSYCL) + find_dependency(MKL) + list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL) + endif() + endif() +endif() + +set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@") +set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@") +#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@") + +if(NOT TARGET ggml::ggml) + find_package(Threads REQUIRED) + + find_library(GGML_LIBRARY ggml + REQUIRED + HINTS ${GGML_LIB_DIR} + NO_CMAKE_FIND_ROOT_PATH) + + add_library(ggml::ggml UNKNOWN IMPORTED) + set_target_properties(ggml::ggml + PROPERTIES + IMPORTED_LOCATION "${GGML_LIBRARY}") + + find_library(GGML_BASE_LIBRARY ggml-base + REQUIRED + HINTS ${GGML_LIB_DIR} + NO_CMAKE_FIND_ROOT_PATH) + + add_library(ggml::ggml-base UNKNOWN IMPORTED) + set_target_properties(ggml::ggml-base + PROPERTIES + IMPORTED_LOCATION "${GGML_BASE_LIBRARY}" + INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}") + + set(_ggml_all_targets "") + if (NOT GGML_BACKEND_DL) + foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS}) + string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}") + string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx) + + find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend} + REQUIRED + HINTS ${GGML_LIB_DIR} + NO_CMAKE_FIND_ROOT_PATH) + + message(STATUS "Found ${${_ggml_backend_pfx}_LIBRARY}") + + add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED) + set_target_properties(ggml::${_ggml_backend} + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${${_ggml_backend_pfx}_LIBRARY}" + INTERFACE_COMPILE_FEATURES c_std_90 + POSITION_INDEPENDENT_CODE ON) + + string(REGEX MATCH "^ggml-cpu" is_cpu_variant "${_ggml_backend}") + if(is_cpu_variant) + list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml-base") + set_target_properties(ggml::${_ggml_backend} + PROPERTIES + INTERFACE_LINK_LIBRARIES "${GGML_CPU_INTERFACE_LINK_LIBRARIES}") + + if(GGML_CPU_INTERFACE_LINK_OPTIONS) + set_target_properties(ggml::${_ggml_backend} + PROPERTIES + INTERFACE_LINK_OPTIONS "${GGML_CPU_INTERFACE_LINK_OPTIONS}") + endif() + + else() + list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml-base") + set_target_properties(ggml::${_ggml_backend} + PROPERTIES + INTERFACE_LINK_LIBRARIES "${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}") + + if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS) + set_target_properties(ggml::${_ggml_backend} + PROPERTIES + INTERFACE_LINK_OPTIONS "${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}") + endif() + endif() + + list(APPEND _ggml_all_targets ggml::${_ggml_backend}) + endforeach() + endif() + + list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base "${_ggml_all_targets}") + set_target_properties(ggml::ggml + PROPERTIES + INTERFACE_LINK_LIBRARIES "${GGML_INTERFACE_LINK_LIBRARIES}") + + add_library(ggml::all INTERFACE IMPORTED) + set_target_properties(ggml::all + PROPERTIES + INTERFACE_LINK_LIBRARIES "${_ggml_all_targets}") + +endif() + +check_required_components(ggml) diff --git a/ggml/docs/gguf.md b/ggml/docs/gguf.md new file mode 100644 index 0000000..13e5334 --- /dev/null +++ b/ggml/docs/gguf.md @@ -0,0 +1,828 @@ +# GGUF + +GGUF is a file format for storing models for inference with GGML and executors based on GGML. GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML. + +It is a successor file format to GGML, GGMF and GGJT, and is designed to be unambiguous by containing all the information needed to load a model. It is also designed to be extensible, so that new information can be added to models without breaking compatibility. + +For more information about the motivation behind GGUF, see [Historical State of Affairs](#historical-state-of-affairs). + +## Specification + +GGUF is a format based on the existing GGJT, but makes a few changes to the format to make it more extensible and easier to use. The following features are desired: + +- Single-file deployment: they can be easily distributed and loaded, and do not require any external files for additional information. +- Extensible: new features can be added to GGML-based executors/new information can be added to GGUF models without breaking compatibility with existing models. +- `mmap` compatibility: models can be loaded using `mmap` for fast loading and saving. +- Easy to use: models can be easily loaded and saved using a small amount of code, with no need for external libraries, regardless of the language used. +- Full information: all information needed to load a model is contained in the model file, and no additional information needs to be provided by the user. + +The key difference between GGJT and GGUF is the use of a key-value structure for the hyperparameters (now referred to as metadata), rather than a list of untyped values. This allows for new metadata to be added without breaking compatibility with existing models, and to annotate the model with additional information that may be useful for inference or for identifying the model. + +### GGUF Naming Convention + +GGUF follow a naming convention of `[].gguf` where each component is delimitated by a `-` if present. Ultimately this is intended to make it easier for humans to at a glance get the most important details of a model. It is not intended to be perfectly parsable in the field due to the diversity of existing gguf filenames. + +The components are: +1. **Sidecar**: (Optional) Prefix marking the file as an auxiliary module loaded alongside a base model, rather than a standalone model. When present, sits at the very front of the filename followed by `-`. Lowercase by convention. + - `mmproj` : Multimodal projector (vision/audio encoder and projection layer for use with a base LLM) + - `mtp` : Multi-Token Prediction heads (speculative-decoding draft module, intended to be loaded alongside a base model of matching architecture and version). Note that oftentimes the MTP weights can be distributed inside the base model, in which case there is no separate `mtp-` sidecar file. +1. **BaseName**: A descriptive name for the model base type or architecture. + - This can be derived from gguf metadata `general.basename` substituting spaces for dashes. +1. **SizeLabel**: Parameter weight class (useful for leader boards) represented as `x` + - This can be derived from gguf metadata `general.size_label` if available or calculated if missing. + - Rounded decimal point is supported in count with a single letter scale prefix to assist in floating point exponent shown below + - `Q`: Quadrillion parameters. + - `T`: Trillion parameters. + - `B`: Billion parameters. + - `M`: Million parameters. + - `K`: Thousand parameters. + - Additional `-` can be appended as needed to indicate other attributes of interest +1. **FineTune**: A descriptive name for the model fine tuning goal (e.g. Chat, Instruct, etc...) + - This can be derived from gguf metadata `general.finetune` substituting spaces for dashes. +1. **Version**: (Optional) Denotes the model version number, formatted as `v.` + - If model is missing a version number then assume `v1.0` (First Public Release) + - This can be derived from gguf metadata `general.version` +1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs. +1. **Type**: Indicates the kind of gguf file and the intended purpose for it + - If missing, then file is by default a typical gguf tensor model file + - `LoRA` : GGUF file is a LoRA adapter + - `vocab` : GGUF file with only vocab data and metadata +1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `-of-`. + - *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros. + - Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`). + - *ShardTotal* : Total number of shards in this model. Must be 5 digits padded by zeros. + + +#### Validating Above Naming Convention + +At a minimum all model files should have at least BaseName, SizeLabel, Version, in order to be easily validated as a file that is keeping with the GGUF Naming Convention. An example of this issue is that it is easy for Encoding to be mistaken as a FineTune if Version is omitted. + +To validate you can use this regular expression `^(?:(?mmproj|mtp)-)?(?[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?[A-Za-z0-9\s-]+))?)?-(?:(?v\d+(?:\.\d+)*))(?:-(?(?!LoRA|vocab)[\w_]+))?(?:-(?LoRA|vocab))?(?:-(?\d{5}-of-\d{5}))?\.gguf$` which will check that you got the minimum BaseName, SizeLabel and Version present in the correct order. + +For example: + + * `Mixtral-8x7B-v0.1-KQ2.gguf`: + - Model Name: Mixtral + - Expert Count: 8 + - Parameter Count: 7B + - Version Number: v0.1 + - Weight Encoding Scheme: KQ2 + + * `Hermes-2-Pro-Llama-3-8B-F16.gguf`: + - Model Name: Hermes 2 Pro Llama 3 + - Expert Count: 0 + - Parameter Count: 8B + - Version Number: v1.0 + - Weight Encoding Scheme: F16 + - Shard: N/A + + * `Grok-100B-v1.0-Q4_0-00003-of-00009.gguf` + - Model Name: Grok + - Expert Count: 0 + - Parameter Count: 100B + - Version Number: v1.0 + - Weight Encoding Scheme: Q4_0 + - Shard: 3 out of 9 total shards + + * `mtp-Qwen3-27B-v1.0-Q4_K_M.gguf` + - Sidecar: mtp (Multi-Token Prediction draft module) + - Model Name: Qwen3 + - Expert Count: 0 + - Parameter Count: 27B (of the main model — sidecar tensors are smaller) + - Version Number: v1.0 + - Weight Encoding Scheme: Q4_K_M + + * `mmproj-Qwen2-VL-7B-v1.0-F16.gguf` + - Sidecar: mmproj (multimodal projector) + - Model Name: Qwen2-VL + - Expert Count: 0 + - Parameter Count: 7B (of the main model — sidecar tensors are smaller) + - Version Number: v1.0 + - Weight Encoding Scheme: F16 + + +
Example Node.js Regex Function + +```js +#!/usr/bin/env node +const ggufRegex = /^(?:(?mmproj|mtp)-)?(?[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?[A-Za-z0-9\s-]+))?)?-(?:(?v\d+(?:\.\d+)*))(?:-(?(?!LoRA|vocab)[\w_]+))?(?:-(?LoRA|vocab))?(?:-(?\d{5}-of-\d{5}))?\.gguf$/; + +function parseGGUFFilename(filename) { + const match = ggufRegex.exec(filename); + if (!match) + return null; + const {Sidecar = null, BaseName = null, SizeLabel = null, FineTune = null, Version = "v1.0", Encoding = null, Type = null, Shard = null} = match.groups; + return {Sidecar: Sidecar, BaseName: BaseName, SizeLabel: SizeLabel, FineTune: FineTune, Version: Version, Encoding: Encoding, Type: Type, Shard: Shard}; +} + +const testCases = [ + {filename: 'Mixtral-8x7B-v0.1-KQ2.gguf', expected: { Sidecar: null, BaseName: 'Mixtral', SizeLabel: '8x7B', FineTune: null, Version: 'v0.1', Encoding: 'KQ2', Type: null, Shard: null}}, + {filename: 'Grok-100B-v1.0-Q4_0-00003-of-00009.gguf', expected: { Sidecar: null, BaseName: 'Grok', SizeLabel: '100B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_0', Type: null, Shard: "00003-of-00009"}}, + {filename: 'Hermes-2-Pro-Llama-3-8B-v1.0-F16.gguf', expected: { Sidecar: null, BaseName: 'Hermes-2-Pro-Llama-3', SizeLabel: '8B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}}, + {filename: 'Phi-3-mini-3.8B-ContextLength4k-instruct-v1.0.gguf', expected: { Sidecar: null, BaseName: 'Phi-3-mini', SizeLabel: '3.8B-ContextLength4k', FineTune: 'instruct', Version: 'v1.0', Encoding: null, Type: null, Shard: null}}, + {filename: 'mtp-Qwen3-27B-v1.0-Q4_K_M.gguf', expected: { Sidecar: 'mtp', BaseName: 'Qwen3', SizeLabel: '27B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_K_M', Type: null, Shard: null}}, + {filename: 'mmproj-Qwen2-VL-7B-v1.0-F16.gguf', expected: { Sidecar: 'mmproj', BaseName: 'Qwen2-VL', SizeLabel: '7B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}}, + {filename: 'not-a-known-arrangement.gguf', expected: null}, +]; + +testCases.forEach(({ filename, expected }) => { + const result = parseGGUFFilename(filename); + const passed = JSON.stringify(result) === JSON.stringify(expected); + console.log(`${filename}: ${passed ? "PASS" : "FAIL"}`); + if (!passed) { + console.log(result); + console.log(expected); + } +}); +``` + +
+ + +### File Structure + +![image](https://github.com/ggerganov/ggml/assets/1991296/c3623641-3a1d-408e-bfaf-1b7c4e16aa63) +*diagram by [@mishig25](https://github.com/mishig25) (GGUF v3)* + +GGUF files are structured as follows. They use a global alignment specified in the `general.alignment` metadata field, referred to as `ALIGNMENT` below. Where required, the file is padded with `0x00` bytes to the next multiple of `general.alignment`. + +Fields, including arrays, are written sequentially without alignment unless otherwise specified. + +Models are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian. + +```c +enum ggml_type: uint32_t { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_COUNT = 40, +}; + +enum gguf_metadata_value_type: uint32_t { + // The value is a 8-bit unsigned integer. + GGUF_METADATA_VALUE_TYPE_UINT8 = 0, + // The value is a 8-bit signed integer. + GGUF_METADATA_VALUE_TYPE_INT8 = 1, + // The value is a 16-bit unsigned little-endian integer. + GGUF_METADATA_VALUE_TYPE_UINT16 = 2, + // The value is a 16-bit signed little-endian integer. + GGUF_METADATA_VALUE_TYPE_INT16 = 3, + // The value is a 32-bit unsigned little-endian integer. + GGUF_METADATA_VALUE_TYPE_UINT32 = 4, + // The value is a 32-bit signed little-endian integer. + GGUF_METADATA_VALUE_TYPE_INT32 = 5, + // The value is a 32-bit IEEE754 floating point number. + GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6, + // The value is a boolean. + // 1-byte value where 0 is false and 1 is true. + // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. + GGUF_METADATA_VALUE_TYPE_BOOL = 7, + // The value is a UTF-8 non-null-terminated string, with length prepended. + GGUF_METADATA_VALUE_TYPE_STRING = 8, + // The value is an array of other values, with the length and type prepended. + /// + // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. + GGUF_METADATA_VALUE_TYPE_ARRAY = 9, + // The value is a 64-bit unsigned little-endian integer. + GGUF_METADATA_VALUE_TYPE_UINT64 = 10, + // The value is a 64-bit signed little-endian integer. + GGUF_METADATA_VALUE_TYPE_INT64 = 11, + // The value is a 64-bit IEEE754 floating point number. + GGUF_METADATA_VALUE_TYPE_FLOAT64 = 12, +}; + +// A string in GGUF. +struct gguf_string_t { + // The length of the string, in bytes. + uint64_t len; + // The string as a UTF-8 non-null-terminated string. + char string[len]; +}; + +union gguf_metadata_value_t { + uint8_t uint8; + int8_t int8; + uint16_t uint16; + int16_t int16; + uint32_t uint32; + int32_t int32; + float float32; + uint64_t uint64; + int64_t int64; + double float64; + bool bool_; + gguf_string_t string; + struct { + // Any value type is valid, including arrays. + gguf_metadata_value_type type; + // Number of elements, not bytes + uint64_t len; + // The array of values. + gguf_metadata_value_t array[len]; + } array; +}; + +struct gguf_metadata_kv_t { + // The key of the metadata. It is a standard GGUF string, with the following caveats: + // - It must be a valid ASCII string. + // - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`. + // - It must be at most 2^16-1/65535 bytes long. + // Any keys that do not follow these rules are invalid. + gguf_string_t key; + + // The type of the value. + // Must be one of the `gguf_metadata_value_type` values. + gguf_metadata_value_type value_type; + // The value. + gguf_metadata_value_t value; +}; + +struct gguf_header_t { + // Magic number to announce that this is a GGUF file. + // Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`. + // Your executor might do little-endian byte order, so it might be + // check for 0x46554747 and letting the endianness cancel out. + // Consider being *very* explicit about the byte order here. + uint32_t magic; + // The version of the format implemented. + // Must be `3` for version described in this spec, which introduces big-endian support. + // + // This version should only be increased for structural changes to the format. + // Changes that do not affect the structure of the file should instead update the metadata + // to signify the change. + uint32_t version; + // The number of tensors in the file. + // This is explicit, instead of being included in the metadata, to ensure it is always present + // for loading the tensors. + uint64_t tensor_count; + // The number of metadata key-value pairs. + uint64_t metadata_kv_count; + // The metadata key-value pairs. + gguf_metadata_kv_t metadata_kv[metadata_kv_count]; +}; + +uint64_t align_offset(uint64_t offset) { + return offset + (ALIGNMENT - (offset % ALIGNMENT)) % ALIGNMENT; +} + +struct gguf_tensor_info_t { + // The name of the tensor. It is a standard GGUF string, with the caveat that + // it must be at most 64 bytes long. + gguf_string_t name; + // The number of dimensions in the tensor. + // Currently at most 4, but this may change in the future. + uint32_t n_dimensions; + // The dimensions of the tensor. + uint64_t dimensions[n_dimensions]; + // The type of the tensor. + ggml_type type; + // The offset of the tensor's data in this file in bytes. + // + // This offset is relative to `tensor_data`, not to the start + // of the file, to make it easier for writers to write the file. + // Readers should consider exposing this offset relative to the + // file to make it easier to read the data. + // + // Must be a multiple of `ALIGNMENT`. That is, `align_offset(offset) == offset`. + uint64_t offset; +}; + +struct gguf_file_t { + // The header of the file. + gguf_header_t header; + + // Tensor infos, which can be used to locate the tensor data. + gguf_tensor_info_t tensor_infos[header.tensor_count]; + + // Padding to the nearest multiple of `ALIGNMENT`. + // + // That is, if `sizeof(header) + sizeof(tensor_infos)` is not a multiple of `ALIGNMENT`, + // this padding is added to make it so. + // + // This can be calculated as `align_offset(position) - position`, where `position` is + // the position of the end of `tensor_infos` (i.e. `sizeof(header) + sizeof(tensor_infos)`). + uint8_t _padding[]; + + // Tensor data. + // + // This is arbitrary binary data corresponding to the weights of the model. This data should be close + // or identical to the data in the original model file, but may be different due to quantization or + // other optimizations for inference. Any such deviations should be recorded in the metadata or as + // part of the architecture definition. + // + // Each tensor's data must be stored within this array, and located through its `tensor_infos` entry. + // The offset of each tensor's data must be a multiple of `ALIGNMENT`, and the space between tensors + // should be padded to `ALIGNMENT` bytes. + uint8_t tensor_data[]; +}; +``` + +## Standardized key-value pairs + +The following key-value pairs are standardized. This list may grow in the future as more use cases are discovered. Where possible, names are shared with the original model definitions to make it easier to map between the two. + +Not all of these are required, but they are all recommended. Keys that are required are bolded. For omitted pairs, the reader should assume that the value is unknown and either default or error as appropriate. + +The community can develop their own key-value pairs to carry additional data. However, these should be namespaced with the relevant community name to avoid collisions. For example, the `rustformers` community might use `rustformers.` as a prefix for all of their keys. + +If a particular community key is widely used, it may be promoted to a standardized key. + +By convention, most counts/lengths/etc are `uint64` unless otherwise specified. This is to allow for larger models to be supported in the future. Some models may use `uint32` for their values; it is recommended that readers support both. + +### General + +#### Required + +- **`general.architecture: string`**: describes what architecture this model implements. All lowercase ASCII, with only `[a-z0-9]+` characters allowed. Known values include: + - `llama` + - `mpt` + - `gptneox` + - `gptj` + - `gpt2` + - `bloom` + - `falcon` + - `mamba` + - `rwkv` +- **`general.quantization_version: uint32`**: The version of the quantization format. Not required if the model is not quantized (i.e. no tensors are quantized). If any tensors are quantized, this _must_ be present. This is separate to the quantization scheme of the tensors itself; the quantization version may change without changing the scheme's name (e.g. the quantization scheme is Q5_K, and the quantization version is 4). +- **`general.alignment: uint32`**: the global alignment to use, as described above. This can vary to allow for different alignment schemes, but it must be a multiple of 8. Some writers may not write the alignment. If the alignment is **not** specified, assume it is `32`. + +#### General metadata + +- `general.name: string`: The name of the model. This should be a human-readable name that can be used to identify the model. It should be unique within the community that the model is defined in. +- `general.author: string`: The author of the model. +- `general.version: string`: The version of the model. +- `general.organization: string`: The organization of the model. +- `general.basename: string`: The base model name / architecture of the model +- `general.finetune: string`: What has the base model been optimized toward. +- `general.description: string`: free-form description of the model including anything that isn't covered by the other fields +- `general.quantized_by: string`: The name of the individual who quantized the model +- `general.size_label: string`: Size class of the model, such as number of weights and experts. (Useful for leader boards) +- `general.license: string`: License of the model, expressed as a [SPDX license expression](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/) (e.g. `"MIT OR Apache-2.0`). Do not include any other information, such as the license text or the URL to the license. +- `general.license.name: string`: Human friendly license name +- `general.license.link: string`: URL to the license. +- `general.url: string`: URL to the model's homepage. This can be a GitHub repo, a paper, etc. +- `general.doi: string`: Digital Object Identifier (DOI) https://www.doi.org/ +- `general.uuid: string`: [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier) +- `general.repo_url: string`: URL to the model's repository such as a GitHub repo or HuggingFace repo +- `general.tags: string[]`: List of tags that can be used as search terms for a search engine or social media +- `general.languages: string[]`: What languages can the model speak. Encoded as [ISO 639](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) two letter codes +- `general.datasets: string[]`: Links or references to datasets that the model was trained upon +- `general.file_type: uint32`: An enumerated value describing the type of the majority of the tensors in the file. Optional; can be inferred from the tensor types. + - `ALL_F32 = 0` + - `MOSTLY_F16 = 1` + - `MOSTLY_Q4_0 = 2` + - `MOSTLY_Q4_1 = 3` + - `MOSTLY_Q4_1_SOME_F16 = 4` + - `MOSTLY_Q4_2 = 5` (support removed) + - `MOSTLY_Q4_3 = 6` (support removed) + - `MOSTLY_Q8_0 = 7` + - `MOSTLY_Q5_0 = 8` + - `MOSTLY_Q5_1 = 9` + - `MOSTLY_Q2_K = 10` + - `MOSTLY_Q3_K_S = 11` + - `MOSTLY_Q3_K_M = 12` + - `MOSTLY_Q3_K_L = 13` + - `MOSTLY_Q4_K_S = 14` + - `MOSTLY_Q4_K_M = 15` + - `MOSTLY_Q5_K_S = 16` + - `MOSTLY_Q5_K_M = 17` + - `MOSTLY_Q6_K = 18` + +#### Source metadata + +Information about where this model came from. This is useful for tracking the provenance of the model, and for finding the original source if the model is modified. For a model that was converted from GGML, for example, these keys would point to the model that was converted from. + +- `general.source.url: string`: URL to the source of the model's homepage. This can be a GitHub repo, a paper, etc. +- `general.source.doi: string`: Source Digital Object Identifier (DOI) https://www.doi.org/ +- `general.source.uuid: string`: Source [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier) +- `general.source.repo_url: string`: URL to the source of the model's repository such as a GitHub repo or HuggingFace repo + +- `general.base_model.count: uint32`: Number of parent models +- `general.base_model.{id}.name: string`: The name of the parent model. +- `general.base_model.{id}.author: string`: The author of the parent model. +- `general.base_model.{id}.version: string`: The version of the parent model. +- `general.base_model.{id}.organization: string`: The organization of the parent model. +- `general.base_model.{id}.url: string`: URL to the source of the parent model's homepage. This can be a GitHub repo, a paper, etc. +- `general.base_model.{id}.doi: string`: Parent Digital Object Identifier (DOI) https://www.doi.org/ +- `general.base_model.{id}.uuid: string`: Parent [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier) +- `general.base_model.{id}.repo_url: string`: URL to the source of the parent model's repository such as a GitHub repo or HuggingFace repo + +### LLM + +In the following, `[llm]` is used to fill in for the name of a specific LLM architecture. For example, `llama` for LLaMA, `mpt` for MPT, etc. If mentioned in an architecture's section, it is required for that architecture, but not all keys are required for all architectures. Consult the relevant section for more information. + +- `[llm].context_length: uint64`: Also known as `n_ctx`. length of the context (in tokens) that the model was trained on. For most architectures, this is the hard limit on the length of the input. Architectures, like RWKV, that are not reliant on transformer-style attention may be able to handle larger inputs, but this is not guaranteed. +- `[llm].embedding_length: uint64`: Also known as `n_embd`. Embedding layer size. +- `[llm].block_count: uint64`: The number of blocks of attention+feed-forward layers (i.e. the bulk of the LLM). Does not include the input or embedding layers. +- `[llm].feed_forward_length: uint64`: Also known as `n_ff`. The length of the feed-forward layer. +- `[llm].use_parallel_residual: bool`: Whether or not the parallel residual logic should be used. +- `[llm].tensor_data_layout: string`: When a model is converted to GGUF, tensors may be rearranged to improve performance. This key describes the layout of the tensor data. This is not required; if not present, it is assumed to be `reference`. + - `reference`: tensors are laid out in the same order as the original model + - further options can be found for each architecture in their respective sections +- `[llm].expert_count: uint32`: Number of experts in MoE models (optional for non-MoE arches). +- `[llm].expert_used_count: uint32`: Number of experts used during each token token evaluation (optional for non-MoE arches). + +#### Attention + +- `[llm].attention.head_count: uint64`: Also known as `n_head`. Number of attention heads. +- `[llm].attention.head_count_kv: uint64`: The number of heads per group used in Grouped-Query-Attention. If not present or if present and equal to `[llm].attention.head_count`, the model does not use GQA. +- `[llm].attention.max_alibi_bias: float32`: The maximum bias to use for ALiBI. +- `[llm].attention.clamp_kqv: float32`: Value (`C`) to clamp the values of the `Q`, `K`, and `V` tensors between (`[-C, C]`). +- `[llm].attention.layer_norm_epsilon: float32`: Layer normalization epsilon. +- `[llm].attention.layer_norm_rms_epsilon: float32`: Layer RMS normalization epsilon. +- `[llm].attention.key_length: uint32`: The optional size of a key head, $d_k$. If not specified, it will be `n_embd / n_head`. +- `[llm].attention.value_length: uint32`: The optional size of a value head, $d_v$. If not specified, it will be `n_embd / n_head`. + +#### RoPE + +- `[llm].rope.dimension_count: uint64`: The number of rotary dimensions for RoPE. +- `[llm].rope.freq_base: float32`: The base frequency for RoPE. + +##### Scaling + +The following keys describe RoPE scaling parameters: + +- `[llm].rope.scaling.type: string`: Can be `none`, `linear`, or `yarn`. +- `[llm].rope.scaling.factor: float32`: A scale factor for RoPE to adjust the context length. +- `[llm].rope.scaling.original_context_length: uint32_t`: The original context length of the base model. +- `[llm].rope.scaling.finetuned: bool`: True if model has been finetuned with RoPE scaling. + +Note that older models may not have these keys, and may instead use the following key: + +- `[llm].rope.scale_linear: float32`: A linear scale factor for RoPE to adjust the context length. + +It is recommended that models use the newer keys if possible, as they are more flexible and allow for more complex scaling schemes. Executors will need to support both indefinitely. + +#### SSM + +- `[llm].ssm.conv_kernel: uint32`: The size of the rolling/shift state. +- `[llm].ssm.inner_size: uint32`: The embedding size of the states. +- `[llm].ssm.state_size: uint32`: The size of the recurrent state. +- `[llm].ssm.time_step_rank: uint32`: The rank of time steps. + +#### Models + +The following sections describe the metadata for each model architecture. Each key specified _must_ be present. + +##### LLaMA + +- `llama.context_length` +- `llama.embedding_length` +- `llama.block_count` +- `llama.feed_forward_length` +- `llama.rope.dimension_count` +- `llama.attention.head_count` +- `llama.attention.layer_norm_rms_epsilon` + +###### Optional + +- `llama.rope.scale` +- `llama.attention.head_count_kv` +- `llama.tensor_data_layout`: + - `Meta AI original pth`: + ```python + def permute(weights: NDArray, n_head: int) -> NDArray: + return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:]) + .swapaxes(1, 2) + .reshape(weights.shape)) + ``` +- `llama.expert_count` +- `llama.expert_used_count` + +##### MPT + +- `mpt.context_length` +- `mpt.embedding_length` +- `mpt.block_count` +- `mpt.attention.head_count` +- `mpt.attention.alibi_bias_max` +- `mpt.attention.clip_kqv` +- `mpt.attention.layer_norm_epsilon` + +##### GPT-NeoX + +- `gptneox.context_length` +- `gptneox.embedding_length` +- `gptneox.block_count` +- `gptneox.use_parallel_residual` +- `gptneox.rope.dimension_count` +- `gptneox.attention.head_count` +- `gptneox.attention.layer_norm_epsilon` + +###### Optional + +- `gptneox.rope.scale` + +##### GPT-J + +- `gptj.context_length` +- `gptj.embedding_length` +- `gptj.block_count` +- `gptj.rope.dimension_count` +- `gptj.attention.head_count` +- `gptj.attention.layer_norm_epsilon` + +###### Optional + +- `gptj.rope.scale` + +##### GPT-2 + +- `gpt2.context_length` +- `gpt2.embedding_length` +- `gpt2.block_count` +- `gpt2.attention.head_count` +- `gpt2.attention.layer_norm_epsilon` + +##### BLOOM + +- `bloom.context_length` +- `bloom.embedding_length` +- `bloom.block_count` +- `bloom.feed_forward_length` +- `bloom.attention.head_count` +- `bloom.attention.layer_norm_epsilon` + +##### Falcon + +- `falcon.context_length` +- `falcon.embedding_length` +- `falcon.block_count` +- `falcon.attention.head_count` +- `falcon.attention.head_count_kv` +- `falcon.attention.use_norm` +- `falcon.attention.layer_norm_epsilon` + +###### Optional + +- `falcon.tensor_data_layout`: + + - `jploski` (author of the original GGML implementation of Falcon): + + ```python + # The original query_key_value tensor contains n_head_kv "kv groups", + # each consisting of n_head/n_head_kv query weights followed by one key + # and one value weight (shared by all query heads in the kv group). + # This layout makes it a big pain to work with in GGML. + # So we rearrange them here,, so that we have n_head query weights + # followed by n_head_kv key weights followed by n_head_kv value weights, + # in contiguous fashion. + + if "query_key_value" in src: + qkv = model[src].view( + n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head) + + q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head) + k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head) + v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head) + + model[src] = torch.cat((q,k,v)).reshape_as(model[src]) + ``` + +##### Mamba + +- `mamba.context_length` +- `mamba.embedding_length` +- `mamba.block_count` +- `mamba.ssm.conv_kernel` +- `mamba.ssm.inner_size` +- `mamba.ssm.state_size` +- `mamba.ssm.time_step_rank` +- `mamba.attention.layer_norm_rms_epsilon` + +##### RWKV + +The vocabulary size is the same as the number of rows in the `head` matrix. + +- `rwkv.architecture_version: uint32`: The only allowed value currently is 4. Version 5 is expected to appear some time in the future. +- `rwkv.context_length: uint64`: Length of the context used during training or fine-tuning. RWKV is able to handle larger context than this limit, but the output quality may suffer. +- `rwkv.block_count: uint64` +- `rwkv.embedding_length: uint64` +- `rwkv.feed_forward_length: uint64` + +##### Whisper + +Keys that do not have types defined should be assumed to share definitions with `llm.` keys. +(For example, `whisper.context_length` is equivalent to `llm.context_length`.) +This is because they are both transformer models. + +- `whisper.encoder.context_length` +- `whisper.encoder.embedding_length` +- `whisper.encoder.block_count` +- `whisper.encoder.mels_count: uint64` +- `whisper.encoder.attention.head_count` + +- `whisper.decoder.context_length` +- `whisper.decoder.embedding_length` +- `whisper.decoder.block_count` +- `whisper.decoder.attention.head_count` + +#### Prompting + +**TODO**: Include prompt format, and/or metadata about how it should be used (instruction, conversation, autocomplete, etc). + +### LoRA + +**TODO**: Figure out what metadata is needed for LoRA. Probably desired features: + +- match an existing model exactly, so that it can't be misapplied +- be marked as a LoRA so executors won't try to run it by itself + +Should this be an architecture, or should it share the details of the original model with additional fields to mark it as a LoRA? + +### Tokenizer + +The following keys are used to describe the tokenizer of the model. It is recommended that model authors support as many of these as possible, as it will allow for better tokenization quality with supported executors. + +#### GGML + +GGML supports an embedded vocabulary that enables inference of the model, but implementations of tokenization using this vocabulary (i.e. `llama.cpp`'s tokenizer) may have lower accuracy than the original tokenizer used for the model. When a more accurate tokenizer is available and supported, it should be used instead. + +It is not guaranteed to be standardized across models, and may change in the future. It is recommended that model authors use a more standardized tokenizer if possible. + +- `tokenizer.ggml.model: string`: The name of the tokenizer model. + - `llama`: Llama style SentencePiece (tokens and scores extracted from HF `tokenizer.model`) + - `replit`: Replit style SentencePiece (tokens and scores extracted from HF `spiece.model`) + - `gpt2`: GPT-2 / GPT-NeoX style BPE (tokens extracted from HF `tokenizer.json`) + - `rwkv`: RWKV tokenizer +- `tokenizer.ggml.tokens: array[string]`: A list of tokens indexed by the token ID used by the model. +- `tokenizer.ggml.scores: array[float32]`: If present, the score/probability of each token. If not present, all tokens are assumed to have equal probability. If present, it must have the same length and index as `tokens`. +- `tokenizer.ggml.token_type: array[int32]`: The token type (1=normal, 2=unknown, 3=control, 4=user defined, 5=unused, 6=byte). If present, it must have the same length and index as `tokens`. +- `tokenizer.ggml.merges: array[string]`: If present, the merges of the tokenizer. If not present, the tokens are assumed to be atomic. +- `tokenizer.ggml.added_tokens: array[string]`: If present, tokens that were added after training. + +##### Special tokens + +- `tokenizer.ggml.bos_token_id: uint32`: Beginning of sequence marker +- `tokenizer.ggml.eos_token_id: uint32`: End of sequence marker +- `tokenizer.ggml.unknown_token_id: uint32`: Unknown token +- `tokenizer.ggml.separator_token_id: uint32`: Separator token +- `tokenizer.ggml.padding_token_id: uint32`: Padding token + +#### Hugging Face + +Hugging Face maintains their own `tokenizers` library that supports a wide variety of tokenizers. If your executor uses this library, it may be able to use the model's tokenizer directly. + +- `tokenizer.huggingface.json: string`: the entirety of the HF `tokenizer.json` for a given model (e.g. ). Included for compatibility with executors that support HF tokenizers directly. + +#### Other + +Other tokenizers may be used, but are not necessarily standardized. They may be executor-specific. They will be documented here as they are discovered/further developed. + +- `tokenizer.rwkv.world: string`: a RWKV World tokenizer, like [this](https://github.com/BlinkDL/ChatRWKV/blob/main/tokenizer/rwkv_vocab_v20230424.txt). This text file should be included verbatim. +- `tokenizer.chat_template : string`: a Jinja template that specifies the input format expected by the model. For more details see: + +### Computation graph + +This is a future extension and still needs to be discussed, and may necessitate a new GGUF version. At the time of writing, the primary blocker is the stabilization of the computation graph format. + +A sample computation graph of GGML nodes could be included in the model itself, allowing an executor to run the model without providing its own implementation of the architecture. This would allow for a more consistent experience across executors, and would allow for more complex architectures to be supported without requiring the executor to implement them. + +## Standardized tensor names + +To minimize complexity and maximize compatibility, it is recommended that models using the transformer architecture use the following naming convention for their tensors: + +### Base layers + +`AA.weight` `AA.bias` + +where `AA` can be: + +- `token_embd`: Token embedding layer +- `pos_embd`: Position embedding layer +- `output_norm`: Output normalization layer +- `output`: Output layer + +### Attention and feed-forward layer blocks + +`blk.N.BB.weight` `blk.N.BB.bias` + +where N signifies the block number a layer belongs to, and where `BB` could be: + +- `attn_norm`: Attention normalization layer +- `attn_norm_2`: Attention normalization layer +- `attn_qkv`: Attention query-key-value layer +- `attn_q`: Attention query layer +- `attn_k`: Attention key layer +- `attn_v`: Attention value layer +- `attn_output`: Attention output layer + +- `ffn_norm`: Feed-forward network normalization layer +- `ffn_up`: Feed-forward network "up" layer +- `ffn_gate`: Feed-forward network "gate" layer +- `ffn_down`: Feed-forward network "down" layer +- `ffn_gate_inp`: Expert-routing layer for the Feed-forward network in MoE models +- `ffn_gate_exp`: Feed-forward network "gate" layer per expert in MoE models +- `ffn_down_exp`: Feed-forward network "down" layer per expert in MoE models +- `ffn_up_exp`: Feed-forward network "up" layer per expert in MoE models + +- `ssm_in`: State space model input projections layer +- `ssm_conv1d`: State space model rolling/shift layer +- `ssm_x`: State space model selective parametrization layer +- `ssm_a`: State space model state compression layer +- `ssm_d`: State space model skip connection layer +- `ssm_dt`: State space model time step layer +- `ssm_out`: State space model output projection layer + +## Version History + +This document is actively updated to describe the current state of the metadata, and these changes are not tracked outside of the commits. + +However, the format _itself_ has changed. The following sections describe the changes to the format itself. + +### v3 + +Adds big-endian support. + +### v2 + +Most countable values (lengths, etc) were changed from `uint32` to `uint64` to allow for larger models to be supported in the future. + +### v1 + +Initial version. + +## Historical State of Affairs + +The following information is provided for context, but is not necessary to understand the rest of this document. + +### Overview + +At present, there are three GGML file formats floating around for LLMs: + +- **GGML** (unversioned): baseline format, with no versioning or alignment. +- **GGMF** (versioned): the same as GGML, but with versioning. Only one version exists. +- **GGJT**: Aligns the tensors to allow for use with `mmap`, which requires alignment. v1, v2 and v3 are identical, but the latter versions use a different quantization scheme that is incompatible with previous versions. + +GGML is primarily used by the examples in `ggml`, while GGJT is used by `llama.cpp` models. Other executors may use any of the three formats, but this is not 'officially' supported. + +These formats share the same fundamental structure: + +- a magic number with an optional version number +- model-specific hyperparameters, including + - metadata about the model, such as the number of layers, the number of heads, etc. + - a `ftype` that describes the type of the majority of the tensors, + - for GGML files, the quantization version is encoded in the `ftype` divided by 1000 +- an embedded vocabulary, which is a list of strings with length prepended. The GGMF/GGJT formats embed a float32 score next to the strings. +- finally, a list of tensors with their length-prepended name, type, and (aligned, in the case of GGJT) tensor data + +Notably, this structure does not identify what model architecture the model belongs to, nor does it offer any flexibility for changing the structure of the hyperparameters. This means that the only way to add new hyperparameters is to add them to the end of the list, which is a breaking change for existing models. + +### Drawbacks + +Unfortunately, over the last few months, there are a few issues that have become apparent with the existing models: + +- There's no way to identify which model architecture a given model is for, because that information isn't present + - Similarly, existing programs cannot intelligently fail upon encountering new architectures +- Adding or removing any new hyperparameters is a breaking change, which is impossible for a reader to detect without using heuristics +- Each model architecture requires its own conversion script to their architecture's variant of GGML +- Maintaining backwards compatibility without breaking the structure of the format requires clever tricks, like packing the quantization version into the ftype, which are not guaranteed to be picked up by readers/writers, and are not consistent between the two formats + +### Why not other formats? + +There are a few other formats that could be used, but issues include: + +- requiring additional dependencies to load or save the model, which is complicated in a C environment +- limited or no support for 4-bit quantization +- existing cultural expectations (e.g. whether or not the model is a directory or a file) +- lack of support for embedded vocabularies +- lack of control over direction of future development + +Ultimately, it is likely that GGUF will remain necessary for the foreseeable future, and it is better to have a single format that is well-documented and supported by all executors than to contort an existing format to fit the needs of GGML. diff --git a/ggml/examples/CMakeLists.txt b/ggml/examples/CMakeLists.txt new file mode 100644 index 0000000..84ed57a --- /dev/null +++ b/ggml/examples/CMakeLists.txt @@ -0,0 +1,34 @@ +if (GGML_ALL_WARNINGS) + if (NOT MSVC) + set(cxx_flags + # TODO(marella): Add other warnings. + -Wpedantic + -Wunused-variable + -Wno-unused-function + -Wno-multichar + ) + add_compile_options("$<$:${cxx_flags}>") + endif() +endif() + +add_library(common STATIC common.cpp) +target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +add_library(common-ggml STATIC common-ggml.cpp) +target_link_libraries(common-ggml PRIVATE ggml) +target_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +add_subdirectory(yolo) + +if (NOT GGML_BACKEND_DL) + add_subdirectory(gpt-2) + add_subdirectory(gpt-j) + add_subdirectory(mnist) + add_subdirectory(sam) + add_subdirectory(simple) + add_subdirectory(magika) +endif() + +if (GGML_METAL) + add_subdirectory(perf-metal) +endif() diff --git a/ggml/examples/common-ggml.cpp b/ggml/examples/common-ggml.cpp new file mode 100644 index 0000000..3f2eded --- /dev/null +++ b/ggml/examples/common-ggml.cpp @@ -0,0 +1,244 @@ +#include "common-ggml.h" + +#include +#include + +static const std::map GGML_FTYPE_MAP = { + {"q4_0", GGML_FTYPE_MOSTLY_Q4_0}, + {"q4_1", GGML_FTYPE_MOSTLY_Q4_1}, + {"q5_0", GGML_FTYPE_MOSTLY_Q5_0}, + {"q5_1", GGML_FTYPE_MOSTLY_Q5_1}, + {"q8_0", GGML_FTYPE_MOSTLY_Q8_0}, + {"q2_k", GGML_FTYPE_MOSTLY_Q2_K}, + {"q3_k", GGML_FTYPE_MOSTLY_Q3_K}, + {"q4_k", GGML_FTYPE_MOSTLY_Q4_K}, + {"q5_k", GGML_FTYPE_MOSTLY_Q5_K}, + {"q6_k", GGML_FTYPE_MOSTLY_Q6_K}, +}; + +void ggml_print_ftypes(FILE * fp) { + for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) { + fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second); + } +} + +enum ggml_ftype ggml_parse_ftype(const char * str) { + enum ggml_ftype ftype; + if (str[0] == 'q') { + const auto it = GGML_FTYPE_MAP.find(str); + if (it == GGML_FTYPE_MAP.end()) { + fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str); + return GGML_FTYPE_UNKNOWN; + } + ftype = it->second; + } else { + ftype = (enum ggml_ftype) atoi(str); + } + + return ftype; +} + +bool ggml_common_quantize_0( + std::ifstream & finp, + std::ofstream & fout, + const ggml_ftype ftype, + const std::vector & to_quant, + const std::vector & to_skip) { + + ggml_type qtype = GGML_TYPE_F32; + + switch (ftype) { + case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break; + case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break; + case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break; + case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break; + case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break; + case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break; + case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break; + case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break; + case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break; + case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break; + case GGML_FTYPE_UNKNOWN: + case GGML_FTYPE_ALL_F32: + case GGML_FTYPE_MOSTLY_F16: + case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: + case GGML_FTYPE_MOSTLY_IQ2_XXS: + case GGML_FTYPE_MOSTLY_IQ2_XS: + case GGML_FTYPE_MOSTLY_IQ2_S: + case GGML_FTYPE_MOSTLY_IQ3_XXS: + case GGML_FTYPE_MOSTLY_IQ3_S: + case GGML_FTYPE_MOSTLY_IQ1_S: + case GGML_FTYPE_MOSTLY_IQ4_NL: + case GGML_FTYPE_MOSTLY_IQ4_XS: + case GGML_FTYPE_MOSTLY_IQ1_M: + case GGML_FTYPE_MOSTLY_BF16: + case GGML_FTYPE_MOSTLY_MXFP4: + case GGML_FTYPE_MOSTLY_NVFP4: + case GGML_FTYPE_MOSTLY_Q1_0: + { + fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype); + return false; + } + }; + + if (!ggml_is_quantized(qtype)) { + fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype)); + return false; + } + + size_t total_size_org = 0; + size_t total_size_new = 0; + + std::vector work; + + std::vector data_u8; + std::vector data_f16; + std::vector data_f32; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + finp.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + finp.read(reinterpret_cast(&length), sizeof(length)); + finp.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (finp.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[4] = { 1, 1, 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + finp.read (reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + finp.read (&name[0], length); + + printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype)); + + bool quantize = false; + + // check if we should quantize this tensor + for (const auto & s : to_quant) { + if (std::regex_match(name, std::regex(s))) { + quantize = true; + break; + } + } + + // check if we should skip this tensor + for (const auto & s : to_skip) { + if (std::regex_match(name, std::regex(s))) { + quantize = false; + break; + } + } + + // quantize only 2D tensors + quantize &= (n_dims == 2); + + if (quantize) { + if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) { + fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype)); + return false; + } + + if (ttype == GGML_TYPE_F16) { + data_f16.resize(nelements); + finp.read(reinterpret_cast(data_f16.data()), nelements * sizeof(ggml_fp16_t)); + data_f32.resize(nelements); + for (int i = 0; i < nelements; ++i) { + data_f32[i] = ggml_fp16_to_fp32(data_f16[i]); + } + } else { + data_f32.resize(nelements); + finp.read(reinterpret_cast(data_f32.data()), nelements * sizeof(float)); + } + + ttype = qtype; + } else { + const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t); + + data_u8.resize(nelements*bpe); + finp.read(reinterpret_cast(data_u8.data()), nelements * bpe); + } + + fout.write(reinterpret_cast(&n_dims), sizeof(n_dims)); + fout.write(reinterpret_cast(&length), sizeof(length)); + fout.write(reinterpret_cast(&ttype), sizeof(ttype)); + for (int i = 0; i < n_dims; ++i) { + fout.write(reinterpret_cast(&ne[i]), sizeof(ne[i])); + } + fout.write(&name[0], length); + + if (quantize) { + work.resize(nelements); // for quantization + + size_t cur_size = 0; + switch ((ggml_type) ttype) { + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q5_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + { + cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr); + } break; + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_I8: + case GGML_TYPE_I16: + case GGML_TYPE_I32: + case GGML_TYPE_I64: + case GGML_TYPE_F64: + case GGML_TYPE_Q8_1: + case GGML_TYPE_Q8_K: + case GGML_TYPE_IQ2_XXS: + case GGML_TYPE_IQ2_XS: + case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ3_XXS: + case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ4_NL: + case GGML_TYPE_IQ4_XS: + case GGML_TYPE_IQ1_M: + case GGML_TYPE_BF16: + case GGML_TYPE_TQ1_0: + case GGML_TYPE_TQ2_0: + case GGML_TYPE_MXFP4: + case GGML_TYPE_NVFP4: + case GGML_TYPE_Q1_0: + case GGML_TYPE_COUNT: + { + fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype)); + return false; + } + } + + fout.write(reinterpret_cast(work.data()), cur_size); + total_size_new += cur_size; + + printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0); + } else { + printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0); + fout.write(reinterpret_cast(data_u8.data()), data_u8.size()); + total_size_new += data_u8.size(); + } + + total_size_org += nelements * sizeof(float); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0); + printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype)); + + return true; +} diff --git a/ggml/examples/common-ggml.h b/ggml/examples/common-ggml.h new file mode 100644 index 0000000..477de34 --- /dev/null +++ b/ggml/examples/common-ggml.h @@ -0,0 +1,18 @@ +#pragma once + +#include "ggml.h" + +#include +#include +#include + +enum ggml_ftype ggml_parse_ftype(const char * str); + +void ggml_print_ftypes(FILE * fp = stderr); + +bool ggml_common_quantize_0( + std::ifstream & finp, + std::ofstream & fout, + const ggml_ftype ftype, + const std::vector & to_quant, + const std::vector & to_skip); diff --git a/ggml/examples/common.cpp b/ggml/examples/common.cpp new file mode 100644 index 0000000..8eb633e --- /dev/null +++ b/ggml/examples/common.cpp @@ -0,0 +1,675 @@ +#define _USE_MATH_DEFINES // for M_PI + +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +// Function to check if the next argument exists +static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) { + if (i + 1 < argc && argv[i + 1][0] != '-') { + return argv[++i]; + } else { + fprintf(stderr, "error: %s requires one argument.\n", flag.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } +} + +bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + + if (arg == "-s" || arg == "--seed") { + params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-t" || arg == "--threads") { + params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-p" || arg == "--prompt") { + params.prompt = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-n" || arg == "--n_predict") { + params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-np" || arg == "--n_parallel") { + params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_k") { + params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--top_p") { + params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--temp") { + params.temp = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-last-n") { + params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--repeat-penalty") { + params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-b" || arg == "--batch_size") { + params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-c" || arg == "--context") { + params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") { + params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "--ignore-eos") { + params.ignore_eos = true; + } else if (arg == "-m" || arg == "--model") { + params.model = get_next_arg(i, argc, argv, arg, params); + } else if (arg == "-i" || arg == "--interactive") { + params.interactive = true; + } else if (arg == "-ip" || arg == "--interactive-port") { + params.interactive = true; + params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params)); + } else if (arg == "-h" || arg == "--help") { + gpt_print_usage(argc, argv, params); + exit(0); + } else if (arg == "-f" || arg == "--file") { + get_next_arg(i, argc, argv, arg, params); + std::ifstream file(argv[i]); + if (!file) { + fprintf(stderr, "error: failed to open file '%s'\n", argv[i]); + break; + } + std::copy(std::istreambuf_iterator(file), std::istreambuf_iterator(), back_inserter(params.prompt)); + if (params.prompt.back() == '\n') { + params.prompt.pop_back(); + } + } else if (arg == "-tt" || arg == "--token_test") { + params.token_test = get_next_arg(i, argc, argv, arg, params); + } + else { + fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + gpt_print_usage(argc, argv, params); + exit(0); + } + } + + return true; +} + +void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); + fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); + fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); + fprintf(stderr, " prompt to start generation with (default: random)\n"); + fprintf(stderr, " -f FNAME, --file FNAME\n"); + fprintf(stderr, " load prompt from a file\n"); + fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n"); + fprintf(stderr, " test tokenization\n"); + fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict); + fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k); + fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p); + fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp); + fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n); + fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty); + fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch); + fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx); + fprintf(stderr, " --ignore-eos ignore EOS token during generation\n"); + fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers); + fprintf(stderr, " -m FNAME, --model FNAME\n"); + fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); + fprintf(stderr, "\n"); +} + +std::string gpt_random_prompt(std::mt19937 & rng) { + const int r = rng() % 10; + switch (r) { + case 0: return "So"; + case 1: return "Once upon a time"; + case 2: return "When"; + case 3: return "The"; + case 4: return "After"; + case 5: return "If"; + case 6: return "import"; + case 7: return "He"; + case 8: return "She"; + case 9: return "They"; + } + + return "The"; +} + +std::string trim(const std::string & s) { + std::regex e("^\\s+|\\s+$"); + return std::regex_replace(s, e, ""); +} + +std::string replace(const std::string & s, const std::string & from, const std::string & to) { + std::string result = s; + size_t pos = 0; + while ((pos = result.find(from, pos)) != std::string::npos) { + result.replace(pos, from.length(), to); + pos += to.length(); + } + return result; +} + +void gpt_vocab::add_special_token(const std::string & token) { + special_tokens.push_back(token); +} + +std::map json_parse(const std::string & fname) { + std::map result; + + // read file into string + std::string json; + { + std::ifstream ifs(fname); + if (!ifs) { + fprintf(stderr, "Failed to open %s\n", fname.c_str()); + exit(1); + } + + json = std::string((std::istreambuf_iterator(ifs)), + (std::istreambuf_iterator())); + } + + if (json[0] != '{') { + return result; + } + + // parse json + { + bool has_key = false; + bool in_token = false; + + std::string str_key = ""; + std::string str_val = ""; + + int n = json.size(); + for (int i = 1; i < n; ++i) { + if (!in_token) { + if (json[i] == ' ') continue; + if (json[i] == '"') { + in_token = true; + continue; + } + } else { + if (json[i] == '\\' && i+1 < n) { + if (has_key == false) { + str_key += json[i]; + } else { + str_val += json[i]; + } + ++i; + } else if (json[i] == '"') { + if (has_key == false) { + has_key = true; + ++i; + while (json[i] == ' ') ++i; + ++i; // : + while (json[i] == ' ') ++i; + if (json[i] != '\"') { + while (json[i] != ',' && json[i] != '}') { + str_val += json[i++]; + } + has_key = false; + } else { + in_token = true; + continue; + } + } else { + has_key = false; + } + + str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space + str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line + str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> " + + try { + result[str_key] = std::stoi(str_val); + } catch (...) { + //fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str()); + + } + str_key = ""; + str_val = ""; + in_token = false; + continue; + } + if (has_key == false) { + str_key += json[i]; + } else { + str_val += json[i]; + } + } + } + } + + return result; +} + +void gpt_split_words(std::string str, std::vector& words) { + const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"; + const std::regex re(pattern); + std::smatch m; + + while (std::regex_search(str, m, re)) { + for (auto x : m) { + words.push_back(x); + } + str = m.suffix(); + } +} + +std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { + std::vector words; + + // first split the text into words + { + std::string str = text; + + // Generate the subpattern from the special_tokens vector if it's not empty + if (!vocab.special_tokens.empty()) { + const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])"); + std::string special_tokens_subpattern; + for (const auto & token : vocab.special_tokens) { + if (!special_tokens_subpattern.empty()) { + special_tokens_subpattern += "|"; + } + special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)"); + } + + std::regex re(special_tokens_subpattern); + std::smatch m; + // Split the text by special tokens. + while (std::regex_search(str, m, re)) { + // Split the substrings in-between special tokens into words. + gpt_split_words(m.prefix(), words); + // Add matched special tokens as words. + for (auto x : m) { + words.push_back(x); + } + str = m.suffix(); + } + // Remaining text without special tokens will be handled below. + } + + gpt_split_words(str, words); + } + + // find the longest token that forms each word in words: + std::vector tokens; + for (const auto & word : words) { + for (int i = 0; i < (int) word.size(); ){ + for (int j = word.size() - 1; j >= i; j--){ + auto cand = word.substr(i, j-i+1); + auto it = vocab.token_to_id.find(cand); + if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab + tokens.push_back(it->second); + i = j + 1; + break; + } + else if (j == i){ // word.substr(i, 1) has no matching + fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data()); + i++; + } + } + } + } + + return tokens; +} + +static std::vector parse_tokens_from_string(const std::string& input, char delimiter) { + std::vector output; + std::stringstream ss(input); + std::string token; + + while (std::getline(ss, token, delimiter)) { + output.push_back(std::stoi(token)); + } + + return output; +} + +static std::map> extract_tests_from_file(const std::string & fpath_test){ + if (fpath_test.empty()){ + fprintf(stderr, "%s : No test file found.\n", __func__); + return std::map>(); + } + + std::map> tests; + + auto fin = std::ifstream(fpath_test, std::ios_base::in); + const char * delimeter = " => "; + const char del_tok = ','; + std::string line; + while (std::getline(fin, line)) { + size_t delimiterPos = line.find(delimeter); + if (delimiterPos != std::string::npos) { + std::string text = line.substr(0, delimiterPos); + std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter)); + tests[text] = parse_tokens_from_string(s_tokens, del_tok); + } + } + return tests; +} + +void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){ + std::map> tests = extract_tests_from_file(fpath_test); + + size_t n_fails = 0; + + for (const auto & test : tests) { + std::vector tokens = gpt_tokenize(vocab, test.first); + + if (tokens != test.second){ + n_fails++; + + // print out failure cases + fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str()); + fprintf(stderr, "%s : tokens in hf: ", __func__); + for (const auto & t : test.second) { + fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); + } + fprintf(stderr, "\n"); + fprintf(stderr, "%s : tokens in ggml: ", __func__); + for (const auto & t : tokens) { + fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t); + } + fprintf(stderr, "\n"); + } + } + + fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size()); +} + +bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) { + printf("%s: loading vocab from '%s'\n", __func__, fname.c_str()); + + vocab.token_to_id = ::json_parse(fname); + + for (const auto & kv : vocab.token_to_id) { + vocab.id_to_token[kv.second] = kv.first; + } + + printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size()); + + // print the vocabulary + //for (auto kv : vocab.token_to_id) { + // printf("'%s' -> %d\n", kv.first.data(), kv.second); + //} + + return true; +} + +gpt_vocab::id gpt_sample_top_k_top_p( + const gpt_vocab & vocab, + const float * logits, + int top_k, + double top_p, + double temp, + std::mt19937 & rng) { + int n_logits = vocab.id_to_token.size(); + + std::vector> logits_id; + logits_id.reserve(n_logits); + + { + const double scale = 1.0/temp; + for (int i = 0; i < n_logits; ++i) { + logits_id.push_back(std::make_pair(logits[i]*scale, i)); + } + } + + // find the top K tokens + std::partial_sort( + logits_id.begin(), + logits_id.begin() + top_k, logits_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + logits_id.resize(top_k); + + double maxl = -INFINITY; + for (const auto & kv : logits_id) { + maxl = std::max(maxl, kv.first); + } + + // compute probs for the top K tokens + std::vector probs; + probs.reserve(logits_id.size()); + + double sum = 0.0; + for (const auto & kv : logits_id) { + double p = exp(kv.first - maxl); + probs.push_back(p); + sum += p; + } + + // normalize the probs + for (auto & p : probs) { + p /= sum; + } + + if (top_p < 1.0f) { + double cumsum = 0.0f; + for (int i = 0; i < top_k; i++) { + cumsum += probs[i]; + if (cumsum >= top_p) { + top_k = i + 1; + probs.resize(top_k); + logits_id.resize(top_k); + break; + } + } + + cumsum = 1.0/cumsum; + for (int i = 0; i < (int) probs.size(); i++) { + probs[i] *= cumsum; + } + } + + //printf("\n"); + //for (int i = 0; i < (int) probs.size(); i++) { + // printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); + //} + //exit(0); + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + int idx = dist(rng); + + return logits_id[idx].second; +} + +gpt_vocab::id gpt_sample_top_k_top_p_repeat( + const gpt_vocab & vocab, + const float * logits, + const int32_t * last_n_tokens_data, + size_t last_n_tokens_data_size, + int top_k, + double top_p, + double temp, + int repeat_last_n, + float repeat_penalty, + std::mt19937 & rng) { + + int n_logits = vocab.id_to_token.size(); + + const auto * plogits = logits; + + const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size); + + if (temp <= 0) { + // select the token with the highest logit directly + float max_logit = plogits[0]; + gpt_vocab::id max_id = 0; + + for (int i = 1; i < n_logits; ++i) { + if (plogits[i] > max_logit) { + max_logit = plogits[i]; + max_id = i; + } + } + return max_id; + } + + + std::vector> logits_id; + logits_id.reserve(n_logits); + + { + const float scale = 1.0f/temp; + for (int i = 0; i < n_logits; ++i) { + // repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858) + // credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main + if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) { + // if score < 0 then repetition penalty has to multiplied to reduce the previous token probability + if (plogits[i] < 0.0f) { + logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i)); + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i)); + } + } else { + logits_id.push_back(std::make_pair(plogits[i]*scale, i)); + } + } + } + + // find the top K tokens + std::partial_sort( + logits_id.begin(), + logits_id.begin() + top_k, logits_id.end(), + [](const std::pair & a, const std::pair & b) { + return a.first > b.first; + }); + + logits_id.resize(top_k); + + double maxl = -INFINITY; + for (const auto & kv : logits_id) { + maxl = std::max(maxl, kv.first); + } + + // compute probs for the top K tokens + std::vector probs; + probs.reserve(logits_id.size()); + + double sum = 0.0; + for (const auto & kv : logits_id) { + double p = exp(kv.first - maxl); + probs.push_back(p); + sum += p; + } + + // normalize the probs + for (auto & p : probs) { + p /= sum; + } + + if (top_p < 1.0f) { + double cumsum = 0.0f; + for (int i = 0; i < top_k; i++) { + cumsum += probs[i]; + if (cumsum >= top_p) { + top_k = i + 1; + probs.resize(top_k); + logits_id.resize(top_k); + break; + } + } + + cumsum = 1.0/cumsum; + for (int i = 0; i < (int) probs.size(); i++) { + probs[i] *= cumsum; + } + } + +// printf("\n"); +// for (int i = 0; i < (int) probs.size(); i++) { +// for (int i = 0; i < 10; i++) { +// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]); +// } + + std::discrete_distribution<> dist(probs.begin(), probs.end()); + int idx = dist(rng); + + return logits_id[idx].second; + +} + +void high_pass_filter(std::vector & data, float cutoff, float sample_rate) { + const float rc = 1.0f / (2.0f * M_PI * cutoff); + const float dt = 1.0f / sample_rate; + const float alpha = dt / (rc + dt); + + float y = data[0]; + + for (size_t i = 1; i < data.size(); i++) { + y = alpha * (y + data[i] - data[i - 1]); + data[i] = y; + } +} + +bool vad_simple(std::vector & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) { + const int n_samples = pcmf32.size(); + const int n_samples_last = (sample_rate * last_ms) / 1000; + + if (n_samples_last >= n_samples) { + // not enough samples - assume no speech + return false; + } + + if (freq_thold > 0.0f) { + high_pass_filter(pcmf32, freq_thold, sample_rate); + } + + float energy_all = 0.0f; + float energy_last = 0.0f; + + for (int i = 0; i < n_samples; i++) { + energy_all += fabsf(pcmf32[i]); + + if (i >= n_samples - n_samples_last) { + energy_last += fabsf(pcmf32[i]); + } + } + + energy_all /= n_samples; + energy_last /= n_samples_last; + + if (verbose) { + fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold); + } + + if (energy_last > vad_thold*energy_all) { + return false; + } + + return true; +} + +float similarity(const std::string & s0, const std::string & s1) { + const size_t len0 = s0.size() + 1; + const size_t len1 = s1.size() + 1; + + std::vector col(len1, 0); + std::vector prevCol(len1, 0); + + for (size_t i = 0; i < len1; i++) { + prevCol[i] = i; + } + + for (size_t i = 0; i < len0; i++) { + col[0] = i; + for (size_t j = 1; j < len1; j++) { + col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1)); + } + col.swap(prevCol); + } + + const float dist = prevCol[len1 - 1]; + + return 1.0f - (dist / std::max(s0.size(), s1.size())); +} + +bool is_file_exist(const char * filename) { + std::ifstream infile(filename); + return infile.good(); +} diff --git a/ggml/examples/common.h b/ggml/examples/common.h new file mode 100644 index 0000000..8f99df7 --- /dev/null +++ b/ggml/examples/common.h @@ -0,0 +1,322 @@ +// Various helper functions and utilities + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +// +// GPT CLI argument parsing +// + +struct gpt_params { + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); + int32_t n_predict = 200; // new tokens to predict + int32_t n_parallel = 1; // number of parallel streams + int32_t n_batch = 32; // batch size for prompt processing + int32_t n_ctx = 2048; // context size (this is the KV cache max size) + int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU + + bool ignore_eos = false; // ignore EOS token when generating text + + // sampling parameters + int32_t top_k = 40; + float top_p = 0.9f; + float temp = 0.9f; + int32_t repeat_last_n = 64; + float repeat_penalty = 1.00f; + + std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path + std::string prompt = ""; + std::string token_test = ""; + + bool interactive = false; + int32_t interactive_port = -1; +}; + +bool gpt_params_parse(int argc, char ** argv, gpt_params & params); + +void gpt_print_usage(int argc, char ** argv, const gpt_params & params); + +std::string gpt_random_prompt(std::mt19937 & rng); + +// +// Vocab utils +// + +std::string trim(const std::string & s); + +std::string replace( + const std::string & s, + const std::string & from, + const std::string & to); + +struct gpt_vocab { + using id = int32_t; + using token = std::string; + + std::map token_to_id; + std::map id_to_token; + std::vector special_tokens; + + void add_special_token(const std::string & token); +}; + +// poor-man's JSON parsing +std::map json_parse(const std::string & fname); + +std::string convert_to_utf8(const std::wstring & input); + +std::wstring convert_to_wstring(const std::string & input); + +void gpt_split_words(std::string str, std::vector& words); + +// split text into tokens +// +// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53 +// +// Regex (Python): +// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" +// +// Regex (C++): +// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)" +// +std::vector gpt_tokenize(const gpt_vocab & vocab, const std::string & text); + +// test outputs of gpt_tokenize +// +// - compare with tokens generated by the huggingface tokenizer +// - test cases are chosen based on the model's main language (under 'prompt' directory) +// - if all sentences are tokenized identically, print 'All tests passed.' +// - otherwise, print sentence, huggingface tokens, ggml tokens +// +void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test); + +// load the tokens from encoder.json +bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab); + +// sample next token given probabilities for each embedding +// +// - consider only the top K tokens +// - from them, consider only the top tokens with cumulative probability > P +// +// TODO: not sure if this implementation is correct +// TODO: temperature is not implemented +// +gpt_vocab::id gpt_sample_top_k_top_p( + const gpt_vocab & vocab, + const float * logits, + int top_k, + double top_p, + double temp, + std::mt19937 & rng); + +gpt_vocab::id gpt_sample_top_k_top_p_repeat( + const gpt_vocab & vocab, + const float * logits, + const int32_t * last_n_tokens_data, + size_t last_n_tokens_data_size, + int top_k, + double top_p, + double temp, + int repeat_last_n, + float repeat_penalty, + std::mt19937 & rng); + +// +// Audio utils +// + +// Write PCM data into WAV audio file +class wav_writer { +private: + std::ofstream file; + uint32_t dataSize = 0; + std::string wav_filename; + + bool write_header(const uint32_t sample_rate, + const uint16_t bits_per_sample, + const uint16_t channels) { + + file.write("RIFF", 4); + file.write("\0\0\0\0", 4); // Placeholder for file size + file.write("WAVE", 4); + file.write("fmt ", 4); + + const uint32_t sub_chunk_size = 16; + const uint16_t audio_format = 1; // PCM format + const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8; + const uint16_t block_align = channels * bits_per_sample / 8; + + file.write(reinterpret_cast(&sub_chunk_size), 4); + file.write(reinterpret_cast(&audio_format), 2); + file.write(reinterpret_cast(&channels), 2); + file.write(reinterpret_cast(&sample_rate), 4); + file.write(reinterpret_cast(&byte_rate), 4); + file.write(reinterpret_cast(&block_align), 2); + file.write(reinterpret_cast(&bits_per_sample), 2); + file.write("data", 4); + file.write("\0\0\0\0", 4); // Placeholder for data size + + return true; + } + + // It is assumed that PCM data is normalized to a range from -1 to 1 + bool write_audio(const float * data, size_t length) { + for (size_t i = 0; i < length; ++i) { + const int16_t intSample = int16_t(data[i] * 32767); + file.write(reinterpret_cast(&intSample), sizeof(int16_t)); + dataSize += sizeof(int16_t); + } + if (file.is_open()) { + file.seekp(4, std::ios::beg); + uint32_t fileSize = 36 + dataSize; + file.write(reinterpret_cast(&fileSize), 4); + file.seekp(40, std::ios::beg); + file.write(reinterpret_cast(&dataSize), 4); + file.seekp(0, std::ios::end); + } + return true; + } + + bool open_wav(const std::string & filename) { + if (filename != wav_filename) { + if (file.is_open()) { + file.close(); + } + } + if (!file.is_open()) { + file.open(filename, std::ios::binary); + wav_filename = filename; + dataSize = 0; + } + return file.is_open(); + } + +public: + bool open(const std::string & filename, + const uint32_t sample_rate, + const uint16_t bits_per_sample, + const uint16_t channels) { + + if (open_wav(filename)) { + write_header(sample_rate, bits_per_sample, channels); + } else { + return false; + } + + return true; + } + + bool close() { + file.close(); + return true; + } + + bool write(const float * data, size_t length) { + return write_audio(data, length); + } + + ~wav_writer() { + if (file.is_open()) { + file.close(); + } + } +}; + + +// Apply a high-pass frequency filter to PCM audio +// Suppresses frequencies below cutoff Hz +void high_pass_filter( + std::vector & data, + float cutoff, + float sample_rate); + +// Basic voice activity detection (VAD) using audio energy adaptive threshold +bool vad_simple( + std::vector & pcmf32, + int sample_rate, + int last_ms, + float vad_thold, + float freq_thold, + bool verbose); + +// compute similarity between two strings using Levenshtein distance +float similarity(const std::string & s0, const std::string & s1); + +// +// Terminal utils +// + +#define SQR(X) ((X) * (X)) +#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40 + +/** + * Quantizes 24-bit RGB to xterm256 code range [16,256). + */ +static int rgb2xterm256(int r, int g, int b) { + unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377}; + int av, ir, ig, ib, il, qr, qg, qb, ql; + av = r * .299 + g * .587 + b * .114 + .5; + ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8; + qr = cube[(ir = UNCUBE(r))]; + qg = cube[(ig = UNCUBE(g))]; + qb = cube[(ib = UNCUBE(b))]; + if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <= + SQR(ql - r) + SQR(ql - g) + SQR(ql - b)) + return ir * 36 + ig * 6 + ib + 020; + return il + 0350; +} + +static std::string set_xterm256_foreground(int r, int g, int b) { + int x = rgb2xterm256(r, g, b); + std::ostringstream oss; + oss << "\033[38;5;" << x << "m"; + return oss.str(); +} + +// Lowest is red, middle is yellow, highest is green. Color scheme from +// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault +const std::vector k_colors = { + set_xterm256_foreground(220, 5, 12), + set_xterm256_foreground(232, 96, 28), + set_xterm256_foreground(241, 147, 45), + set_xterm256_foreground(246, 193, 65), + set_xterm256_foreground(247, 240, 86), + set_xterm256_foreground(144, 201, 135), + set_xterm256_foreground( 78, 178, 101), +}; + +// ANSI formatting codes +static std::string set_inverse() { + return "\033[7m"; +} + +static std::string set_underline() { + return "\033[4m"; +} + +static std::string set_dim() { + return "\033[2m"; +} + +// Style scheme for different confidence levels +const std::vector k_styles = { + set_inverse(), // Low confidence - inverse (highlighted) + set_underline(), // Medium confidence - underlined + set_dim(), // High confidence - dim +}; + +// +// Other utils +// + +// check if file exists using ifstream +bool is_file_exist(const char * filename); diff --git a/ggml/examples/gpt-2/CMakeLists.txt b/ggml/examples/gpt-2/CMakeLists.txt new file mode 100644 index 0000000..686a992 --- /dev/null +++ b/ggml/examples/gpt-2/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# gpt-2 + +set(TEST_TARGET gpt-2-ctx) +add_executable(${TEST_TARGET} main-ctx.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +set(TEST_TARGET gpt-2-alloc) +add_executable(${TEST_TARGET} main-alloc.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +set(TEST_TARGET gpt-2-backend) +add_executable(${TEST_TARGET} main-backend.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +set(TEST_TARGET gpt-2-sched) +add_executable(${TEST_TARGET} main-sched.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +# +# gpt-2-quantize + +set(TEST_TARGET gpt-2-quantize) +add_executable(${TEST_TARGET} quantize.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +# +# gpt-2-batched + +set(TEST_TARGET gpt-2-batched) +add_executable(${TEST_TARGET} main-batched.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) diff --git a/ggml/examples/gpt-2/README.md b/ggml/examples/gpt-2/README.md new file mode 100644 index 0000000..1057162 --- /dev/null +++ b/ggml/examples/gpt-2/README.md @@ -0,0 +1,225 @@ +# gpt-2 + +This is a C++ example running GPT-2 inference using the [ggml](https://github.com/ggerganov/ggml) library. + +The program runs on the CPU - no video card is required. + +The [Cerebras-GPT](https://huggingface.co/cerebras) models are also supported. + +The example supports the following GPT-2 models: + +| Model | Description | Disk Size | +| --- | --- | --- | +| 117M | Small model | 240 MB | +| 345M | Medium model | 680 MB | +| 774M | Large model | 1.5 GB | +| 1558M | XL model | 3.0 GB | + +Sample performance on MacBook M1 Pro: + +| Model | Size | Time / Token | +| --- | --- | --- | +| GPT-2 | 117M | 5 ms | +| GPT-2 | 345M | 12 ms | +| GPT-2 | 774M | 23 ms | +| GPT-2 | 1558M | 42 ms | + +*TODO: add tables for Cerebras-GPT models* + +Sample output: + +```bash +$ ./bin/gpt-2 -h +usage: ./bin/gpt-2 [options] + +options: + -h, --help show this help message and exit + -s SEED, --seed SEED RNG seed (default: -1) + -t N, --threads N number of threads to use during computation (default: 8) + -p PROMPT, --prompt PROMPT + prompt to start generation with (default: random) + -n N, --n_predict N number of tokens to predict (default: 200) + --top_k N top-k sampling (default: 40) + --top_p N top-p sampling (default: 0.9) + --temp N temperature (default: 1.0) + -b N, --batch_size N batch size for prompt processing (default: 8) + -m FNAME, --model FNAME + model path (default: models/gpt-2-117M/ggml-model.bin) + +$ ./bin/gpt-2 +gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin' +gpt2_model_load: n_vocab = 50257 +gpt2_model_load: n_ctx = 1024 +gpt2_model_load: n_embd = 768 +gpt2_model_load: n_head = 12 +gpt2_model_load: n_layer = 12 +gpt2_model_load: f16 = 1 +gpt2_model_load: ggml ctx size = 311.12 MB +gpt2_model_load: memory size = 72.00 MB, n_mem = 12288 +gpt2_model_load: model size = 239.08 MB +main: number of tokens in prompt = 1 + +So this is going to be the end of the line for us. + +If the Dolphins continue to do their business, it's possible that the team could make a bid to bring in new defensive coordinator Scott Linehan. + +Linehan's job is a little daunting, but he's a great coach and an excellent coach. I don't believe we're going to make the playoffs. + +We're going to have to work hard to keep our heads down and get ready to go.<|endoftext|> + +main: mem per token = 2048612 bytes +main: load time = 106.32 ms +main: sample time = 7.10 ms +main: predict time = 506.40 ms / 5.06 ms per token +main: total time = 629.84 ms +``` + +## Downloading and converting the original models (GPT-2) + +You can download the original model files using the [download-model.sh](download-model.sh) Bash script. The models are +in Tensorflow format, so in order to use them with ggml, you need to convert them to appropriate format. This is done +via the [convert-ckpt-to-ggml.py](convert-ckpt-to-ggml.py) python script. + +Here is the entire process for the GPT-2 117M model (download from official site + conversion): + +```bash +cd ggml/build +../examples/gpt-2/download-model.sh 117M + +Downloading model 117M ... +models/gpt-2-117M/checkpoint 100%[=============================>] 77 --.-KB/s in 0s +models/gpt-2-117M/encoder.json 100%[=============================>] 1018K 1.20MB/s in 0.8s +models/gpt-2-117M/hparams.json 100%[=============================>] 90 --.-KB/s in 0s +models/gpt-2-117M/model.ckpt.data-00000-of-00001 100%[=============================>] 474.70M 1.21MB/s in 8m 39s +models/gpt-2-117M/model.ckpt.index 100%[=============================>] 5.09K --.-KB/s in 0s +models/gpt-2-117M/model.ckpt.meta 100%[=============================>] 460.11K 806KB/s in 0.6s +models/gpt-2-117M/vocab.bpe 100%[=============================>] 445.62K 799KB/s in 0.6s +Done! Model '117M' saved in 'models/gpt-2-117M/' + +Run the convert-ckpt-to-ggml.py script to convert the model to ggml format. + + python /Users/john/ggml/examples/gpt-2/convert-ckpt-to-ggml.py models/gpt-2-117M/ 1 + +``` + +This conversion requires that you have python and Tensorflow installed on your computer. Still, if you want to avoid +this, you can download the already converted ggml models as described below. + +## Downloading and converting the original models (Cerebras-GPT) + +Clone the respective repository from here: https://huggingface.co/cerebras + +Use the [convert-cerebras-to-ggml.py](convert-cerebras-to-ggml.py) script to convert the model to `ggml` format: + +```bash +cd ggml/build +git clone https://huggingface.co/cerebras/Cerebras-GPT-111M models/ +python ../examples/gpt-2/convert-cerebras-to-ggml.py models/Cerebras-GPT-111M/ + +``` + +## Downloading the ggml model directly (GPT-2) + +For convenience, I will be hosting the converted ggml model files in order to make it easier to run the examples. This +way, you can directly download a single binary file and start using it. No python or Tensorflow is required. + +Here is how to get the 117M ggml model: + +```bash +cd ggml/build +../examples/gpt-2/download-ggml-model.sh 117M + +Downloading ggml model 117M ... +models/gpt-2-117M/ggml-model.bin 100%[===============================>] 239.58M 8.52MB/s in 28s +Done! Model '117M' saved in 'models/gpt-2-117M/ggml-model.bin' +You can now use it like this: + + $ ./bin/gpt-2 -m models/gpt-2-117M/ggml-model.bin -p "This is an example" + +``` + +At some point, I might decide to stop hosting these models. So in that case, simply revert to the manual process above. + +## Quantizing the models + +You can also try to quantize the `ggml` models via 4-bit integer quantization. +Keep in mind that for smaller models, this will render them completely useless. +You generally want to quantize larger models. + +```bash +# quantize GPT-2 F16 to Q4_0 (faster but less precise) +./bin/gpt-2-quantize models/gpt-2-1558M/ggml-model-f16.bin models/gpt-2-1558M/ggml-model-q4_0.bin 2 +./bin/gpt-2 -m models/gpt-2-1558M/ggml-model-q4_0.bin -p "This is an example" + +# quantize Cerebras F16 to Q4_1 (slower but more precise) +./bin/gpt-2-quantize models/Cerebras-GPT-6.7B/ggml-model-f16.bin models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin 3 +./bin/gpt-2 -m models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin -p "This is an example" + +``` + +## Batched generation example + +You can try the batched generation from a given prompt using the gpt-2-batched binary. + +Sample output: + +```bash +$ gpt-2-batched -np 5 -m models/gpt-2-117M/ggml-model.bin -p "Hello my name is" -n 50 + +main: seed = 1697037431 +gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin' +gpt2_model_load: n_vocab = 50257 +gpt2_model_load: n_ctx = 1024 +gpt2_model_load: n_embd = 768 +gpt2_model_load: n_head = 12 +gpt2_model_load: n_layer = 12 +gpt2_model_load: ftype = 1 +gpt2_model_load: qntvr = 0 +gpt2_model_load: ggml tensor size = 320 bytes +gpt2_model_load: backend buffer size = 312.72 MB +ggml_init_cublas: found 1 CUDA devices: + Device 0: NVIDIA GeForce GTX 1660, compute capability 7.5 +gpt2_model_load: using CPU backend +gpt2_model_load: memory size = 72.00 MB, n_mem = 12288 +gpt2_model_load: model size = 239.08 MB +extract_tests_from_file : No test file found. +test_gpt_tokenizer : 0 tests failed out of 0 tests. +main: compute buffer size: 3.26 MB + + +main: generating 5 sequences ... +main: prompt: 'Hello my name is' +main: number of tokens in prompt = 4, first 8 tokens: 15496 616 1438 318 + + +sequence 0: + +Hello my name is John. You can call me any way you want, if you want, but for my very first date, I will be on the phone with you. We're both in our early 20s, but I feel like it's all + +sequence 1: + +Hello my name is Robert, and I want to say that we're proud to have your company here on the world's largest platform for sharing your stories with us. This is a huge opportunity for our community. We have hundreds of people on this team and + +sequence 2: + +Hello my name is Jack. I'm the one who created you. + +Jack is a boy with a big smile and a big heart. He is a handsome guy. He loves the outdoors and loves the people he meets. He wants to be a + +sequence 3: + +Hello my name is John. I am a Canadian citizen with a large number of family in Quebec and I am interested in studying. My aim is to take up a post in the Journal of the International Academy of Sciences of Canada which I am currently finishing. + +sequence 4: + +Hello my name is Dan. I am an entrepreneur. I am a great father. I am a great husband. I am a great husband. I am a great dad. And I am a great husband. + +I love my life. I love + + + +main: load time = 880.80 ms +main: sample time = 91.43 ms +main: predict time = 2518.29 ms +main: total time = 3544.32 ms +``` diff --git a/ggml/examples/gpt-2/convert-cerebras-to-ggml.py b/ggml/examples/gpt-2/convert-cerebras-to-ggml.py new file mode 100644 index 0000000..6057f81 --- /dev/null +++ b/ggml/examples/gpt-2/convert-cerebras-to-ggml.py @@ -0,0 +1,183 @@ +# Convert Cerebras models to ggml format +# +# ref: https://www.cerebras.net/blog/cerebras-gpt-a-family-of-open-compute-efficient-large-language-models/ +# + +import sys +import struct +import json +import torch +import numpy as np +import re + +from transformers import AutoModelForCausalLM + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +if len(sys.argv) < 2: + print("Usage: convert-cerebras-to-ggml.py dir-model [use-f32]\n") + sys.exit(1) + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model-f16.bin" + +with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f: + encoder = json.load(f) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# use 16-bit or 32-bit floats +use_f16 = True +if len(sys.argv) > 2: + use_f16 = False + fname_out = sys.argv[1] + "/ggml-model-f32.bin" + +model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True) +#print (model) + +list_vars = model.state_dict() +#print (list_vars) + +print(hparams) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["vocab_size"])) +fout.write(struct.pack("i", hparams["n_positions"])) +fout.write(struct.pack("i", hparams["n_embd"])) +fout.write(struct.pack("i", hparams["n_head"])) +fout.write(struct.pack("i", hparams["n_layer"])) +fout.write(struct.pack("i", use_f16)) + +byte_encoder = bytes_to_unicode() +byte_decoder = {v:k for k, v in byte_encoder.items()} + +fout.write(struct.pack("i", len(encoder))) + +for key in encoder: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Processing variable: " + name + " with shape: ", data.shape) + + # rename headers to keep compatibility + if name == "transformer.ln_f.weight": + name = "model/ln_f/g" + elif name == "transformer.ln_f.bias": + name = "model/ln_f/b" + elif name == "transformer.wte.weight": + name = "model/wte" + elif name == "transformer.wpe.weight": + name = "model/wpe" + elif name == "lm_head.weight": + name = "model/lm_head" + elif re.match(r"transformer.h\.\d+\.ln_1\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_1/g" + elif re.match(r"transformer.h\.\d+\.ln_1\.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_1/b" + elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_attn/w" + elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_attn/b" + elif re.match(r"transformer.h\.\d+\.attn\.c_proj\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_proj/w" + elif re.match(r"transformer.h.\d+.attn.c_proj.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_proj/b" + elif re.match(r"transformer.h.\d+.ln_2.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_2/g" + elif re.match(r"transformer.h.\d+.ln_2.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_2/b" + elif re.match(r"transformer.h.\d+.mlp.c_fc.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_fc/w" + elif re.match(r"transformer.h.\d+.mlp.c_fc.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_fc/b" + elif re.match(r"transformer.h.\d+.mlp.c_proj.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_proj/w" + elif re.match(r"transformer.h.\d+.mlp.c_proj.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_proj/b" + else: + print("Unrecognized variable name. %s", name) + + # we don't need these + if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"): + print(" Skipping variable: " + name) + continue + + n_dims = len(data.shape); + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype = 0; + if use_f16: + if (name == "model/wte" or name == "model/lm_head" or name[-2:] == "/g" or name[-2:] == "/w") and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype = 0 + + # for efficiency - transpose the projection matrices + # "model/h.*/attn/c_attn/w" + # "model/h.*/attn/c_proj/w" + # "model/h.*/mlp/c_fc/w" + # "model/h.*/mlp/c_proj/w" + if name[-14:] == "/attn/c_attn/w" or \ + name[-14:] == "/attn/c_proj/w" or \ + name[-11:] == "/mlp/c_fc/w" or \ + name[-13:] == "/mlp/c_proj/w": + print(" Transposing") + data = data.transpose() + + # header + str = name.encode('utf-8') + fout.write(struct.pack("iii", n_dims, len(str), ftype)) + for i in range(n_dims): + fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) + fout.write(str); + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/ggml/examples/gpt-2/convert-ckpt-to-ggml.py b/ggml/examples/gpt-2/convert-ckpt-to-ggml.py new file mode 100644 index 0000000..9113141 --- /dev/null +++ b/ggml/examples/gpt-2/convert-ckpt-to-ggml.py @@ -0,0 +1,159 @@ +# Convert a model checkpoint to a ggml compatible file +# +# Load the model using TensorFlow. +# Iterate over all variables and write them to a binary file. +# +# For each variable, write the following: +# - Number of dimensions (int) +# - Name length (int) +# - Dimensions (int[n_dims]) +# - Name (char[name_length]) +# - Data (float[n_dims]) +# +# By default, the bigger matrices are converted to 16-bit floats. +# This can be disabled by adding the "use-f32" CLI argument. +# +# At the start of the ggml file we write the model parameters +# and vocabulary. +# + +import sys +import json +import struct +import numpy as np +import tensorflow as tf + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +# helper method to convert a numpy array to different float types +def convert_to_ftype(data, ftype): + # fp16 + if ftype == 1: + return data.astype(np.float16) + + assert False, "Invalid ftype: " + str(ftype) + +if len(sys.argv) < 3: + print("Usage: convert-ckpt-to-ggml.py dir-model ftype\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + +with open(dir_model + "/encoder.json", "r", encoding="utf-8") as f: + encoder = json.load(f) + +with open(dir_model + "/hparams.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# possible data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 2: + ftype = int(sys.argv[2]) + if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin" + +list_vars = tf.train.list_variables(dir_model) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["n_vocab"])) +fout.write(struct.pack("i", hparams["n_ctx"])) +fout.write(struct.pack("i", hparams["n_embd"])) +fout.write(struct.pack("i", hparams["n_head"])) +fout.write(struct.pack("i", hparams["n_layer"])) +fout.write(struct.pack("i", ftype)) + +byte_encoder = bytes_to_unicode() +byte_decoder = {v:k for k, v in byte_encoder.items()} + +fout.write(struct.pack("i", len(encoder))) + +for key in encoder: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for name, shape in list_vars: + print("Processing variable: " + name + " with shape: ", shape) + + data = tf.train.load_variable(dir_model, name).squeeze() + n_dims = len(data.shape); + + # for efficiency - transpose the projection matrices + # "model/h.*/attn/c_attn/w" + # "model/h.*/attn/c_proj/w" + # "model/h.*/mlp/c_fc/w" + # "model/h.*/mlp/c_proj/w" + if name[-14:] == "/attn/c_attn/w" or \ + name[-14:] == "/attn/c_proj/w" or \ + name[-11:] == "/mlp/c_fc/w" or \ + name[-13:] == "/mlp/c_proj/w": + print(" Transposing") + data = data.transpose() + + dshape = data.shape + + ftype_cur = 0 + if ftype != 0: + # match name: + # "model/wte" + # "model/h.*/attn/c_attn/w" + # "model/h.*/attn/c_proj/w" + # "model/h.*/mlp/c_fc/w" + # "model/h.*/mlp/c_proj/w" + if name == "model/wte" or name[-2:] == "/w": + print(" Converting to " + ftype_str[ftype]) + data = convert_to_ftype(data, ftype) + ftype_cur = ftype + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + + # header + str = name.encode('utf-8') + fout.write(struct.pack("iii", n_dims, len(str), ftype_cur)) + for i in range(n_dims): + fout.write(struct.pack("i", dshape[n_dims - 1 - i])) + fout.write(str); + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/ggml/examples/gpt-2/convert-h5-to-ggml.py b/ggml/examples/gpt-2/convert-h5-to-ggml.py new file mode 100644 index 0000000..6a2b865 --- /dev/null +++ b/ggml/examples/gpt-2/convert-h5-to-ggml.py @@ -0,0 +1,195 @@ +# Convert GPT-2 h5 transformer model to ggml format +# +# Load the model using GPT2Model. +# Iterate over all variables and write them to a binary file. +# +# For each variable, write the following: +# - Number of dimensions (int) +# - Name length (int) +# - Dimensions (int[n_dims]) +# - Name (char[name_length]) +# - Data (float[n_dims]) +# +# By default, the bigger matrices are converted to 16-bit floats. +# This can be disabled by adding the "use-f32" CLI argument. +# +# At the start of the ggml file we write the model parameters +# and vocabulary. +# + +import sys +import struct +import json +import numpy as np +import re + +from transformers import GPT2Model + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +if len(sys.argv) < 2: + print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n") + sys.exit(1) + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + +with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f: + encoder = json.load(f) + +with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f: + encoder_added = json.load(f) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# use 16-bit or 32-bit floats +use_f16 = True +if len(sys.argv) > 2: + use_f16 = False + fname_out = sys.argv[1] + "/ggml-model-f32.bin" + +model = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True) +#print (model) + +list_vars = model.state_dict() +#print (list_vars) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["vocab_size"])) +fout.write(struct.pack("i", hparams["n_positions"])) +fout.write(struct.pack("i", hparams["n_embd"])) +fout.write(struct.pack("i", hparams["n_head"])) +fout.write(struct.pack("i", hparams["n_layer"])) +#fout.write(struct.pack("i", hparams["rotary_dim"])) +fout.write(struct.pack("i", use_f16)) + +byte_encoder = bytes_to_unicode() +byte_decoder = {v:k for k, v in byte_encoder.items()} + +fout.write(struct.pack("i", len(encoder) + len(encoder_added))) + +for key in encoder: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for key in encoder_added: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Processing variable: " + name + " with shape: ", data.shape) + + # we don't need these + if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"): + print(" Skipping variable: " + name) + continue + + n_dims = len(data.shape); + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype = 0; + if use_f16: + if name[-7:] == ".weight" and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype = 0 + + # for efficiency - transpose these matrices: + # "transformer.h.*.mlp.c_proj.weight + if name.endswith(".mlp.c_proj.weight"): + print(" Transposing") + data = data.transpose() + + # rename headers to keep compatibility + if name == "ln_f.weight": + name = "model/ln_f/g" + elif name == "ln_f.bias": + name = "model/ln_f/b" + elif name == "wte.weight": + name = "model/wte" + elif name == "wpe.weight": + name = "model/wpe" + elif re.match(r"h\.\d+\.ln_1\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_1/g" + elif re.match(r"h\.\d+\.ln_1\.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_1/b" + elif re.match(r"h\.\d+\.attn\.c_attn\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_attn/w" + elif re.match(r"h\.\d+\.attn\.c_attn\.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_attn/b" + elif re.match(r"h\.\d+\.attn\.c_proj\.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_proj/w" + elif re.match(r"h.\d+.attn.c_proj.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/attn/c_proj/b" + elif re.match(r"h.\d+.ln_2.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_2/g" + elif re.match(r"h.\d+.ln_2.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/ln_2/b" + elif re.match(r"h.\d+.mlp.c_fc.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_fc/w" + elif re.match(r"h.\d+.mlp.c_fc.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_fc/b" + elif re.match(r"h.\d+.mlp.c_proj.weight", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_proj/w" + elif re.match(r"h.\d+.mlp.c_proj.bias", name): + i = re.findall("\d+", name)[0] + name = f"model/h{i}/mlp/c_proj/b" + else: + print("Unrecognized variable name. %s", name) + + str = name.encode('utf-8') + + fout.write(struct.pack("iii", n_dims, len(str), ftype)) + for i in range(n_dims): + fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) + fout.write(str); + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/ggml/examples/gpt-2/download-ggml-model.sh b/ggml/examples/gpt-2/download-ggml-model.sh new file mode 100755 index 0000000..3aae015 --- /dev/null +++ b/ggml/examples/gpt-2/download-ggml-model.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# This script downloads GPT-2 model files that have already been converted to ggml format. +# This way you don't have to convert them yourself. +# +# If you want to download the original GPT-2 model files, use the "download-model.sh" script instead. + +#src="https://ggml.ggerganov.com" +#pfx="ggml-model-gpt-2" + +src="https://huggingface.co/ggerganov/ggml" +pfx="resolve/main/ggml-model-gpt-2" + +ggml_path=$(dirname $(realpath $0)) + +# GPT-2 models +models=( "117M" "345M" "774M" "1558M" ) + +# list available models +function list_models { + printf "\n" + printf " Available models:" + for model in "${models[@]}"; do + printf " $model" + done + printf "\n\n" +} + +if [ "$#" -ne 1 ]; then + printf "Usage: $0 \n" + list_models + + exit 1 +fi + +model=$1 + +if [[ ! " ${models[@]} " =~ " ${model} " ]]; then + printf "Invalid model: $model\n" + list_models + + exit 1 +fi + +# download ggml model + +printf "Downloading ggml model $model ...\n" + +mkdir -p models/gpt-2-$model + +if [ -x "$(command -v wget)" ]; then + wget --quiet --show-progress -O models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin +elif [ -x "$(command -v curl)" ]; then + curl -L --output models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin +else + printf "Either wget or curl is required to download models.\n" + exit 1 +fi + +if [ $? -ne 0 ]; then + printf "Failed to download ggml model $model \n" + printf "Please try again later or download the original GPT-2 model files and convert them yourself.\n" + exit 1 +fi + +printf "Done! Model '$model' saved in 'models/gpt-2-$model/ggml-model.bin'\n" +printf "You can now use it like this:\n\n" +printf " $ ./bin/gpt-2 -m models/gpt-2-$model/ggml-model.bin -p \"This is an example\"\n" +printf "\n" diff --git a/ggml/examples/gpt-2/download-model.sh b/ggml/examples/gpt-2/download-model.sh new file mode 100755 index 0000000..f0c62f4 --- /dev/null +++ b/ggml/examples/gpt-2/download-model.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +ggml_path=$(dirname $(realpath $0)) + +# GPT-2 models +models=( "117M" "345M" "774M" "1558M" ) + +# list available models +function list_models { + printf "\n" + printf " Available models:" + for model in "${models[@]}"; do + printf " $model" + done + printf "\n\n" +} + +if [ "$#" -ne 1 ]; then + printf "Usage: $0 \n" + list_models + + exit 1 +fi + +model=$1 + +if [[ ! " ${models[@]} " =~ " ${model} " ]]; then + printf "Invalid model: $model\n" + list_models + + exit 1 +fi + +# download model + +printf "Downloading model $model ...\n" + +mkdir -p models/gpt-2-$model + +for file in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do + wget --quiet --show-progress -O models/gpt-2-$model/$file https://openaipublic.blob.core.windows.net/gpt-2/models/$model/$file +done + +printf "Done! Model '$model' saved in 'models/gpt-2-$model/'\n\n" +printf "Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.\n" +printf "\n" +printf " python $ggml_path/convert-ckpt-to-ggml.py models/gpt-2-$model/\n" +printf "\n" diff --git a/ggml/examples/gpt-2/main-alloc.cpp b/ggml/examples/gpt-2/main-alloc.cpp new file mode 100644 index 0000000..172d3eb --- /dev/null +++ b/ggml/examples/gpt-2/main-alloc.cpp @@ -0,0 +1,880 @@ +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gpt2_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // mlp + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt2_model { + gpt2_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // token embedding + struct ggml_tensor * wpe; // position embedding + struct ggml_tensor * lm_head; // language model head + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct ggml_context * ctx_w; + std::map tensors; +}; + +// load the model's weights from a file +bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) { + printf("%s: loading model from '%s'\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx_w; + + size_t ctx_size = 0; + + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b + + ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte + ctx_size += ggml_row_size(GGML_TYPE_F32 , n_ctx*n_embd); // wpe + ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head + + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b + + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b + + ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b + + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b + + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v + + ctx_size += (6 + 12*n_layer)*512; // object overhead + + printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor)); + printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); + } + + // create the ggml context + { + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + model.ctx_w = ggml_init(params); + if (!model.ctx_w) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // prepare memory for the weights + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); + model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + // map by name + model.tensors["model/ln_f/g"] = model.ln_f_g; + model.tensors["model/ln_f/b"] = model.ln_f_b; + + model.tensors["model/wte"] = model.wte; + model.tensors["model/wpe"] = model.wpe; + model.tensors["model/lm_head"] = model.lm_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd); + layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; + model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + + model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; + model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + } + } + + // key + value memory + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + + const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); + + printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + } + + // load weights + { + size_t total_size = 0; + + bool has_lm_head = false; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + + // GPT-2 models share the WTE tensor as the LM head + if (name == "model/wte" && has_lm_head == false) { + memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor)); + } + + if (name == "model/lm_head") { + has_lm_head = true; + } + + total_size += ggml_nbytes(tensor); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0); + } + + fin.close(); + + return true; +} + +// build the computation graph +struct ggml_cgraph * gpt2_graph( + const gpt2_model & model, + const int n_past, + const int n_tokens) { + const int N = n_tokens; + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + + // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data + static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(); + static std::vector buf(buf_size); + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf.data(), + /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph() + }; + + struct ggml_context * ctx = ggml_init(params); + + struct ggml_cgraph * gf = ggml_new_graph(ctx); + + struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); + // at this point, the tensor data is not allocated yet and cannot be set + // we will find the tensor after the graph is allocated by its name, and set the data then + ggml_set_name(embd, "embd"); + // setting a tensor as an input will ensure that it is allocated at the beginning of the graph + // this is important to ensure that the input tensors are not overwritten before they are used + ggml_set_input(embd); + + struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); + ggml_set_name(position, "position"); + ggml_set_input(position); + + // wte + wpe + struct ggml_tensor * inpL = + ggml_add(ctx, + ggml_get_rows(ctx, model.wte, embd), + ggml_get_rows(ctx, model.wpe, position)); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + // [ 768, N] + cur = ggml_norm(ctx, inpL, hparams.eps); + + // cur = ln_1_g*cur + ln_1_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + ggml_repeat(ctx, model.layers[il].ln_1_g, cur), + cur), + ggml_repeat(ctx, model.layers[il].ln_1_b, cur)); + } + + // attn + // [2304, 768] - model.layers[il].c_attn_attn_w + // [2304, 1] - model.layers[il].c_attn_attn_b + // [ 768, N] - cur (in) + // [2304, N] - cur (out) + // + // cur = attn_w*cur + attn_b + // [2304, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_attn_w, + cur); + + cur = ggml_add(ctx, + ggml_repeat(ctx, model.layers[il].c_attn_attn_b, cur), + cur); + } + + // self-attention + { + struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd); + struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd); + struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd); + + // store key and value to memory + if (N >= 1) { + struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + // [64, N, 12] + struct ggml_tensor * Q = + ggml_permute(ctx, + ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N), + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + // [64, n_past + N, 12] + struct ggml_tensor * K = + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + + // GG: flash attention + //struct ggml_tensor * V = + // ggml_cpy(ctx0, + // ggml_permute(ctx0, + // ggml_reshape_3d(ctx0, + // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + // n_embd/n_head, n_head, n_past + N), + // 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head)); + + //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true); + + // K * Q + // [n_past + N, N, 12] + struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_scaled = + ggml_scale(ctx, + KQ, + 1.0f/sqrtf(float(n_embd)/n_head)); + + // KQ_masked = mask_past(KQ_scaled) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + + // KQ = soft_max(KQ_masked) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + // [n_past + N, 64, 12] + struct ggml_tensor * V_trans = + ggml_cont_3d(ctx, + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + n_embd/n_head, n_head, n_past + N), + 1, 2, 0, 3), + n_past + N, n_embd/n_head, n_head); + + // KQV = transpose(V) * KQ_soft_max + // [64, N, 12] + struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + // [64, 12, N] + struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + // [768, N] + cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N); + } + + // projection + // [ 768, 768] - model.layers[il].c_attn_proj_w + // [ 768, 1] - model.layers[il].c_attn_proj_b + // [ 768, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_proj_w, + cur); + + cur = ggml_add(ctx, + ggml_repeat(ctx, model.layers[il].c_attn_proj_b, cur), + cur); + } + + // add the input + cur = ggml_add(ctx, cur, inpL); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx, inpFF, hparams.eps); + + // cur = ln_2_g*cur + ln_2_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + ggml_repeat(ctx, model.layers[il].ln_2_g, cur), + cur), + ggml_repeat(ctx, model.layers[il].ln_2_b, cur)); + } + + // fully connected + // [3072, 768] - model.layers[il].c_mlp_fc_w + // [3072, 1] - model.layers[il].c_mlp_fc_b + // [ 768, N] - cur (in) + // [3072, N] - cur (out) + // + // cur = fc_w*cur + fc_b + // [3072, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_fc_w, + cur); + + cur = ggml_add(ctx, + ggml_repeat(ctx, model.layers[il].c_mlp_fc_b, cur), + cur); + + // GELU activation + // [3072, N] + cur = ggml_gelu(ctx, cur); + + // projection + // [ 768, 3072] - model.layers[il].c_mlp_proj_w + // [ 768, 1] - model.layers[il].c_mlp_proj_b + // [3072, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_proj_w, + cur); + + cur = ggml_add(ctx, + ggml_repeat(ctx, model.layers[il].c_mlp_proj_b, cur), + cur); + } + + // input for next layer + inpL = ggml_add(ctx, cur, inpFF); + } + + // norm + { + // [ 768, N] + inpL = ggml_norm(ctx, inpL, hparams.eps); + + // inpL = ln_f_g*inpL + ln_f_b + // [ 768, N] + inpL = ggml_add(ctx, + ggml_mul(ctx, + ggml_repeat(ctx, model.ln_f_g, inpL), + inpL), + ggml_repeat(ctx, model.ln_f_b, inpL)); + } + + // inpL = WTE * inpL + // [ 768, 50257] - model.lm_head + // [ 768, N] - inpL + inpL = ggml_mul_mat(ctx, model.lm_head, inpL); + ggml_set_name(inpL, "logits"); + // setting a tensor as the output will ensure that it is not overwritten by subsequent operations + ggml_set_output(inpL); + + // logits -> probs + //inpL = ggml_soft_max(ctx0, inpL); + + ggml_build_forward_expand(gf, inpL); + + ggml_free(ctx); + + return gf; +} + +// evaluate the transformer +// +// - model: the model +// - allocr: ggml_gallocr to use to allocate the compute buffer +// - n_threads: number of threads to use +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +bool gpt2_eval( + const gpt2_model & model, + ggml_gallocr_t allocr, + const int n_threads, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_vocab = hparams.n_vocab; + + struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size()); + + // allocate the graph tensors + ggml_gallocr_alloc_graph(allocr, gf); + + // set the graph inputs + struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd"); + memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd)); + + struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position"); + for (int i = 0; i < N; ++i) { + ((int32_t *) position->data)[i] = n_past + i; + } + + // run the computation + struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr); + static std::vector work_buffer; + work_buffer.resize(plan.work_size); + plan.work_data = work_buffer.data(); + ggml_graph_compute(gf, &plan); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + // get the graph outputs + struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits"); + + //embd_w.resize(n_vocab*N); + //memcpy(embd_w.data(), ggml_get_data(logits), sizeof(float)*n_vocab*N); + + // return result just for the last token + embd_w.resize(n_vocab); + memcpy(embd_w.data(), (float *) ggml_get_data(logits) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + params.model = "models/gpt-2-117M/ggml-model.bin"; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt2_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_load(params.model, model, vocab)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + ggml_gallocr_t allocr = NULL; + // allocate the compute buffer + { + allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // create the worst case graph for memory usage estimation + int n_tokens = std::min(model.hparams.n_ctx, params.n_batch); + int n_past = model.hparams.n_ctx - n_tokens; + struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens); + + // pre-allocate the compute buffer for the worst case (optional) + ggml_gallocr_reserve(allocr, gf); + size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0); + fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0); + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str()); + printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size()); + for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) { + printf("%d ", embd_inp[i]); + } + printf("\n\n"); + + // submit the input prompt token-by-token + // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning + std::vector embd; + + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (size_t k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (int32_t(embd.size()) >= params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (embd.back() == 50256) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx_w); + + return 0; +} diff --git a/ggml/examples/gpt-2/main-backend.cpp b/ggml/examples/gpt-2/main-backend.cpp new file mode 100644 index 0000000..6c68712 --- /dev/null +++ b/ggml/examples/gpt-2/main-backend.cpp @@ -0,0 +1,946 @@ +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" + +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#define GPT2_MAX_NODES 4096 + +static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); +} + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gpt2_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // mlp + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt2_model { + gpt2_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // token embedding + struct ggml_tensor * wpe; // position embedding + struct ggml_tensor * lm_head; // language model head + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct ggml_context * ctx_w; + struct ggml_context * ctx_kv; + + ggml_backend_t backend = NULL; + + ggml_backend_buffer_t buffer_w; + ggml_backend_buffer_t buffer_kv; + + std::map tensors; +}; + +// load the model's weights from a file +bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) { + printf("%s: loading model from '%s'\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + ggml_log_set(ggml_log_callback_default, nullptr); + + auto & ctx = model.ctx_w; + + // create the ggml context + { + size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer; + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * n_tensors, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // initialize the backend +#ifdef GGML_USE_CUDA + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: using CUDA backend\n", __func__); + model.backend = ggml_backend_cuda_init(0); + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__); + } + } +#endif + +#ifdef GGML_USE_METAL + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: using Metal backend\n", __func__); + model.backend = ggml_backend_metal_init(); + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); + } + } +#endif + + if (!model.backend) { + // fallback to CPU backend + fprintf(stderr, "%s: using CPU backend\n", __func__); + model.backend = ggml_backend_cpu_init(); + } + + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__); + return false; + } + + // create the tensors for the model + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); + model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + // map by name + model.tensors["model/ln_f/g"] = model.ln_f_g; + model.tensors["model/ln_f/b"] = model.ln_f_b; + + model.tensors["model/wte"] = model.wte; + model.tensors["model/wpe"] = model.wpe; + model.tensors["model/lm_head"] = model.lm_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd); + layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; + model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + + model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; + model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + } + } + + // allocate the model tensors in a backend buffer + model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend); + + printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor)); + printf("%s: backend buffer size = %6.2f MB\n", __func__, ggml_backend_buffer_get_size(model.buffer_w)/(1024.0*1024.0)); + + // override the default training context with the user-provided + model.hparams.n_ctx = n_ctx; + + // key + value memory + { + auto * ctx = model.ctx_kv; + + // create the ggml context + { + size_t n_tensors = 2; + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * n_tensors, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + // k and v here can also be GGML_TYPE_F16 to save memory and speed up the computation + // if backend supports it + model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + + // allocate the KV memory in a backend buffer + model.buffer_kv = ggml_backend_alloc_ctx_tensors(ctx, model.backend); + + const size_t memory_size = ggml_backend_buffer_get_size(model.buffer_kv); + printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + } + + // load weights + { + size_t total_size = 0; + + bool has_lm_head = false; + + std::vector read_buf; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + ggml_set_name(tensor, name.c_str()); + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + if (ggml_backend_buffer_is_host(model.buffer_w)) { + // for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + } else { + // read into a temporary buffer first, then copy to device memory + read_buf.resize(ggml_nbytes(tensor)); + fin.read(read_buf.data(), ggml_nbytes(tensor)); + ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor)); + } + + // GPT-2 models share the WTE tensor as the LM head + if (name == "model/wte" && has_lm_head == false) { + //ggml_backend_tensor_copy(tensor, model.lm_head); + model.lm_head = tensor; + } + + if (name == "model/lm_head") { + has_lm_head = true; + } + + total_size += ggml_nbytes(tensor); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0); + } + + fin.close(); + + return true; +} + +// build the computation graph +struct ggml_cgraph * gpt2_graph( + const gpt2_model & model, + const int n_past, + const int n_tokens) { + const int N = n_tokens; + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + + // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data + static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false); + static std::vector buf(buf_size); + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf.data(), + /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph() + }; + + struct ggml_context * ctx = ggml_init(params); + + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false); + + struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); + // at this point, the tensor data is not allocated yet and cannot be set + // we will find the tensor after the graph is allocated by its name, and set the data then + ggml_set_name(embd, "embd"); + // setting a tensor as an input will ensure that it is allocated at the beginning of the graph + // this is important to ensure that the input tensors are not overwritten before they are used + ggml_set_input(embd); + + struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N); + ggml_set_name(position, "position"); + ggml_set_input(position); + + // wte + wpe + struct ggml_tensor * inpL = + ggml_add(ctx, + ggml_get_rows(ctx, model.wte, embd), + ggml_get_rows(ctx, model.wpe, position)); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + // [ 768, N] + cur = ggml_norm(ctx, inpL, hparams.eps); + + // cur = ln_1_g*cur + ln_1_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_1_g), + model.layers[il].ln_1_b); + } + + // attn + // [2304, 768] - model.layers[il].c_attn_attn_w + // [2304, 1] - model.layers[il].c_attn_attn_b + // [ 768, N] - cur (in) + // [2304, N] - cur (out) + // + // cur = attn_w*cur + attn_b + // [2304, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_attn_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_attn_b); + } + + // self-attention + { + struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd); + struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd); + struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd); + + // store key and value to memory + if (N >= 1) { + struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + // [64, N, 12] + struct ggml_tensor * Q = + ggml_permute(ctx, + ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N), + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + // [64, n_past + N, 12] + struct ggml_tensor * K = + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + + // GG: flash attention + //struct ggml_tensor * V = + // ggml_cpy(ctx0, + // ggml_permute(ctx0, + // ggml_reshape_3d(ctx0, + // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + // n_embd/n_head, n_head, n_past + N), + // 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head)); + + //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true); + + // K * Q + // [n_past + N, N, 12] + struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_scaled = + ggml_scale(ctx, + KQ, + 1.0f/sqrtf(float(n_embd)/n_head)); + + // KQ_masked = mask_past(KQ_scaled) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + + // KQ = soft_max(KQ_masked) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + // [n_past + N, 64, 12] + struct ggml_tensor * V_trans = + ggml_cont_3d(ctx, + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + n_embd/n_head, n_head, n_past + N), + 1, 2, 0, 3), + n_past + N, n_embd/n_head, n_head); + + // KQV = transpose(V) * KQ_soft_max + // [64, N, 12] + struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + // [64, 12, N] + struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + // [768, N] + cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N); + } + + // projection + // [ 768, 768] - model.layers[il].c_attn_proj_w + // [ 768, 1] - model.layers[il].c_attn_proj_b + // [ 768, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_proj_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_proj_b); + } + + // add the input + cur = ggml_add(ctx, cur, inpL); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx, inpFF, hparams.eps); + + // cur = ln_2_g*cur + ln_2_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_2_g), + model.layers[il].ln_2_b); + } + + // fully connected + // [3072, 768] - model.layers[il].c_mlp_fc_w + // [3072, 1] - model.layers[il].c_mlp_fc_b + // [ 768, N] - cur (in) + // [3072, N] - cur (out) + // + // cur = fc_w*cur + fc_b + // [3072, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_fc_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_fc_b); + + // GELU activation + // [3072, N] + cur = ggml_gelu(ctx, cur); + + // projection + // [ 768, 3072] - model.layers[il].c_mlp_proj_w + // [ 768, 1] - model.layers[il].c_mlp_proj_b + // [3072, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_proj_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_proj_b); + } + + // input for next layer + inpL = ggml_add(ctx, cur, inpFF); + } + + // norm + { + // [ 768, N] + inpL = ggml_norm(ctx, inpL, hparams.eps); + + // inpL = ln_f_g*inpL + ln_f_b + // [ 768, N] + inpL = ggml_add(ctx, + ggml_mul(ctx, + inpL, + model.ln_f_g), + model.ln_f_b); + } + + // inpL = WTE * inpL + // [ 768, 50257] - model.lm_head + // [ 768, N] - inpL + inpL = ggml_mul_mat(ctx, model.lm_head, inpL); + ggml_set_name(inpL, "logits"); + // setting a tensor as the output will ensure that it is not overwritten by subsequent operations + ggml_set_output(inpL); + + // logits -> probs + //inpL = ggml_soft_max(ctx0, inpL); + + ggml_build_forward_expand(gf, inpL); + + ggml_free(ctx); + + return gf; +} + +// evaluate the transformer +// +// - model: the model +// - allocr: ggml_gallocr to use to allocate the compute buffer +// - n_threads: number of threads to use +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +bool gpt2_eval( + const gpt2_model & model, + ggml_gallocr_t allocr, + const int n_threads, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_vocab = hparams.n_vocab; + + struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size()); + + // allocate the graph tensors + ggml_gallocr_alloc_graph(allocr, gf); + + // set the graph inputs + struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd"); + ggml_backend_tensor_set(embd, embd_inp.data(), 0, N*ggml_element_size(embd)); + + struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position"); + for (int i = 0; i < N; ++i) { + int32_t v = n_past + i; + ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v)); + } + + // set backend options + if (ggml_backend_is_cpu(model.backend)) { + ggml_backend_cpu_set_n_threads(model.backend, n_threads); + } + + // run the computation + ggml_backend_graph_compute(model.backend, gf); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + // get the graph outputs + struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits"); + + //embd_w.resize(n_vocab*N); + //ggml_backend_tensor_get(logits, embd_w.data(), 0, sizeof(float)*n_vocab*N); + + // return result just for the last token + embd_w.resize(n_vocab); + ggml_backend_tensor_get(logits, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + params.model = "models/gpt-2-117M/ggml-model.bin"; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt2_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + ggml_gallocr_t allocr = NULL; + // allocate the compute buffer + { + // create a graph allocator with the backend's default buffer type + allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + + // create the worst case graph for memory usage estimation + int n_tokens = std::min(model.hparams.n_ctx, params.n_batch); + int n_past = model.hparams.n_ctx - n_tokens; + struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens); + + // pre-allocate the compute buffer for the worst case (optional) + ggml_gallocr_reserve(allocr, gf); + size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0); + fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0); + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str()); + printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size()); + for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) { + printf("%d ", embd_inp[i]); + } + printf("\n\n"); + + // submit the input prompt token-by-token + // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning + std::vector embd; + + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (size_t k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (int32_t(embd.size()) >= params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (!params.ignore_eos && embd.back() == 50256) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx_w); + + ggml_gallocr_free(allocr); + ggml_backend_buffer_free(model.buffer_w); + ggml_backend_buffer_free(model.buffer_kv); + ggml_backend_free(model.backend); + + return 0; +} diff --git a/ggml/examples/gpt-2/main-batched.cpp b/ggml/examples/gpt-2/main-batched.cpp new file mode 100644 index 0000000..e907d82 --- /dev/null +++ b/ggml/examples/gpt-2/main-batched.cpp @@ -0,0 +1,1210 @@ +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" + +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#define GPT2_MAX_NODES 4096 + +static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); +} + +typedef int32_t gpt2_pos; +typedef int32_t gpt2_seq_id; + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gpt2_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // mlp + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt2_kv_cell { + gpt2_pos pos = -1; + gpt2_pos delta = 0; + + std::set seq_id; + + bool has_seq_id(const gpt2_seq_id & id) const { + return seq_id.find(id) != seq_id.end(); + } +}; + +struct gpt2_kv_cache { + // key + value memory + struct ggml_tensor * k; + struct ggml_tensor * v; + // + + uint32_t head = 0; + uint32_t size = 0; + + // computed before each graph build + uint32_t n = 0; + + std::vector cells; + + ggml_backend_buffer_t buffer; +}; + +struct gpt2_model { + gpt2_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // token embedding + struct ggml_tensor * wpe; // position embedding + struct ggml_tensor * lm_head; // language model head + + std::vector layers; + + gpt2_kv_cache kv_cache; + + struct ggml_context * ctx_w; + + ggml_backend_t backend = NULL; + + ggml_backend_buffer_t buffer_w; + + std::map tensors; +}; + +// Input data for gpt2_decode +// A gpt2_batch object can contain input about one or many sequences +// The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens +// +// - token : the token ids of the input (used when embd is NULL) +// - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) +// - pos : the positions of the respective token in the sequence +// - seq_id : the sequence to which the respective token belongs +// - logits : if zero, the logits for the respective token will not be output +// +struct gpt2_batch { + int32_t n_tokens = -1; + + gpt_vocab::id * token = {}; + float * embd = {}; + gpt2_pos * pos = {}; + gpt2_seq_id * seq_id = {}; + int8_t * logits = {}; +}; + +// load the model's weights from a file +bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) { + printf("%s: loading model from '%s'\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx_w; + + size_t buffer_size = 0; + + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + buffer_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g + buffer_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b + + buffer_size += ggml_row_size(wtype, n_vocab*n_embd); // wte + buffer_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe + buffer_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head + + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b + + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b + + buffer_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b + + buffer_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b + + buffer_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b + + buffer_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w + buffer_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b + + buffer_size += (6 + 12*n_layer)*128; // alignment overhead + + printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor)); + printf("%s: backend buffer size = %6.2f MB\n", __func__, buffer_size/(1024.0*1024.0)); + } + + ggml_log_set(ggml_log_callback_default, nullptr); + + // create the ggml context + { + size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer; + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * n_tensors, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + model.ctx_w = ggml_init(params); + if (!model.ctx_w) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // initialize the backend +#ifdef GGML_USE_CUDA + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: using CUDA backend\n", __func__); + model.backend = ggml_backend_cuda_init(0); + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__); + } + } +#endif + +#ifdef GGML_USE_METAL + if (n_gpu_layers > 0) { + fprintf(stderr, "%s: using Metal backend\n", __func__); + model.backend = ggml_backend_metal_init(); + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); + } + } +#endif + + if (!model.backend) { + // fallback to CPU backend + fprintf(stderr, "%s: using CPU backend\n", __func__); + model.backend = ggml_backend_cpu_init(); + } + + if (!model.backend) { + fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__); + return false; + } + + // allocate weights buffer + model.buffer_w = ggml_backend_alloc_buffer(model.backend, buffer_size); + + // prepare memory for the weights + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); + model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + // map by name + model.tensors["model/ln_f/g"] = model.ln_f_g; + model.tensors["model/ln_f/b"] = model.ln_f_b; + + model.tensors["model/wte"] = model.wte; + model.tensors["model/wpe"] = model.wpe; + model.tensors["model/lm_head"] = model.lm_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd); + layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; + model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + + model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; + model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + } + } + + // override the default training context with the user-provided + model.hparams.n_ctx = n_ctx; + + // key + value memory + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + model.kv_cache.k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + model.kv_cache.v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + + model.kv_cache.head = 0; + model.kv_cache.size = n_ctx; + + model.kv_cache.cells.resize(n_ctx); + + const size_t memory_size = ggml_nbytes(model.kv_cache.k) + ggml_nbytes(model.kv_cache.v); + + printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + + // create a backend buffer (can be in host or device memory) + model.kv_cache.buffer = ggml_backend_alloc_buffer(model.backend, memory_size + 256); + + // allocate the tensors into the backend buffer + { + ggml_tallocr alloc = ggml_tallocr_new(model.kv_cache.buffer); + + // this updates the pointers in the tensors to point to the correct location in the buffer + // this is necessary since the ggml_context is .no_alloc == true + // note that the buffer can actually be a device buffer, depending on the backend + ggml_tallocr_alloc(&alloc, model.kv_cache.k); + ggml_tallocr_alloc(&alloc, model.kv_cache.v); + } + } + + // load weights + { + ggml_tallocr alloc = ggml_tallocr_new(model.buffer_w); + + size_t total_size = 0; + + bool has_lm_head = false; + + std::vector read_buf; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + ggml_set_name(tensor, name.c_str()); + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + ggml_tallocr_alloc(&alloc, tensor); + + if (ggml_backend_is_cpu (model.backend) +#ifdef GGML_USE_METAL + || ggml_backend_is_metal(model.backend) +#endif + ) { + // for the CPU and Metal backend, we can read directly into the tensor + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + } else { + // read into a temporary buffer first, then copy to device memory + read_buf.resize(ggml_nbytes(tensor)); + fin.read(read_buf.data(), ggml_nbytes(tensor)); + ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor)); + } + + // GPT-2 models share the WTE tensor as the LM head + if (name == "model/wte" && has_lm_head == false) { + //ggml_tallocr_alloc(alloc, model.lm_head); + //ggml_backend_tensor_copy(tensor, model.lm_head); + model.lm_head = tensor; + } + + if (name == "model/lm_head") { + has_lm_head = true; + } + + total_size += ggml_nbytes(tensor); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0); + } + + fin.close(); + + return true; +} + +// build the computation graph +struct ggml_cgraph * gpt2_graph( + const gpt2_model & model, + const gpt2_batch & batch, + bool measure) { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + + const auto & kv_cache = model.kv_cache; + + const int32_t n_tokens = batch.n_tokens; + const int32_t n_kv = measure ? n_ctx : kv_cache.n; + const int32_t kv_head = measure ? n_ctx - n_tokens : kv_cache.head; + + // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data + static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false); + static std::vector buf(buf_size); + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf.data(), + /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph() + }; + + struct ggml_context * ctx = ggml_init(params); + + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false); + + struct ggml_tensor * inpL; + if (batch.token) { + struct ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens); + ggml_set_name(inp_tokens, "inp_tokens"); + ggml_set_input(inp_tokens); + + struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_tokens); + ggml_set_name(position, "position"); + ggml_set_input(position); + + // wte + wpe + inpL = + ggml_add(ctx, + ggml_get_rows(ctx, model.wte, inp_tokens), + ggml_get_rows(ctx, model.wpe, position)); + } else { + GGML_ASSERT(batch.embd); + + inpL = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_tokens); + ggml_set_name(inpL, "embd"); + ggml_set_input(inpL); + } + + // KQ_mask (mask for 1 head, it will be broadcasted to all heads) + struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_kv, n_tokens, 1); + ggml_set_name(KQ_mask, "KQ_mask"); + ggml_set_input(KQ_mask); + + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + // [ 768, N] + cur = ggml_norm(ctx, inpL, hparams.eps); + + // cur = ln_1_g*cur + ln_1_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_1_g), + model.layers[il].ln_1_b); + } + + // attn + // [2304, 768] - model.layers[il].c_attn_attn_w + // [2304, 1] - model.layers[il].c_attn_attn_b + // [ 768, n_tokens] - cur (in) + // [2304, n_tokens] - cur (out) + // + // cur = attn_w*cur + attn_b + // [2304, n_tokens] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_attn_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_attn_b); + } + + // self-attention + { + struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*n_embd); + struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 1*sizeof(float)*n_embd); + struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, n_tokens, cur->nb[1], 2*sizeof(float)*n_embd); + + // store key and value to memory + if (n_tokens >= 1) { + struct ggml_tensor * k = ggml_view_1d(ctx, model.kv_cache.k, n_tokens*n_embd, (ggml_element_size(model.kv_cache.k)*n_embd)*(il*n_ctx + kv_head)); + struct ggml_tensor * v = ggml_view_1d(ctx, model.kv_cache.v, n_tokens*n_embd, (ggml_element_size(model.kv_cache.v)*n_embd)*(il*n_ctx + kv_head)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + // [64, N, 12] + struct ggml_tensor * Q = + ggml_permute(ctx, + ggml_cont_3d(ctx, + Qcur, + n_embd/n_head, n_head, n_tokens), + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_kv).permute(0, 2, 1, 3) + // [64, n_kv, 12] + struct ggml_tensor * K = + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.kv_cache.k, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.k)*n_embd), + n_embd/n_head, n_head, n_kv), + 0, 2, 1, 3); + + // GG: flash attention + //struct ggml_tensor * V = + // ggml_cpy(ctx0, + // ggml_permute(ctx0, + // ggml_reshape_3d(ctx0, + // ggml_view_1d(ctx0, model.kv_cache.v, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.v)*n_embd), + // n_embd/n_head, n_head, n_kv), + // 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_embd/n_head, n_head)); + + //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true); + + // K * Q + // [n_kv, n_tokens, 12] + struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + // [n_kv, n_tokens, 12] + struct ggml_tensor * KQ_scaled = + ggml_scale(ctx, + KQ, + 1.0f/sqrtf(float(n_embd)/n_head)); + + // KQ_masked = mask_past(KQ_scaled) + // [n_kv, n_tokens, 12] + struct ggml_tensor * KQ_masked = ggml_add(ctx, KQ_scaled, KQ_mask); + + // KQ = soft_max(KQ_masked) + // [n_kv, N, 12] + struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_kv).permute(1, 2, 0, 3).contiguous() + // [n_kv, 64, 12] + struct ggml_tensor * V_trans = + ggml_cont_3d(ctx, + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.kv_cache.v, n_kv*n_embd, il*n_ctx*ggml_element_size(model.kv_cache.v)*n_embd), + n_embd/n_head, n_head, n_kv), + 1, 2, 0, 3), + n_kv, n_embd/n_head, n_head); + + // KQV = transpose(V) * KQ_soft_max + // [64, n_tokens, 12] + struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + // [64, 12, n_tokens] + struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + // [768, n_tokens] + cur = ggml_cont_2d(ctx, KQV_merged, n_embd, n_tokens); + } + + // projection + // [ 768, 768] - model.layers[il].c_attn_proj_w + // [ 768, 1] - model.layers[il].c_attn_proj_b + // [ 768, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_proj_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_proj_b); + } + + // add the input + cur = ggml_add(ctx, cur, inpL); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx, inpFF, hparams.eps); + + // cur = ln_2_g*cur + ln_2_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_2_g), + model.layers[il].ln_2_b); + } + + // fully connected + // [3072, 768] - model.layers[il].c_mlp_fc_w + // [3072, 1] - model.layers[il].c_mlp_fc_b + // [ 768, N] - cur (in) + // [3072, N] - cur (out) + // + // cur = fc_w*cur + fc_b + // [3072, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_fc_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_fc_b); + + // GELU activation + // [3072, N] + cur = ggml_gelu(ctx, cur); + + // projection + // [ 768, 3072] - model.layers[il].c_mlp_proj_w + // [ 768, 1] - model.layers[il].c_mlp_proj_b + // [3072, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_proj_w, + cur); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_proj_b); + } + + // input for next layer + inpL = ggml_add(ctx, cur, inpFF); + } + + // norm + { + // [ 768, N] + inpL = ggml_norm(ctx, inpL, hparams.eps); + + // inpL = ln_f_g*inpL + ln_f_b + // [ 768, N] + inpL = ggml_add(ctx, + ggml_mul(ctx, + inpL, + model.ln_f_g), + model.ln_f_b); + } + + // inpL = WTE * inpL + // [ 768, 50257] - model.lm_head + // [ 768, N] - inpL + inpL = ggml_mul_mat(ctx, model.lm_head, inpL); + + // logits -> probs + //inpL = ggml_soft_max(ctx0, inpL); + + ggml_build_forward_expand(gf, inpL); + + ggml_free(ctx); + + return gf; +} + +static void gpt2_kv_cache_seq_cp( + struct gpt2_kv_cache & cache, + gpt2_seq_id seq_id_src, + gpt2_seq_id seq_id_dst, + gpt2_pos p0, + gpt2_pos p1) { + if (p0 < 0) p0 = 0; + if (p1 < 0) p1 = std::numeric_limits::max(); + + for (uint32_t i = 0; i < cache.size; ++i) { + if (cache.cells[i].has_seq_id(seq_id_src) && cache.cells[i].pos >= p0 && cache.cells[i].pos < p1) { + cache.cells[i].seq_id.insert(seq_id_dst); + } + } +} + +struct gpt2_batch gpt2_batch_init(int32_t n_tokens, int32_t embd) { + gpt2_batch batch; + + if (embd) { + batch.embd = (float *) malloc(sizeof(float) * n_tokens * embd); + } else { + batch.token = (gpt_vocab::id *) malloc(sizeof(gpt_vocab::id) * n_tokens); + } + + batch.pos = (gpt2_pos *) malloc(sizeof(gpt2_pos) * n_tokens); + batch.seq_id = (gpt2_seq_id *) malloc(sizeof(gpt2_seq_id) * n_tokens); + batch.logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens); + + return batch; +} + +void gpt2_batch_free(struct gpt2_batch batch) { + if (batch.token) free(batch.token); + if (batch.embd) free(batch.embd); + if (batch.pos) free(batch.pos); + if (batch.seq_id) free(batch.seq_id); + if (batch.logits) free(batch.logits); +} + +// Positive return values does not mean a fatal error, but rather a warning. +// 0 - success +// < 0 - error +int gpt2_decode( + struct gpt2_model & model, + ggml_gallocr_t allocr, + struct gpt2_batch batch, + int n_threads, + std::vector & logits) { + const int32_t n_tokens = batch.n_tokens; + const auto & hparams = model.hparams; + const int n_vocab = hparams.n_vocab; + + if (n_tokens == 0) { + printf("%s: n_tokens == 0", __func__); + return -1; + } + + GGML_ASSERT((!batch.token && batch.embd) || (batch.token && !batch.embd)); + + auto & cache = model.kv_cache; + + for (int i = 0; i < n_tokens; i++) { + cache.cells[cache.head + i].pos = batch.pos[i]; + cache.cells[cache.head + i].seq_id.insert(batch.seq_id[i]); + } + + cache.n = cache.head + n_tokens; + + struct ggml_cgraph * gf = gpt2_graph(model, batch, false); + + // allocate tensors + ggml_gallocr_alloc_graph(allocr, gf); + + // set the graph inputs + if (batch.token) { + struct ggml_tensor * inp_tokens = ggml_graph_get_tensor(gf, "inp_tokens"); + ggml_backend_tensor_set(inp_tokens, batch.token, 0, n_tokens*ggml_element_size(inp_tokens)); + + struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position"); + for (int i = 0; i < n_tokens; ++i) { + int32_t v = batch.pos[i]; + ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v)); + } + } else { + struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd"); + ggml_backend_tensor_set(embd, batch.embd, 0, n_tokens * hparams.n_embd * ggml_element_size(embd)); + } + + { + struct ggml_tensor * KQ_mask = ggml_graph_get_tensor(gf, "KQ_mask"); + const auto & kv_cache = model.kv_cache; + const int32_t n_tokens = batch.n_tokens; + const int32_t n_kv = kv_cache.n; + + std::vector data_buf(n_kv*n_tokens); + const float neg_inf_v = -INFINITY; + + for (int h = 0; h < 1; ++h) { + int h_offset = h*(n_kv*n_tokens); + for (int j = 0; j < n_tokens; ++j) { + const gpt2_pos pos = batch.pos[j]; + const gpt2_seq_id seq_id = batch.seq_id[j]; + + for (int i = 0; i < n_kv; ++i) { + if (!kv_cache.cells[i].has_seq_id(seq_id) || kv_cache.cells[i].pos > pos) { + data_buf[h_offset + j*n_kv + i] = neg_inf_v; + } + } + } + } + + ggml_backend_tensor_set(KQ_mask, data_buf.data(), 0, data_buf.size() * sizeof(float)); + } + + // run the computation + if (ggml_backend_is_cpu(model.backend)) { + ggml_backend_cpu_set_n_threads(model.backend, n_threads); + } + ggml_backend_graph_compute(model.backend, gf); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + // in this case, the output tensor is the last one in the graph + struct ggml_tensor * inpL = ggml_graph_node(gf, -1); + + if (batch.logits) { + // return logits for all tokens + logits.resize(n_vocab*n_tokens); + for (int32_t i = 0; i < n_tokens; i++) { + if (batch.logits[i] == 0) { + continue; + } + ggml_backend_tensor_get(inpL, logits.data() + n_vocab*i, n_vocab*i*sizeof(float), sizeof(float)*n_vocab); + } + } else { + // return result just for the last token + logits.resize(n_vocab); + ggml_backend_tensor_get(inpL, logits.data(), (n_vocab*(n_tokens-1))*sizeof(float), sizeof(float)*n_vocab); + } + + // update the kv ring buffer + cache.head += n_tokens; + + // ensure kv cache head points to a valid index. + if (cache.head >= cache.size) { + printf("%s: cache.head >= cache.size\n", __func__); + return -2; + } + + return 0; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt2_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + const int n_parallel = params.n_parallel; + const int n_batch_max = std::max(embd_inp.size(), (size_t)n_parallel); + + // create a gpt2_batch + // we use this object to submit token data for decoding + gpt2_batch batch = gpt2_batch_init(n_batch_max, 0); + + // prepare required memory and allocate the compute buffer + ggml_gallocr_t allocr = NULL; + { + // create an allocator to measure the memory usage + allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + + // create the worst case graph for memory usage estimation + batch.n_tokens = n_batch_max; + struct ggml_cgraph * gf = gpt2_graph(model, batch, true); + + // pre-allocate the compute buffer for the worst case (optional) + ggml_gallocr_reserve(allocr, gf); + size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0); + fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0); + } + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // evaluate the initial prompt + batch.n_tokens = embd_inp.size(); + + for (int32_t i = 0; i < batch.n_tokens; i++) { + batch.token[i] = embd_inp[i]; + batch.pos[i] = i; + batch.seq_id[i] = 0; + batch.logits[i] = false; + } + + // gpt2_decode will output logits only for the last token of the prompt + batch.logits[batch.n_tokens - 1] = true; + + if (gpt2_decode(model, allocr, batch, params.n_threads, logits) != 0) { + printf("%s: gpt2_decode() failed\n", __func__); + return 1; + } + + // assign the system KV cache to all parallel sequences + // this way, the parallel sequences will "reuse" the prompt tokens without having to copy them + for (int32_t i = 1; i < n_parallel; ++i) { + gpt2_kv_cache_seq_cp(model.kv_cache, 0, i, 0, batch.n_tokens); + } + + if (n_parallel > 1) { + printf("\n\n%s: generating %d sequences ...\n", __func__, n_parallel); + } + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str()); + printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size()); + for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) { + printf("%d ", embd_inp[i]); + } + printf("\n\n"); + + std::vector streams(n_parallel); + + // remember the batch index of the last token for each parallel sequence + // we need this to determine which logits to sample from + std::vector i_batch(n_parallel, batch.n_tokens - 1); + + int n_cur = batch.n_tokens; + int n_len = batch.n_tokens + params.n_predict; + int n_decoded = 0; + + const int n_vocab = model.hparams.n_vocab; + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + while (n_cur < n_len) { + batch.n_tokens = 0; + + for (int32_t i = 0; i < n_parallel; ++i) { + if (i_batch[i] < 0) { + // the stream has already finished + continue; + } + + auto * logits_i = logits.data() + i_batch[i]*n_vocab; + + gpt_vocab::id id = 0; + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits_i, top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // is it an end of stream? -> mark the stream as finished + if ((!params.ignore_eos && id == 50256) || n_cur == n_len - 1) { + i_batch[i] = -1; + printf("\n"); + if (n_parallel > 1) { + printf("%s: stream %d finished at n_cur = %d", __func__, i, n_cur); + } + + continue; + } + + auto& token = vocab.id_to_token[id]; + if (n_parallel == 1) { + printf("%s", token.c_str()); + fflush(stdout); + } + + streams[i] += token; + + // push this new token for next evaluation + batch.token [batch.n_tokens] = id; + batch.pos [batch.n_tokens] = n_cur; + batch.seq_id[batch.n_tokens] = i; + batch.logits[batch.n_tokens] = true; + + i_batch[i] = batch.n_tokens; + + batch.n_tokens += 1; + + n_decoded += 1; + } + + // all streams are finished + if (batch.n_tokens == 0) { + break; + } + + n_cur += 1; + + { + const int64_t t_start_us = ggml_time_us(); + + // evaluate the current batch with the transformer model + int ret_code = gpt2_decode(model, allocr, batch, params.n_threads, logits); + if (ret_code != 0) { + fprintf(stderr, "%s : failed to eval, return code %d\n", __func__, ret_code); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + } + + if (n_parallel > 1) { + printf("\n"); + + for (int32_t i = 0; i < n_parallel; ++i) { + printf("sequence %d:\n\n%s%s\n\n", i, params.prompt.c_str(), streams[i].c_str()); + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: n_decoded = %8d\n", __func__, n_decoded); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms\n", __func__, t_predict_us/1000.0f); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + gpt2_batch_free(batch); + ggml_free(model.ctx_w); + + ggml_gallocr_free(allocr); + ggml_backend_buffer_free(model.buffer_w); + ggml_backend_buffer_free(model.kv_cache.buffer); + ggml_backend_free(model.backend); + + return 0; +} diff --git a/ggml/examples/gpt-2/main-ctx.cpp b/ggml/examples/gpt-2/main-ctx.cpp new file mode 100644 index 0000000..fe09997 --- /dev/null +++ b/ggml/examples/gpt-2/main-ctx.cpp @@ -0,0 +1,840 @@ +#include "ggml.h" +#include "ggml-cpu.h" +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gpt2_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // mlp + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt2_model { + gpt2_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // token embedding + struct ggml_tensor * wpe; // position embedding + struct ggml_tensor * lm_head; // language model head + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct ggml_context * ctx_w; + std::map tensors; +}; + +// load the model's weights from a file +bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) { + printf("%s: loading model from '%s'\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx_w; + + size_t ctx_size = 0; + + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b + + ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte + ctx_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe + ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head + + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b + + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b + + ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b + + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b + + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v + + ctx_size += (6 + 12*n_layer)*512; // object overhead + + printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor)); + printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); + } + + // create the ggml context + { + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + model.ctx_w = ggml_init(params); + if (!model.ctx_w) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // prepare memory for the weights + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); + model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + // map by name + model.tensors["model/ln_f/g"] = model.ln_f_g; + model.tensors["model/ln_f/b"] = model.ln_f_b; + + model.tensors["model/wte"] = model.wte; + model.tensors["model/wpe"] = model.wpe; + model.tensors["model/lm_head"] = model.lm_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd); + layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; + model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + + model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; + model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + } + } + + // key + value memory + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + + const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); + + printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + } + + // load weights + { + size_t total_size = 0; + + bool has_lm_head = false; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + + // GPT-2 models share the WTE tensor as the LM head + if (name == "model/wte" && has_lm_head == false) { + memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor)); + } + + if (name == "model/lm_head") { + has_lm_head = true; + } + + total_size += ggml_nbytes(tensor); + } + + printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0); + } + + fin.close(); + + return true; +} + +// evaluate the transformer +// +// - model: the model +// - n_threads: number of threads to use +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +bool gpt2_eval( + const gpt2_model & model, + const int n_threads, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w, + size_t & mem_per_token) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + const int n_vocab = hparams.n_vocab; + + static size_t buf_size = 256u*1024*1024; + static void * buf = malloc(buf_size); + + if (mem_per_token > 0 && mem_per_token*N > buf_size) { + const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead + //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); + + // reallocate + buf_size = buf_size_new; + buf = realloc(buf, buf_size); + if (buf == nullptr) { + fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size); + return false; + } + } + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx0 = ggml_init(params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd)); + + struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + for (int i = 0; i < N; ++i) { + ((int32_t *) position->data)[i] = n_past + i; + } + + // wte + wpe + struct ggml_tensor * inpL = + ggml_add(ctx0, + ggml_get_rows(ctx0, model.wte, embd), + ggml_get_rows(ctx0, model.wpe, position)); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + // [ 768, N] + cur = ggml_norm(ctx0, inpL, hparams.eps); + + // cur = ln_1_g*cur + ln_1_b + // [ 768, N] + cur = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.layers[il].ln_1_g, cur), + cur), + ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); + } + + // attn + // [2304, 768] - model.layers[il].c_attn_attn_w + // [2304, 1] - model.layers[il].c_attn_attn_b + // [ 768, N] - cur (in) + // [2304, N] - cur (out) + // + // cur = attn_w*cur + attn_b + // [2304, N] + { + cur = ggml_mul_mat(ctx0, + model.layers[il].c_attn_attn_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur), + cur); + } + + // self-attention + { + struct ggml_tensor * Qcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd); + struct ggml_tensor * Kcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd); + struct ggml_tensor * Vcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd); + + // store key and value to memory + if (N >= 1) { + struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + // [64, N, 12] + struct ggml_tensor * Q = + ggml_permute(ctx0, + ggml_cpy(ctx0, + Qcur, + ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)), + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + // [64, n_past + N, 12] + struct ggml_tensor * K = + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + + // GG: flash attention + //struct ggml_tensor * V = + // ggml_cpy(ctx0, + // ggml_permute(ctx0, + // ggml_reshape_3d(ctx0, + // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + // n_embd/n_head, n_head, n_past + N), + // 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head)); + + //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true); + + // K * Q + // [n_past + N, N, 12] + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head)); + + // KQ_masked = mask_past(KQ_scaled) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + + // KQ = soft_max(KQ_masked) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + // [n_past + N, 64, 12] + struct ggml_tensor * V_trans = + ggml_cpy(ctx0, + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + n_embd/n_head, n_head, n_past + N), + 1, 2, 0, 3), + ggml_new_tensor_3d(ctx0, model.memory_v->type, n_past + N, n_embd/n_head, n_head)); + + // KQV = transpose(V) * KQ_soft_max + // [64, N, 12] + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + // [64, 12, N] + struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + // [768, N] + cur = ggml_cpy(ctx0, + KQV_merged, + ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + } + + // projection + // [ 768, 768] - model.layers[il].c_attn_proj_w + // [ 768, 1] - model.layers[il].c_attn_proj_b + // [ 768, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + { + cur = ggml_mul_mat(ctx0, + model.layers[il].c_attn_proj_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur), + cur); + } + + // add the input + cur = ggml_add(ctx0, cur, inpL); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx0, inpFF, hparams.eps); + + // cur = ln_2_g*cur + ln_2_b + // [ 768, N] + cur = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.layers[il].ln_2_g, cur), + cur), + ggml_repeat(ctx0, model.layers[il].ln_2_b, cur)); + } + + // fully connected + // [3072, 768] - model.layers[il].c_mlp_fc_w + // [3072, 1] - model.layers[il].c_mlp_fc_b + // [ 768, N] - cur (in) + // [3072, N] - cur (out) + // + // cur = fc_w*cur + fc_b + // [3072, N] + cur = ggml_mul_mat(ctx0, + model.layers[il].c_mlp_fc_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur), + cur); + + // GELU activation + // [3072, N] + cur = ggml_gelu(ctx0, cur); + + // projection + // [ 768, 3072] - model.layers[il].c_mlp_proj_w + // [ 768, 1] - model.layers[il].c_mlp_proj_b + // [3072, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + cur = ggml_mul_mat(ctx0, + model.layers[il].c_mlp_proj_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur), + cur); + } + + // input for next layer + inpL = ggml_add(ctx0, cur, inpFF); + } + + // norm + { + // [ 768, N] + inpL = ggml_norm(ctx0, inpL, hparams.eps); + + // inpL = ln_f_g*inpL + ln_f_b + // [ 768, N] + inpL = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.ln_f_g, inpL), + inpL), + ggml_repeat(ctx0, model.ln_f_b, inpL)); + } + + // inpL = WTE * inpL + // [ 768, 50257] - model.lm_head + // [ 768, N] - inpL + inpL = ggml_mul_mat(ctx0, model.lm_head, inpL); + + // logits -> probs + //inpL = ggml_soft_max_inplace(ctx0, inpL); + + // run the computation + ggml_build_forward_expand(gf, inpL); + ggml_graph_compute_with_ctx(ctx0, gf, n_threads); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + //embd_w.resize(n_vocab*N); + //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N); + + // return result just for the last token + embd_w.resize(n_vocab); + memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + //printf("used_mem = %zu\n", ggml_used_mem(ctx0)); + + ggml_free(ctx0); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + params.model = "models/gpt-2-117M/ggml-model.bin"; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt2_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_load(params.model, model, vocab)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str()); + printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size()); + for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) { + printf("%d ", embd_inp[i]); + } + printf("\n\n"); + + // submit the input prompt token-by-token + // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning + std::vector embd; + + // determine the required inference memory per token: + size_t mem_per_token = 0; + gpt2_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); + + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (size_t k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (int32_t(embd.size()) >= params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (embd.back() == 50256) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx_w); + + return 0; +} diff --git a/ggml/examples/gpt-2/main-sched.cpp b/ggml/examples/gpt-2/main-sched.cpp new file mode 100644 index 0000000..7d789d6 --- /dev/null +++ b/ggml/examples/gpt-2/main-sched.cpp @@ -0,0 +1,1079 @@ +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" + +#ifdef GGML_USE_CUDA +#include "ggml-cuda.h" +#endif + +#ifdef GGML_USE_METAL +#include "ggml-metal.h" +#endif + +#ifdef GGML_USE_BLAS +#include "ggml-blas.h" +#endif + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +#define GPT2_MAX_NODES 4096 + +static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); +} + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gpt2_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + struct ggml_tensor * ln_2_g; + struct ggml_tensor * ln_2_b; + + // attention + struct ggml_tensor * c_attn_attn_w; + struct ggml_tensor * c_attn_attn_b; + + struct ggml_tensor * c_attn_proj_w; + struct ggml_tensor * c_attn_proj_b; + + // mlp + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gpt2_model { + gpt2_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // tkoen embedding + struct ggml_tensor * wpe; // position embedding + struct ggml_tensor * lm_head; // language model head + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct ggml_context * ctx_w; + + std::vector backends; + std::vector buffers_w; + ggml_backend_buffer_t buffer_kv; + ggml_backend_buffer_t buffer_input; + + std::map tensors; + + // inputs/constants + struct ggml_tensor * embd; + struct ggml_tensor * position; +}; + +void init_backends(gpt2_model & model, const gpt_params & params) { + ggml_backend_t gpu_backend = NULL; + + ggml_log_set(ggml_log_callback_default, nullptr); + + // initialize the backends +#ifdef GGML_USE_CUDA + if (params.n_gpu_layers > 0) { + fprintf(stderr, "%s: using CUDA backend\n", __func__); + gpu_backend = ggml_backend_cuda_init(0); + if (!gpu_backend) { + fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__); + } + } +#endif + +#ifdef GGML_USE_METAL + if (params.n_gpu_layers > 0) { + fprintf(stderr, "%s: using Metal backend\n", __func__); + gpu_backend = ggml_backend_metal_init(); + if (!gpu_backend) { + fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); + } + } +#endif + if (gpu_backend) { + model.backends.push_back(gpu_backend); + } + +#ifdef GGML_USE_BLAS + ggml_backend_t blas_backend = ggml_backend_blas_init(); + if (!blas_backend) { + fprintf(stderr, "%s: failed to initialize BLAS backend\n", __func__); + } else { + ggml_backend_blas_set_n_threads(blas_backend, params.n_threads); + model.backends.push_back(blas_backend); + } +#endif + + // always add the CPU backend as a fallback + ggml_backend_t cpu_backend = ggml_backend_cpu_init(); + ggml_backend_cpu_set_n_threads(cpu_backend, params.n_threads); + model.backends.push_back(cpu_backend); +} + +// load the model's weights from a file +bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, const gpt_params & params) { + printf("%s: loading model from '%s'\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx_w; + + // create the ggml context + { + size_t n_tensors = 3 /* input */ + 2 /* kv */ + 6 + 12*model.hparams.n_layer; + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead() * n_tensors, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + + model.ctx_w = ggml_init(params); + if (!model.ctx_w) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // create tensors for the weights + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); + model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + // map by name + model.tensors["model/ln_f/g"] = model.ln_f_g; + model.tensors["model/ln_f/b"] = model.ln_f_b; + + model.tensors["model/wte"] = model.wte; + model.tensors["model/wpe"] = model.wpe; + model.tensors["model/lm_head"] = model.lm_head; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd); + layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; + model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + + model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; + model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; + model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w; + model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + } + } + + // assign tensors to backends + init_backends(model, params); + ggml_backend_t backend_gpu = model.backends.front(); + ggml_backend_t backend_cpu = model.backends.back(); + std::map tensor_backends; + { + const int i_gpu_first_layer = model.hparams.n_layer - params.n_gpu_layers; + for (auto it : model.tensors) { + const std::string & name = it.first; + // input tensors + if (name == "model/wte" || name == "model/wpe") { + if (params.n_gpu_layers > model.hparams.n_layer) { + tensor_backends[name] = backend_gpu; + } else { + tensor_backends[name] = backend_cpu; + } + } + // output tensors + if (name == "model/ln_f/g" || name == "model/ln_f/b" || name == "model/lm_head") { + if (params.n_gpu_layers > 0) { + tensor_backends[name] = backend_gpu; + } else { + tensor_backends[name] = backend_cpu; + } + } + // layer tensors + if (name.substr(0, 7) == "model/h") { + // parse layer number + int layer = std::stoi(name.substr(7, 2)); + if (layer >= i_gpu_first_layer) { + tensor_backends[name] = backend_gpu; + } else { + tensor_backends[name] = backend_cpu; + } + } + } + } + + // allocate buffers + std::map backend_buffers; + for (auto backend : model.backends) { + // compute the size of the buffer + size_t size = 0; + for (auto it : model.tensors) { + if (tensor_backends[it.first] == backend) { + size += ggml_nbytes(it.second) + 512; + } + } + if (size > 0) { + printf("%s: %8s buffer size = %8.2f MB\n", __func__, ggml_backend_name(backend), size/1024.0/1024.0); + // allocate the buffer + ggml_backend_buffer_t buffer = ggml_backend_alloc_buffer(backend, size); + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + model.buffers_w.push_back(buffer); + + // create an allocator for the buffer to allocate the tensors + auto alloc = ggml_tallocr_new(buffer); + backend_buffers.insert(std::make_pair(backend, std::move(alloc))); + } else { + model.buffers_w.push_back(NULL); + } + } + + // allocate key + value memory + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements); + + ggml_set_name(model.memory_k, "model/memory_k"); + ggml_set_name(model.memory_v, "model/memory_v"); + + const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); + + printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + + // create a backend buffer (can be in host or device memory) + ggml_backend_t backend_kv = params.n_gpu_layers >= hparams.n_layer/2 ? backend_gpu : backend_cpu; + printf("%s: backend_kv = %s\n", __func__, ggml_backend_name(backend_kv)); + model.buffer_kv = ggml_backend_alloc_buffer(backend_kv, memory_size + 512*2); + + // allocate the tensors into the backend buffer + { + ggml_tallocr alloc = ggml_tallocr_new(model.buffer_kv); + + // this updates the pointers in the tensors to point to the correct location in the buffer + // this is necessary since the ggml_context is .no_alloc == true + // note that the buffer can actually be a device buffer, depending on the backend + ggml_tallocr_alloc(&alloc, model.memory_k); + ggml_tallocr_alloc(&alloc, model.memory_v); + } + } + + // load weights + { + size_t total_size = 0; + + bool has_lm_head = false; + + std::vector read_buf; + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + ggml_set_name(tensor, name.c_str()); + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + // allocate the tensor + ggml_backend_t backend = tensor_backends[name]; + ggml_tallocr * alloc = &backend_buffers.find(backend)->second; + ggml_tallocr_alloc(alloc, tensor); + //printf("%s: [%5.5s] %s\n", __func__, ggml_backend_name(backend), name.c_str()); + + if (ggml_backend_is_cpu(backend) +#ifdef GGML_USE_METAL + || ggml_backend_is_metal(backend) +#endif + ) { + // for the CPU and Metal backend, we can read directly into the tensor + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + } else { + // read into a temporary buffer first, then copy to device memory + read_buf.resize(ggml_nbytes(tensor)); + fin.read(read_buf.data(), ggml_nbytes(tensor)); + ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor)); + } + + // GPT-2 models share the WTE tensor as the LM head + if (name == "model/wte" && has_lm_head == false) { + ggml_tallocr * alloc_head = &backend_buffers.find(tensor_backends["model/lm_head"])->second; + ggml_tallocr_alloc(alloc_head, model.lm_head); + //printf("%s: [%5.5s] %s (copied)\n", __func__, ggml_backend_name(tensor_backends["model/lm_head"]), "model/lm_head"); + ggml_backend_tensor_copy(tensor, model.lm_head); + total_size += ggml_nbytes(model.lm_head); + } + + if (name == "model/lm_head") { + has_lm_head = true; + } + + total_size += ggml_nbytes(tensor); + } + printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0); + } + + fin.close(); + + // allocate input tensors + { + model.embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, model.hparams.n_ctx); + model.position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, model.hparams.n_ctx); + + ggml_set_name(model.embd, "in/embd"); + ggml_set_name(model.position, "in/position"); + + // add input tensors to cpu backend + size_t input_size = ggml_nbytes(model.embd) + ggml_nbytes(model.position); + + // FIXME: use cpu backend after sched impl + ggml_backend_t backend_input = params.n_gpu_layers >= model.hparams.n_layer ? backend_gpu : backend_cpu; + model.buffer_input = ggml_backend_alloc_buffer(backend_input, input_size + 512*3); + printf("%s: backend_in = %s (%zu bytes)\n", __func__, ggml_backend_name(backend_input), input_size); + + // allocate the tensors into the backend buffer + ggml_tallocr alloc = ggml_tallocr_new(model.buffer_input); + ggml_tallocr_alloc(&alloc, model.embd); + ggml_tallocr_alloc(&alloc, model.position); + } + + return true; +} + +// build the computation graph +struct ggml_cgraph * gpt2_graph( + const gpt2_model & model, + const int n_past, + const std::vector & embd_inp) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + + // since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data + static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false); + static std::vector buf(buf_size); + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf.data(), + /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph() + }; + + struct ggml_context * ctx = ggml_init(params); + + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false); + + struct ggml_tensor * embd = ggml_view_1d(ctx, model.embd, N, 0); + + // set inputs + // TODO: move to gpt2_eval + ggml_backend_tensor_set(model.embd, embd_inp.data(), 0, N*ggml_element_size(embd)); + + struct ggml_tensor * position = ggml_view_1d(ctx, model.position, N, 0); + for (int i = 0; i < N; ++i) { + int32_t v = n_past + i; + ggml_backend_tensor_set(model.position, &v, i*sizeof(int32_t), sizeof(v)); + } + + const float KQ_scale = 1.0f/sqrtf(float(model.hparams.n_embd)/model.hparams.n_head); + + // wte + wpe + struct ggml_tensor * inpL = + ggml_add(ctx, + ggml_get_rows(ctx, model.wte, embd), + ggml_get_rows(ctx, model.wpe, position)); + ggml_set_name(inpL, "inpL"); + ggml_set_name(inpL->src[0], "wte"); + ggml_set_name(inpL->src[1], "wpe"); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + // [ 768, N] + cur = ggml_norm(ctx, inpL, hparams.eps); + ggml_format_name(cur, "l%d.norm", il); + + // cur = ln_1_g*cur + ln_1_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_1_g), + model.layers[il].ln_1_b); + ggml_format_name(cur, "l%d.ln_1_b", il); + ggml_format_name(cur->src[0], "l%d.ln_1_g", il); + } + + // attn + // [2304, 768] - model.layers[il].c_attn_attn_w + // [2304, 1] - model.layers[il].c_attn_attn_b + // [ 768, N] - cur (in) + // [2304, N] - cur (out) + // + // cur = attn_w*cur + attn_b + // [2304, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_attn_w, + cur); + ggml_format_name(cur, "l%d.attn_w", il); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_attn_b); + ggml_format_name(cur, "l%d.attn_b", il); + } + + // self-attention + { + struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd); + struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd); + struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd); + + ggml_format_name(Qcur, "l%d.Qcur", il); + ggml_format_name(Kcur, "l%d.Kcur", il); + ggml_format_name(Vcur, "l%d.Vcur", il); + + // store key and value to memory + if (N >= 1) { + struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + // [64, N, 12] + struct ggml_tensor * Q = + ggml_permute(ctx, + ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N), + 0, 2, 1, 3); + ggml_format_name(Q, "l%d.Q", il); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + // [64, n_past + N, 12] + struct ggml_tensor * K = + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + ggml_format_name(K, "l%d.K", il); + + // GG: flash attention + //struct ggml_tensor * V = + // ggml_cpy(ctx0, + // ggml_permute(ctx0, + // ggml_reshape_3d(ctx0, + // ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + // n_embd/n_head, n_head, n_past + N), + // 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head)); + + //struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true); + + // K * Q + // [n_past + N, N, 12] + struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q); + ggml_format_name(KQ, "l%d.KQ", il); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_scaled = ggml_scale(ctx, KQ, KQ_scale); + ggml_format_name(KQ_scaled, "l%d.KQ_scaled", il); + + // KQ_masked = mask_past(KQ_scaled) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past); + ggml_format_name(KQ_masked, "l%d.KQ_masked", il); + + // KQ = soft_max(KQ_masked) + // [n_past + N, N, 12] + struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked); + ggml_format_name(KQ_soft_max, "l%d.KQ_soft_max", il); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + // [n_past + N, 64, 12] + struct ggml_tensor * V_trans = + ggml_cont_3d(ctx, + ggml_permute(ctx, + ggml_reshape_3d(ctx, + ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd), + n_embd/n_head, n_head, n_past + N), + 1, 2, 0, 3), + n_past + N, n_embd/n_head, n_head); + + // KQV = transpose(V) * KQ_soft_max + // [64, N, 12] + struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max); + ggml_format_name(KQV, "l%d.KQV", il); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + // [64, 12, N] + struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3); + ggml_format_name(KQV_merged, "l%d.KQV_merged", il); + + // cur = KQV_merged.contiguous().view(n_embd, N) + // [768, N] + cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N); + ggml_format_name(cur, "l%d.KQV_merged_contiguous", il); + } + + // projection + // [ 768, 768] - model.layers[il].c_attn_proj_w + // [ 768, 1] - model.layers[il].c_attn_proj_b + // [ 768, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + { + cur = ggml_mul_mat(ctx, + model.layers[il].c_attn_proj_w, + cur); + ggml_format_name(cur, "l%d.attn_proj_w", il); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_attn_proj_b); + ggml_format_name(cur, "l%d.attn_proj_b", il); + } + + // add the input + cur = ggml_add(ctx, cur, inpL); + ggml_format_name(cur, "l%d.add", il); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx, inpFF, hparams.eps); + ggml_format_name(cur, "l%d.FFnorm", il); + + // cur = ln_2_g*cur + ln_2_b + // [ 768, N] + cur = ggml_add(ctx, + ggml_mul(ctx, + cur, + model.layers[il].ln_2_g), + model.layers[il].ln_2_b); + ggml_format_name(cur, "l%d.ln_2_b", il); + ggml_format_name(cur->src[0], "l%d.ln_2_g", il); + } + + // fully connected + // [3072, 768] - model.layers[il].c_mlp_fc_w + // [3072, 1] - model.layers[il].c_mlp_fc_b + // [ 768, N] - cur (in) + // [3072, N] - cur (out) + // + // cur = fc_w*cur + fc_b + // [3072, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_fc_w, + cur); + ggml_format_name(cur, "l%d.mlp_fc_w", il); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_fc_b); + ggml_format_name(cur, "l%d.mlp_fc_b", il); + + // GELU activation + // [3072, N] + cur = ggml_gelu(ctx, cur); + ggml_format_name(cur, "l%d.gelu", il); + + // projection + // [ 768, 3072] - model.layers[il].c_mlp_proj_w + // [ 768, 1] - model.layers[il].c_mlp_proj_b + // [3072, N] - cur (in) + // [ 768, N] - cur (out) + // + // cur = proj_w*cur + proj_b + // [768, N] + cur = ggml_mul_mat(ctx, + model.layers[il].c_mlp_proj_w, + cur); + ggml_format_name(cur, "l%d.mlp_proj_w", il); + + cur = ggml_add(ctx, + cur, + model.layers[il].c_mlp_proj_b); + ggml_format_name(cur, "l%d.mlp_proj_b", il); + } + + // input for next layer + inpL = ggml_add(ctx, cur, inpFF); + ggml_format_name(inpL, "l%d.add2", il); + } + + // norm + { + // [ 768, N] + inpL = ggml_norm(ctx, inpL, hparams.eps); + ggml_format_name(inpL, "out_norm"); + + // inpL = ln_f_g*inpL + ln_f_b + // [ 768, N] + inpL = ggml_add(ctx, + ggml_mul(ctx, + inpL, + model.ln_f_g), + model.ln_f_b); + ggml_format_name(inpL, "out_ln_f_b"); + ggml_format_name(inpL->src[0], "out_ln_f_g"); + } + + // inpL = WTE * inpL + // [ 768, 50257] - model.lm_head + // [ 768, N] - inpL + inpL = ggml_mul_mat(ctx, model.lm_head, inpL); + ggml_format_name(inpL, "out_lm_head"); + + // logits -> probs + //inpL = ggml_soft_max(ctx0, inpL); + + ggml_build_forward_expand(gf, inpL); + + ggml_free(ctx); + + return gf; +} + +// evaluate the transformer +// +// - model: the model +// - sched: the backend scheduler +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +bool gpt2_eval( + const gpt2_model & model, + ggml_backend_sched_t sched, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_vocab = hparams.n_vocab; + + struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp); + + // run the computation + ggml_backend_sched_reset(sched); + ggml_backend_sched_graph_compute(sched, gf); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot"); + //} + + // in this case, the output tensor is the last one in the graph + struct ggml_tensor * inpL = ggml_graph_node(gf, -1); + + //embd_w.resize(n_vocab*N); + //ggml_backend_tensor_get(inpL, embd_w.data(), 0, sizeof(float)*n_vocab*N); + + // return result just for the last token + embd_w.resize(n_vocab); + ggml_backend_tensor_get(inpL, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + params.model = "models/gpt-2-117M/ggml-model.bin"; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gpt2_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_load(params.model, model, vocab, params)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + // create the backend scheduler + // the scheduler handles the allocation of the compute buffers and the scheduling of the computation between the different backends + ggml_backend_sched_t sched; + { + // initialize the scheduler + sched = ggml_backend_sched_new(model.backends.data(), NULL, model.backends.size(), GPT2_MAX_NODES, false, true); + + // create the worst case graph for memory usage estimation + int n_tokens = std::min(model.hparams.n_ctx, params.n_batch); + int n_past = model.hparams.n_ctx - n_tokens; + struct ggml_cgraph * gf = gpt2_graph(model, n_past, std::vector(n_tokens, 0)); + + ggml_backend_sched_reserve(sched, gf); + + + // compute the required memory + size_t mem_size = 0; + for (size_t i = 0; i < model.backends.size(); i++) { + size_t size = ggml_backend_sched_get_buffer_size(sched, model.backends[i]); + if (size > 0) { + mem_size += size; + printf("%s: %8s compute buffer size = %8.2f MB\n", __func__, ggml_backend_name(model.backends[i]), size/1024.0/1024.0); + //printf("%s: %8s compute buffer size = %zu bytes\n", __func__, ggml_backend_name(model.backends[i]), size); + } + } + + printf("%s: total compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0); + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str()); + printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size()); + for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) { + printf("%d ", embd_inp[i]); + } + printf("\n\n"); + + // submit the input prompt token-by-token + // this reduces the memory usage during inference, at the cost of a bit of speed at the beginning + std::vector embd; + + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_eval(model, sched, n_past, embd, logits)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (size_t k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (int32_t(embd.size()) >= params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (embd.back() == 50256) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx_w); + + ggml_backend_sched_free(sched); + ggml_backend_buffer_free(model.buffer_kv); + for (auto buf : model.buffers_w) { + ggml_backend_buffer_free(buf); + } + for (auto backend : model.backends) { + ggml_backend_free(backend); + } + + return 0; +} diff --git a/ggml/examples/gpt-2/quantize.cpp b/ggml/examples/gpt-2/quantize.cpp new file mode 100644 index 0000000..f81c04e --- /dev/null +++ b/ggml/examples/gpt-2/quantize.cpp @@ -0,0 +1,184 @@ +#include "ggml.h" + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// default hparams (GPT-2 117M) +struct gpt2_hparams { + int32_t n_vocab = 50257; + int32_t n_ctx = 1024; + int32_t n_embd = 768; + int32_t n_head = 12; + int32_t n_layer = 12; + int32_t ftype = 1; +}; + +// quantize a model +bool gpt2_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) { + gpt_vocab vocab; + + printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str()); + + auto finp = std::ifstream(fname_inp, std::ios::binary); + if (!finp) { + fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str()); + return false; + } + + auto fout = std::ofstream(fname_out, std::ios::binary); + if (!fout) { + fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + finp.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str()); + return false; + } + + fout.write((char *) &magic, sizeof(magic)); + } + + gpt2_hparams hparams; + + // load hparams + { + finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + finp.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + finp.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR; + const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype (src) = %d\n", __func__, hparams.ftype); + printf("%s: qntvr (src) = %d\n", __func__, qntvr_src); + printf("%s: ftype (dst) = %d\n", __func__, ftype_dst); + printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION); + + fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fout.write((char *) &hparams.n_head, sizeof(hparams.n_head)); + fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fout.write((char *) &ftype_dst, sizeof(ftype_dst)); + } + + // load vocab + { + int32_t n_vocab = 0; + finp.read ((char *) &n_vocab, sizeof(n_vocab)); + fout.write((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab); + return false; + } + + std::string word; + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + finp.read ((char *) &len, sizeof(len)); + fout.write((char *) &len, sizeof(len)); + + word.resize(len); + finp.read ((char *) word.data(), len); + fout.write((char *) word.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // regexes of tensor names to be quantized + const std::vector to_quant = { + "model/wte", + "model/lm_head", + "model/h.*/attn/c_attn/w", + "model/h.*/attn/c_proj/w", + "model/h.*/mlp/c_fc/w", + "model/h.*/mlp/c_proj/w", + }; + + if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) { + fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str()); + return false; + } + + finp.close(); + fout.close(); + + return true; +} + +// usage: +// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type +// +int main(int argc, char ** argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]); + ggml_print_ftypes(stderr); + return 1; + } + + // needed to initialize f16 tables + { + struct ggml_init_params params = { 0, NULL, false }; + struct ggml_context * ctx = ggml_init(params); + ggml_free(ctx); + } + + const std::string fname_inp = argv[1]; + const std::string fname_out = argv[2]; + + const ggml_ftype ftype = ggml_parse_ftype(argv[3]); + + const int64_t t_main_start_us = ggml_time_us(); + + int64_t t_quantize_us = 0; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gpt2_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) { + fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str()); + return 1; + } + + t_quantize_us = ggml_time_us() - t_start_us; + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n"); + printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + return 0; +} diff --git a/ggml/examples/gpt-j/CMakeLists.txt b/ggml/examples/gpt-j/CMakeLists.txt new file mode 100644 index 0000000..3675b7d --- /dev/null +++ b/ggml/examples/gpt-j/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# gpt-j + +set(TEST_TARGET gpt-j) +add_executable(${TEST_TARGET} main.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +# +# gpt-j-quantize + +set(TEST_TARGET gpt-j-quantize) +add_executable(${TEST_TARGET} quantize.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) diff --git a/ggml/examples/gpt-j/README.md b/ggml/examples/gpt-j/README.md new file mode 100644 index 0000000..d63458b --- /dev/null +++ b/ggml/examples/gpt-j/README.md @@ -0,0 +1,239 @@ +# gpt-j + +Local GPT-J inference on your computer using C/C++ + +No video card required. You just need to have 16 GB of RAM. + +## Motivation + +The GPT-J 6B model is the open-source alternative to OpenAI's GPT-3. It's basically a neural network that allows you to +generate coherent, human-like text given a certain context (prompt). + +The GPT-J model is quite big - the compact version of the model uses 16-bit floating point representation of the weights +and is still 12 GB big. This means that in order to run inference on your computer, you would need to have a video card +with at least 12 GB of video RAM. Alternatively, you can try to run the python implementations on the CPU, but that +would probably not be very efficient as they are primarily optimized for running on a GPU (or at least this is my guess - +I don't have much experience with python). + +I wanted to try and run the model on my MacBook, so I decided to implement the model inference from scratch using my own +custom build tensor library. The tensor library (called [ggml](https://github.com/ggerganov/ggml), written in C) is in +early development stage, but it already allows me to run the GPT-J model. + +On my 32GB MacBook M1 Pro, I achieve an inference speed of about `125 ms/token` or about ~6 words per second (1 word +typically consists of 1 or 2 tokens). + +Here is a sample run with prompt `int main(int argc, char ** argv) {`: + +```bash +$ time ./bin/gpt-j -p "int main(int argc, char ** argv) {" + +gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ... +gptj_model_load: n_vocab = 50400 +gptj_model_load: n_ctx = 2048 +gptj_model_load: n_embd = 4096 +gptj_model_load: n_head = 16 +gptj_model_load: n_layer = 28 +gptj_model_load: n_rot = 64 +gptj_model_load: f16 = 1 +gptj_model_load: ggml ctx size = 13334.86 MB +gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344 +gptj_model_load: ................................... done +gptj_model_load: model size = 11542.79 MB / num tensors = 285 +main: number of tokens in prompt = 13 + +int main(int argc, char ** argv) { + (void)argc; + (void)argv; + + { + struct sockaddr_in addr; + int addrlen; + char * ip = "192.168.1.4"; + int i; + + if ( (addrlen = sizeof(addr)) == -1 ) + return -1; + + for (i = 0; i < 10; ++i) { + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = inet_addr(ip); + +main: mem per token = 16430420 bytes +main: load time = 6211.48 ms +main: sample time = 13.74 ms +main: predict time = 26420.34 ms / 124.62 ms per token +main: total time = 33035.37 ms + +real 0m33.171s +user 3m32.269s +sys 0m3.686s + +$ +``` + +It took ~6.2 seconds to load the model to memory. After that, it took ~26.4 seconds to generate 200 tokens of what +looks like to be the beginning of a networking program in C. Pretty cool! + +Here is another run, just for fun: + +```bash +time ./bin/gpt-j -n 500 -t 8 -p "Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it? +" + +gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ... +gptj_model_load: n_vocab = 50400 +gptj_model_load: n_ctx = 2048 +gptj_model_load: n_embd = 4096 +gptj_model_load: n_head = 16 +gptj_model_load: n_layer = 28 +gptj_model_load: n_rot = 64 +gptj_model_load: f16 = 1 +gptj_model_load: ggml ctx size = 13334.86 MB +gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344 +gptj_model_load: ................................... done +gptj_model_load: model size = 11542.79 MB / num tensors = 285 +main: number of tokens in prompt = 24 + +Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it? + +I've inherited a team with some very strange and un-documented practices, one of them is that they use an old custom +application with a very slow tech stack written in Python that the team doesn't want to touch but also doesn't want to +throw away as it has some "legacy" code in it. + +The problem is, the tech stack is very very slow. + +They have a single web server on a VM that is slow. +The server is a little bit busy (not very busy though) and they have a lot of processes (30+ that are constantly being +spawned by the application) +They have an application that is single threaded and was written in Python and the team don't want to touch this, and +the application is very slow. + +My task as a new member of the team is to fix this. + +I'm a senior dev on the team (3 years on the project) and have been told that I will take the lead on this task. I know +next to nothing about Python. So here is what I have so far. + +What I have done is I've been trying to debug the processes with the "ps" command. This way I can see what is running +and where. From what I see, the application spawns 10 processes a minute and some of them are used for nothing. + +I have also started to look for the code. The application source is not in GitHub or any other repository, it is only on +our internal GitLab. + +What I've found so far: + +The application uses a custom SQLAlchemy implementation to interact with the data. I've looked at the source, it looks +like an object cache or something like that. But from what I've seen, the cache gets full every 20 minutes and then gets +cleared with a special command. + +Another strange thing is that the application creates a file for every entry in the database (even if the entry already +exists). I've looked at the file to see if it contains something, but it seems to be a JSON file with lots of records. + +The other strange thing is that I can only find the database tables in the GitLab repository and not the code. So I +can't really understand how the application is supposed to interact with the database. + +I also found a "log" directory, but the code is encrypted with AES. From what I've found, it is in + +main: mem per token = 16430420 bytes +main: load time = 3900.10 ms +main: sample time = 32.58 ms +main: predict time = 68049.91 ms / 130.11 ms per token +main: total time = 73020.05 ms + +real 1m13.156s +user 9m1.328s +sys. 0m7.103s +``` + +## Implementation details + +The high level implementation of the model is contained in the [main.cpp](main.cpp) file. The core computations are +performed by the [ggml](https://github.com/ggerganov/ggml/blob/master/include/ggml.h) library. + + +#### Matrix multiplication + +The most performance critical part of the implementation is of course the matrix multiplication routine. 99% of the time +is spent here, so it was important to optimize this as much as possible. + +On Arm64, I utilize the 128-bit NEON intrinsics for 16-bit floating point operations: + +https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/src/ggml.c#L187-L243 + +These instructions allow each core to operate simultaneously on 64 16-bit floats. I'm no expert in SIMD, but after quite +some trials this was the most efficient code for dot product of a row and column that I could come up with. Combined +with the parallel computation on 8 CPU threads, I believe I'm close to the maximum performance that one could possibly +get on the M1 CPU. Still, I'm curious to know if there is a more efficient way to implement this. + + +#### Attempt to use the M1 GPU + +One interesting property of the GPT-J transformer architecture is that it allows you to perform part of the inference in +parallel - i.e. the Feed-forward network can be computed in parallel to the Self-attention layer: + +https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/examples/gpt-j/main.cpp#L507-L531 + +So I thought why not try and bring in the M1 GPU to compute half of the neural network in parallel to the CPU and +potentially gain some extra performance. Thanks to the M1's shared memory model, it was relatively easy to offload part +of the computation to the GPU using Apple's [Metal Performance +Shaders](https://developer.apple.com/documentation/metalperformanceshaders). The GPU shares the host memory, so there is +no need to copy the data back and forth as you would normally do with Cuda or OpenCL. The weight matrices are directly +available to be used by the GPU. + +However, to my surprise, using MPS together with the CPU did not lead to any performance improvement at all. My +conclusion was that the 8-thread NEON CPU computation is already saturating the memory bandwidth of the M1 and since +the CPU and the GPU on the MacBook are sharing that bandwidth, it does not help to offload the computation to the GPU. +Another observation was that the MPS GPU matrix multiplication using 16-bit floats had the same performance as the +8-thread NEON CPU implementation. Again, I explain this with a saturated memory channel. But of course, my explanation +could be totally wrong and somehow the implementation wasn't utilizing the resources correctly. + +In the end, I decided to not use MPS or the GPU all together. + +### Zero memory allocations + +Another property of my implementation is that it does not perform any memory allocations once the model is loaded into +memory. All required memory is allocated at the start of the program with a single `malloc` (technically 2 calls, but +that is not important). + +## Usage + +If you want to give this a try and you are on Linux or Mac OS, simply follow these instructions: + +```bash +# Download the ggml-compatible GPT-J 6B model (requires 12GB disk space) +../examples/gpt-j/download-ggml-model.sh 6B + +# Run the inference (requires 16GB of CPU RAM) +./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin -p "This is an example" + +# Input prompt through pipe and run the inference. +echo "This is an example" > prompt.txt +cat prompt.txt | ./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin +``` + +To run the `gpt-j` tool, you need the 12GB `ggml-model.bin` file which contains the GPT-J model in +[ggml](https://github.com/ggerganov/ggml) compatible format. In the instructions above, the binary file +is downloaded from my repository on Hugging Face using the [download-ggml-model.sh](download-ggml-model.sh) script. +You can also, download the file manually from this link: + +https://huggingface.co/ggerganov/ggml/tree/main + +--- + +Alternatively, if you don't want to download the 12GB ggml model file, you can perform the conversion yourself using +python. + +First, you need to download the full GPT-J model from here: https://huggingface.co/EleutherAI/gpt-j-6B + +Note that the full model is quite big - about 72 GB. After you download it, you need to convert it to ggml format using +the [convert-h5-to-ggml.py](convert-h5-to-ggml.py) script. This will generate the `ggml-model.bin` file, which you can +then use with the `gpt-j` program. + + +## GPT-2 + +I also implemented a tool for CPU inference using the smaller GPT-2 models. They have worse quality compared to GPT-J, +but are much faster to execute. + +For example, the Small GPT-2 model is only 240 MB big and the inference speed on my MacBook is about 200 tokens/sec. + +For more details, checkout the GPT-2 example here: [gpt-2](https://github.com/ggerganov/ggml/tree/master/examples/gpt-2) diff --git a/ggml/examples/gpt-j/convert-h5-to-ggml.py b/ggml/examples/gpt-j/convert-h5-to-ggml.py new file mode 100644 index 0000000..cb77317 --- /dev/null +++ b/ggml/examples/gpt-j/convert-h5-to-ggml.py @@ -0,0 +1,173 @@ +# Convert GPT-J-6B h5 transformer model to ggml format +# +# Load the model using GPTJForCausalLM. +# Iterate over all variables and write them to a binary file. +# +# For each variable, write the following: +# - Number of dimensions (int) +# - Name length (int) +# - Dimensions (int[n_dims]) +# - Name (char[name_length]) +# - Data (float[n_dims]) +# +# By default, the bigger matrices are converted to 16-bit floats. +# This can be disabled by adding the "use-f32" CLI argument. +# +# At the start of the ggml file we write the model parameters +# and vocabulary. +# + +import sys +import struct +import json +import torch +import numpy as np + +from transformers import GPTJForCausalLM + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +if len(sys.argv) < 3: + print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + +with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f: + encoder = json.load(f) + +with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f: + encoder_added = json.load(f) + +with open(dir_model + "/config.json", "r", encoding="utf-8") as f: + hparams = json.load(f) + +# possible data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 2: + ftype = int(sys.argv[2]) + if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin" + + +model = GPTJForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True) +#print (model) + +list_vars = model.state_dict() +#print (list_vars) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["vocab_size"])) +fout.write(struct.pack("i", hparams["n_positions"])) +fout.write(struct.pack("i", hparams["n_embd"])) +fout.write(struct.pack("i", hparams["n_head"])) +fout.write(struct.pack("i", hparams["n_layer"])) +fout.write(struct.pack("i", hparams["rotary_dim"])) +fout.write(struct.pack("i", ftype)) + +byte_encoder = bytes_to_unicode() +byte_decoder = {v:k for k, v in byte_encoder.items()} + +fout.write(struct.pack("i", len(encoder) + len(encoder_added))) + +for key in encoder: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for key in encoder_added: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Processing variable: " + name + " with shape: ", data.shape) + + # we don't need these + if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"): + print(" Skipping variable: " + name) + continue + + n_dims = len(data.shape); + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype_cur = 0; + if ftype != 0: + if name[-7:] == ".weight" and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype_cur = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + else: + if data.dtype != np.float32: + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + + # for efficiency - transpose these matrices: + # (note - with latest ggml this is no longer more efficient, so disabling it) + # "transformer.h.*.mlp.fc_in.weight" + # "transformer.h.*.attn.out_proj.weight" + # "transformer.h.*.attn.q_proj.weight" + # "transformer.h.*.attn.k_proj.weight" + # "transformer.h.*.attn.v_proj.weight" + #if name.endswith(".mlp.fc_in.weight") or \ + # name.endswith(".attn.out_proj.weight") or \ + # name.endswith(".attn.q_proj.weight") or \ + # name.endswith(".attn.k_proj.weight") or \ + # name.endswith(".attn.v_proj.weight"): + # print(" Transposing") + # data = data.transpose() + + # header + str = name.encode('utf-8') + fout.write(struct.pack("iii", n_dims, len(str), ftype_cur)) + for i in range(n_dims): + fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) + fout.write(str); + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/ggml/examples/gpt-j/download-ggml-model.sh b/ggml/examples/gpt-j/download-ggml-model.sh new file mode 100755 index 0000000..a9e2aa5 --- /dev/null +++ b/ggml/examples/gpt-j/download-ggml-model.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# This script downloads GPT-J model files that have already been converted to ggml format. +# This way you don't have to convert them yourself. +# +# If you want to download the original GPT-J model files, use the "download-model.sh" script instead. + +#src="https://ggml.ggerganov.com" +#pfx="ggml-model-gpt-j" + +src="https://huggingface.co/ggerganov/ggml" +pfx="resolve/main/ggml-model-gpt-j" + +ggml_path=$(dirname $(realpath $0)) + +# GPT-J models +models=( "6B" ) + +# list available models +function list_models { + printf "\n" + printf " Available models:" + for model in "${models[@]}"; do + printf " $model" + done + printf "\n\n" +} + +if [ "$#" -ne 1 ]; then + printf "Usage: $0 \n" + list_models + + exit 1 +fi + +model=$1 + +if [[ ! " ${models[@]} " =~ " ${model} " ]]; then + printf "Invalid model: $model\n" + list_models + + exit 1 +fi + +# download ggml model + +printf "Downloading ggml model $model ...\n" + +mkdir -p models/gpt-j-$model + +if [ -x "$(command -v wget)" ]; then + wget --quiet --show-progress -O models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin +elif [ -x "$(command -v curl)" ]; then + curl -L --output models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin +else + printf "Either wget or curl is required to download models.\n" + exit 1 +fi + +if [ $? -ne 0 ]; then + printf "Failed to download ggml model $model \n" + printf "Please try again later or download the original GPT-J model files and convert them yourself.\n" + exit 1 +fi + +printf "Done! Model '$model' saved in 'models/gpt-j-$model/ggml-model.bin'\n" +printf "You can now use it like this:\n\n" +printf " $ ./bin/gpt-j -m models/gpt-j-$model/ggml-model.bin -p \"This is an example\"\n" +printf "\n" diff --git a/ggml/examples/gpt-j/download-model.sh b/ggml/examples/gpt-j/download-model.sh new file mode 100755 index 0000000..c773baf --- /dev/null +++ b/ggml/examples/gpt-j/download-model.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +printf "To obtain the GPT-J 6B model files, please visit: https://huggingface.co/EleutherAI/gpt-j-6B\n\n" + +printf "The model is very big. For example, the reposirory above is 72GB in size.\n" +printf "If you are sure that you want to clone it, simply run the following command:\n\n" + +printf " $ git clone https://huggingface.co/EleutherAI/gpt-j-6B models/gpt-j-6B\n\n" + +printf "Alternatively, use the 'download-ggml-model.sh' script to download a 12GB ggml version of the model.\n" +printf "This version is enough to run inference using the ggml library.\n\n" diff --git a/ggml/examples/gpt-j/main.cpp b/ggml/examples/gpt-j/main.cpp new file mode 100644 index 0000000..2c17577 --- /dev/null +++ b/ggml/examples/gpt-j/main.cpp @@ -0,0 +1,755 @@ +#include "ggml.h" +#include "ggml-cpu.h" + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + + +// default hparams (GPT-J 6B) +struct gptj_hparams { + int32_t n_vocab = 50400; + int32_t n_ctx = 2048; + int32_t n_embd = 4096; + int32_t n_head = 16; + int32_t n_layer = 28; + int32_t n_rot = 64; + int32_t ftype = 1; + float eps = 1e-5f; +}; + +struct gptj_layer { + // normalization + struct ggml_tensor * ln_1_g; + struct ggml_tensor * ln_1_b; + + // attention + struct ggml_tensor * c_attn_q_proj_w; + struct ggml_tensor * c_attn_k_proj_w; + struct ggml_tensor * c_attn_v_proj_w; + + struct ggml_tensor * c_attn_proj_w; + + // ff + struct ggml_tensor * c_mlp_fc_w; + struct ggml_tensor * c_mlp_fc_b; + + struct ggml_tensor * c_mlp_proj_w; + struct ggml_tensor * c_mlp_proj_b; +}; + +struct gptj_model { + gptj_hparams hparams; + + // normalization + struct ggml_tensor * ln_f_g; + struct ggml_tensor * ln_f_b; + + struct ggml_tensor * wte; // token embedding + + struct ggml_tensor * lmh_g; // language model head + struct ggml_tensor * lmh_b; // language model bias + + std::vector layers; + + // key + value memory + struct ggml_tensor * memory_k; + struct ggml_tensor * memory_v; + + // + struct ggml_context * ctx; + std::map tensors; +}; + +// load the model's weights from a file +bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) { + printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str()); + + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str()); + return false; + } + } + + // load hparams + { + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fin.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: n_rot = %d\n", __func__, hparams.n_rot); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + } + + // load vocab + { + int32_t n_vocab = 0; + fin.read((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != model.hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname.c_str(), n_vocab, model.hparams.n_vocab); + return false; + } + + std::string word; + std::vector buf(128); + + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + fin.read((char *) &len, sizeof(len)); + + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, fname.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx; + + size_t ctx_size = 0; + + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_vocab = hparams.n_vocab; + + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g + ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b + + ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // wte + + ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // lmh_g + ctx_size += ggml_row_size(GGML_TYPE_F32, n_vocab); // lmh_b + + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b + + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_q_proj_w + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_k_proj_w + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_v_proj_w + + ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b + + ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w + ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_mlp_proj_b + + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_k + ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_v + + ctx_size += (5 + 10*n_layer)*512; // object overhead + + printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); + } + + // create the ggml context + { + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + model.ctx = ggml_init(params); + if (!model.ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // prepare memory for the weights + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_vocab = hparams.n_vocab; + + model.layers.resize(n_layer); + + model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + + model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + model.lmh_g = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab); + model.lmh_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab); + + // map by name + model.tensors["transformer.wte.weight"] = model.wte; + + model.tensors["transformer.ln_f.weight"] = model.ln_f_g; + model.tensors["transformer.ln_f.bias"] = model.ln_f_b; + + model.tensors["lm_head.weight"] = model.lmh_g; + model.tensors["lm_head.bias"] = model.lmh_b; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = model.layers[i]; + + layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + + layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd); + + layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd); + layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd); + + layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd); + layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); + + // map by name + model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"] = layer.ln_1_g; + model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"] = layer.ln_1_b; + + model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"] = layer.c_attn_q_proj_w; + model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"] = layer.c_attn_k_proj_w; + model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"] = layer.c_attn_v_proj_w; + + model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w; + + model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"] = layer.c_mlp_fc_w; + model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"] = layer.c_mlp_fc_b; + + model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"] = layer.c_mlp_proj_w; + model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"] = layer.c_mlp_proj_b; + } + } + + // key + value memory + { + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + + const int n_mem = n_layer*n_ctx; + const int n_elements = n_embd*n_mem; + + model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements); + model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements); + + const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v); + + printf("%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem); + } + + // load weights + { + int n_tensors = 0; + size_t total_size = 0; + + printf("%s: ", __func__); + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ttype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ttype), sizeof(ttype)); + + if (fin.eof()) { + break; + } + + int32_t nelements = 1; + int32_t ne[2] = { 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + fin.read(reinterpret_cast(&ne[i]), sizeof(ne[i])); + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str()); + return false; + } + + auto tensor = model.tensors[name]; + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str()); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n", + __func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]); + return false; + } + + // for debugging + if (0) { + printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor)); + } + + const size_t bpe = ggml_type_size(ggml_type(ttype)); + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe); + return false; + } + + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + + //printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.c_str(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0); + total_size += ggml_nbytes(tensor); + if (++n_tensors % 8 == 0) { + printf("."); + fflush(stdout); + } + } + + printf(" done\n"); + + printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors); + } + + fin.close(); + + return true; +} + +// evaluate the transformer +// +// - model: the model +// - n_threads: number of threads to use +// - n_past: the context size so far +// - embd_inp: the embeddings of the tokens in the context +// - embd_w: the predicted logits for the next token +// +// The GPT-J model requires about 16MB of memory per input token. +// +bool gptj_eval( + const gptj_model & model, + const int n_threads, + const int n_past, + const std::vector & embd_inp, + std::vector & embd_w, + size_t & mem_per_token) { + const int N = embd_inp.size(); + + const auto & hparams = model.hparams; + + const int n_embd = hparams.n_embd; + const int n_layer = hparams.n_layer; + const int n_ctx = hparams.n_ctx; + const int n_head = hparams.n_head; + const int n_vocab = hparams.n_vocab; + const int n_rot = hparams.n_rot; + + static size_t buf_size = 256u*1024*1024; + static void * buf = malloc(buf_size); + + if (mem_per_token > 0 && mem_per_token*N > buf_size) { + const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead + //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); + + // reallocate + buf_size = buf_size_new; + buf = realloc(buf, buf_size); + if (buf == nullptr) { + fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size); + return false; + } + } + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf, + /*.no_alloc =*/ false, + }; + + struct ggml_context * ctx0 = ggml_init(params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + // KQ_pos - contains the positions + struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + int * data = (int *) KQ_pos->data; + for (int i = 0; i < N; ++i) { + data[i] = n_past + i; + } + + struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N); + memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd)); + + // wte + struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd); + + for (int il = 0; il < n_layer; ++il) { + struct ggml_tensor * cur; + + // norm + { + cur = ggml_norm(ctx0, inpL, hparams.eps); + + // cur = ln_1_g*cur + ln_1_b + cur = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.layers[il].ln_1_g, cur), + cur), + ggml_repeat(ctx0, model.layers[il].ln_1_b, cur)); + } + + struct ggml_tensor * inpSA = cur; + + // self-attention + { + struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0); + struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0); + + // store key and value to memory + { + struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur)); + + struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past)); + struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd, + ( n_ctx)*ggml_element_size(model.memory_v), + (il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v)); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v)); + } + + // Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3) + struct ggml_tensor * Q = + ggml_permute(ctx0, + Qcur, + 0, 2, 1, 3); + + // K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3) + struct ggml_tensor * K = + ggml_permute(ctx0, + ggml_reshape_3d(ctx0, + ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd), + n_embd/n_head, n_head, n_past + N), + 0, 2, 1, 3); + + // K * Q + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + + // KQ_scaled = KQ / sqrt(n_embd/n_head) + struct ggml_tensor * KQ_scaled = + ggml_scale_inplace(ctx0, + KQ, + 1.0f/sqrt(float(n_embd)/n_head)); + + // KQ_masked = mask_past(KQ_scaled) + struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past); + + // KQ = soft_max(KQ_masked) + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked); + + // V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous() + struct ggml_tensor * V = + ggml_view_3d(ctx0, model.memory_v, + n_past + N, n_embd/n_head, n_head, + n_ctx*ggml_element_size(model.memory_v), + n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head, + il*n_ctx*ggml_element_size(model.memory_v)*n_embd); + + // KQV = transpose(V) * KQ_soft_max + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); + + // KQV_merged = KQV.permute(0, 2, 1, 3) + struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3); + + // cur = KQV_merged.contiguous().view(n_embd, N) + cur = ggml_cpy(ctx0, + KQV_merged, + ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N)); + + // projection (no bias) + cur = ggml_mul_mat(ctx0, + model.layers[il].c_attn_proj_w, + cur); + } + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + // this is independent of the self-attention result, so it could be done in parallel to the self-attention + { + // note here we pass inpSA instead of cur + cur = ggml_mul_mat(ctx0, + model.layers[il].c_mlp_fc_w, + inpSA); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur), + cur); + + // GELU activation + cur = ggml_gelu(ctx0, cur); + + // projection + // cur = proj_w*cur + proj_b + cur = ggml_mul_mat(ctx0, + model.layers[il].c_mlp_proj_w, + cur); + + cur = ggml_add(ctx0, + ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur), + cur); + } + + // self-attention + FF + cur = ggml_add(ctx0, cur, inpFF); + + // input for next layer + inpL = ggml_add(ctx0, cur, inpL); + } + + // norm + { + inpL = ggml_norm(ctx0, inpL, hparams.eps); + + // inpL = ln_f_g*inpL + ln_f_b + inpL = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, model.ln_f_g, inpL), + inpL), + ggml_repeat(ctx0, model.ln_f_b, inpL)); + } + + // lm_head + { + inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL); + + inpL = ggml_add(ctx0, + ggml_repeat(ctx0, model.lmh_b, inpL), + inpL); + } + + // logits -> probs + //inpL = ggml_soft_max_inplace(ctx0, inpL); + + // run the computation + ggml_build_forward_expand(gf, inpL); + ggml_graph_compute_with_ctx(ctx0, gf, n_threads); + + //if (n_past%100 == 0) { + // ggml_graph_print (&gf); + // ggml_graph_dump_dot(&gf, NULL, "gpt-j.dot"); + //} + + //embd_w.resize(n_vocab*N); + //memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N); + + // return result for just the last token + embd_w.resize(n_vocab); + memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab); + + if (mem_per_token == 0) { + mem_per_token = ggml_used_mem(ctx0)/N; + } + //printf("used_mem = %zu\n", ggml_used_mem(ctx0)); + + ggml_free(ctx0); + + return true; +} + +int main(int argc, char ** argv) { + ggml_time_init(); + + const int64_t t_main_start_us = ggml_time_us(); + + gpt_params params; + params.model = "models/gpt-j-6B/ggml-model.bin"; + + if (gpt_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + + printf("%s: seed = %d\n", __func__, params.seed); + + std::mt19937 rng(params.seed); + if (params.prompt.empty()) { + params.prompt = gpt_random_prompt(rng); + } + + int64_t t_load_us = 0; + + gpt_vocab vocab; + gptj_model model; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gptj_model_load(params.model, model, vocab)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + + test_gpt_tokenizer(vocab, params.token_test); + } + + int n_past = 0; + + int64_t t_sample_us = 0; + int64_t t_predict_us = 0; + + std::vector logits; + + // tokenize the prompt + std::vector embd_inp = ::gpt_tokenize(vocab, params.prompt); + + params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size()); + + printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size()); + printf("\n"); + + std::vector embd; + + // determine the required inference memory per token: + size_t mem_per_token = 0; + gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token); + + for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) { + // predict + if (embd.size() > 0) { + const int64_t t_start_us = ggml_time_us(); + + if (!gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) { + printf("Failed to predict\n"); + return 1; + } + + t_predict_us += ggml_time_us() - t_start_us; + } + + n_past += embd.size(); + embd.clear(); + + if (i >= embd_inp.size()) { + // sample next token + const int top_k = params.top_k; + const float top_p = params.top_p; + const float temp = params.temp; + + const int n_vocab = model.hparams.n_vocab; + + gpt_vocab::id id = 0; + + { + const int64_t t_start_sample_us = ggml_time_us(); + + id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng); + + t_sample_us += ggml_time_us() - t_start_sample_us; + } + + // add it to the context + embd.push_back(id); + } else { + // if here, it means we are still processing the input prompt + for (size_t k = i; k < embd_inp.size(); k++) { + embd.push_back(embd_inp[k]); + if (int32_t(embd.size()) > params.n_batch) { + break; + } + } + i += embd.size() - 1; + } + + // display text + for (auto id : embd) { + printf("%s", vocab.id_to_token[id].c_str()); + } + fflush(stdout); + + // end of text token + if (embd.back() == 50256) { + break; + } + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n\n"); + printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token); + printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f); + printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx); + + return 0; +} diff --git a/ggml/examples/gpt-j/quantize.cpp b/ggml/examples/gpt-j/quantize.cpp new file mode 100644 index 0000000..c6f258c --- /dev/null +++ b/ggml/examples/gpt-j/quantize.cpp @@ -0,0 +1,182 @@ +#include "ggml.h" + +#include "common.h" +#include "common-ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// default hparams (GPT-J 6B) +struct gptj_hparams { + int32_t n_vocab = 50400; + int32_t n_ctx = 2048; + int32_t n_embd = 4096; + int32_t n_head = 16; + int32_t n_layer = 28; + int32_t n_rot = 64; + int32_t ftype = 1; +}; + +// quantize a model +bool gptj_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) { + gpt_vocab vocab; + + printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str()); + + auto finp = std::ifstream(fname_inp, std::ios::binary); + if (!finp) { + fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str()); + return false; + } + + auto fout = std::ofstream(fname_out, std::ios::binary); + if (!fout) { + fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + finp.read((char *) &magic, sizeof(magic)); + if (magic != GGML_FILE_MAGIC) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str()); + return false; + } + + fout.write((char *) &magic, sizeof(magic)); + } + + gptj_hparams hparams; + + // load hparams + { + finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + finp.read((char *) &hparams.n_head, sizeof(hparams.n_head)); + finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + finp.read((char *) &hparams.n_rot, sizeof(hparams.n_rot)); + finp.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR; + const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype; + + printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab); + printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx); + printf("%s: n_embd = %d\n", __func__, hparams.n_embd); + printf("%s: n_head = %d\n", __func__, hparams.n_head); + printf("%s: n_layer = %d\n", __func__, hparams.n_layer); + printf("%s: ftype (src) = %d\n", __func__, hparams.ftype); + printf("%s: qntvr (src) = %d\n", __func__, qntvr_src); + printf("%s: ftype (dst) = %d\n", __func__, ftype_dst); + printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION); + + fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab)); + fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx)); + fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd)); + fout.write((char *) &hparams.n_head, sizeof(hparams.n_head)); + fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer)); + fout.write((char *) &hparams.n_rot, sizeof(hparams.n_rot)); + fout.write((char *) &ftype_dst, sizeof(ftype_dst)); + } + + // load vocab + { + int32_t n_vocab = 0; + finp.read ((char *) &n_vocab, sizeof(n_vocab)); + fout.write((char *) &n_vocab, sizeof(n_vocab)); + + if (n_vocab != hparams.n_vocab) { + fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n", + __func__, fname_inp.c_str(), n_vocab, hparams.n_vocab); + return false; + } + + std::string word; + for (int i = 0; i < n_vocab; i++) { + uint32_t len; + finp.read ((char *) &len, sizeof(len)); + fout.write((char *) &len, sizeof(len)); + + word.resize(len); + finp.read ((char *) word.data(), len); + fout.write((char *) word.data(), len); + + vocab.token_to_id[word] = i; + vocab.id_to_token[i] = word; + } + } + + // regexes of tensor names to be quantized + const std::vector to_quant = { + ".*weight", + }; + + if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) { + fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str()); + return false; + } + + finp.close(); + fout.close(); + + return true; +} + +// usage: +// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type +// +int main(int argc, char ** argv) { + if (argc != 4) { + fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]); + ggml_print_ftypes(stderr); + return 1; + } + + // needed to initialize f16 tables + { + struct ggml_init_params params = { 0, NULL, false }; + struct ggml_context * ctx = ggml_init(params); + ggml_free(ctx); + } + + const std::string fname_inp = argv[1]; + const std::string fname_out = argv[2]; + + const ggml_ftype ftype = ggml_parse_ftype(argv[3]); + + const int64_t t_main_start_us = ggml_time_us(); + + int64_t t_quantize_us = 0; + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!gptj_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) { + fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str()); + return 1; + } + + t_quantize_us = ggml_time_us() - t_start_us; + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + printf("\n"); + printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f); + printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + return 0; +} diff --git a/ggml/examples/magika/CMakeLists.txt b/ggml/examples/magika/CMakeLists.txt new file mode 100644 index 0000000..be7da8b --- /dev/null +++ b/ggml/examples/magika/CMakeLists.txt @@ -0,0 +1,17 @@ +# +# magika + +set(TEST_TARGET magika) +add_executable(${TEST_TARGET} main.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml) + +# +# For GPU offloading + +if (GGML_CUDA) + add_compile_definitions(GGML_USE_CUDA) +endif() + +if (GGML_METAL) + add_compile_definitions(GGML_USE_METAL) +endif() diff --git a/ggml/examples/magika/README.md b/ggml/examples/magika/README.md new file mode 100644 index 0000000..5a1a979 --- /dev/null +++ b/ggml/examples/magika/README.md @@ -0,0 +1,23 @@ +# Google Magika inference + +Simple example that shows how to use GGML for inference with the [Google Magika](https://github.com/google/magika) file type detection model. + +### Usage + +- Obtain the Magika model in H5 format + - Pinned version: https://github.com/google/magika/blob/4460acb5d3f86807c3b53223229dee2afa50c025/assets_generation/models/standard_v1/model.h5 +- Use `convert.py` to convert the model to gguf format: +```bash + $ python examples/magika/convert.py /path/to/model.h5 +``` +- Invoke the program with the model file and a list of files to identify: +```bash + $ build/bin/magika model.h5.gguf examples/sam/example.jpg examples/magika/convert.py README.md src/ggml.c /bin/gcc write.exe jfk.wav + examples/sam/example.jpg : jpeg (100.00%) pptx (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%) + examples/magika/convert.py : python (99.99%) javascript (0.00%) txt (0.00%) asm (0.00%) scala (0.00%) + README.md : markdown (100.00%) txt (0.00%) yaml (0.00%) ppt (0.00%) shell (0.00%) + src/ggml.c : c (99.95%) txt (0.04%) asm (0.01%) yaml (0.00%) html (0.00%) + /bin/gcc : elf (99.98%) odex (0.02%) pptx (0.00%) smali (0.00%) shell (0.00%) + write.exe : pebin (100.00%) ppt (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%) + jfk.wav : wav (100.00%) ppt (0.00%) shell (0.00%) sevenzip (0.00%) scala (0.00%) +``` diff --git a/ggml/examples/magika/convert.py b/ggml/examples/magika/convert.py new file mode 100644 index 0000000..b901a34 --- /dev/null +++ b/ggml/examples/magika/convert.py @@ -0,0 +1,32 @@ +import sys +from tensorflow import keras +import gguf + +def convert(model_name): + model = keras.models.load_model(model_name, compile=False) + gguf_model_name = model_name + ".gguf" + gguf_writer = gguf.GGUFWriter(gguf_model_name, "magika") + + for layer in model.layers: + # export layers with weights + if layer.weights: + for weight in layer.weights: + print(f" [{weight.name}] {weight.shape} {weight.dtype}") + weight_data = weight.numpy() + gguf_writer.add_tensor(weight.name, weight_data.T) + + + gguf_writer.write_header_to_file() + gguf_writer.write_kv_data_to_file() + gguf_writer.write_tensors_to_file() + gguf_writer.close() + print("Model converted and saved to '{}'".format(gguf_model_name)) + + +if __name__ == '__main__': + if len(sys.argv) > 1: + model_file = sys.argv[1] + else: + model_file = "model.h5" + + convert(model_file) diff --git a/ggml/examples/magika/main.cpp b/ggml/examples/magika/main.cpp new file mode 100644 index 0000000..e8b8031 --- /dev/null +++ b/ggml/examples/magika/main.cpp @@ -0,0 +1,374 @@ +#include "ggml.h" +#include "gguf.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include + +static const char * magika_labels[] = { + "ai", "apk", "appleplist", "asm", "asp", + "batch", "bmp", "bzip", "c", "cab", + "cat", "chm", "coff", "crx", "cs", + "css", "csv", "deb", "dex", "dmg", + "doc", "docx", "elf", "emf", "eml", + "epub", "flac", "gif", "go", "gzip", + "hlp", "html", "ico", "ini", "internetshortcut", + "iso", "jar", "java", "javabytecode", "javascript", + "jpeg", "json", "latex", "lisp", "lnk", + "m3u", "macho", "makefile", "markdown", "mht", + "mp3", "mp4", "mscompress", "msi", "mum", + "odex", "odp", "ods", "odt", "ogg", + "outlook", "pcap", "pdf", "pebin", "pem", + "perl", "php", "png", "postscript", "powershell", + "ppt", "pptx", "python", "pythonbytecode", "rar", + "rdf", "rpm", "rst", "rtf", "ruby", + "rust", "scala", "sevenzip", "shell", "smali", + "sql", "squashfs", "svg", "swf", "symlinktext", + "tar", "tga", "tiff", "torrent", "ttf", + "txt", "unknown", "vba", "wav", "webm", + "webp", "winregistry", "wmf", "xar", "xls", + "xlsb", "xlsx", "xml", "xpi", "xz", + "yaml", "zip", "zlibstream" +}; + +struct magika_hparams { + const int block_size = 4096; + const int beg_size = 512; + const int mid_size = 512; + const int end_size = 512; + const int min_file_size_for_dl = 16; + const int n_label = 113; + const float f_norm_eps = 0.001f; + const int padding_token = 256; +}; + +struct magika_model { + ~magika_model() { + ggml_backend_buffer_free(buf_w); + ggml_backend_free(backend); + ggml_free(ctx_w); + } + + magika_hparams hparams; + + struct ggml_tensor * dense_w; + struct ggml_tensor * dense_b; + + struct ggml_tensor * layer_norm_gamma; + struct ggml_tensor * layer_norm_beta; + + struct ggml_tensor * dense_1_w; + struct ggml_tensor * dense_1_b; + + struct ggml_tensor * dense_2_w; + struct ggml_tensor * dense_2_b; + + struct ggml_tensor * layer_norm_1_gamma; + struct ggml_tensor * layer_norm_1_beta; + + struct ggml_tensor * target_label_w; + struct ggml_tensor * target_label_b; + + ggml_backend_t backend = ggml_backend_cpu_init(); + ggml_backend_buffer_t buf_w = nullptr; + struct ggml_context * ctx_w = nullptr; +}; + +struct ggml_tensor * checked_get_tensor(struct ggml_context * ctx, const char * name) { + struct ggml_tensor * tensor = ggml_get_tensor(ctx, name); + if (!tensor) { + fprintf(stderr, "%s: tensor '%s' not found\n", __func__, name); + throw std::runtime_error("ggml_get_tensor() failed"); + } + return tensor; +} + +bool magika_model_load(const std::string & fname, magika_model & model) { + auto & ctx = model.ctx_w; + + struct gguf_init_params params = { + /*.no_alloc =*/ true, + /*.ctx =*/ &ctx, + }; + + struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), params); + if (!ctx_gguf) { + fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__); + return false; + } + + model.buf_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend); + if (!model.buf_w) { + fprintf(stderr, "%s: ggml_backend_alloc_ctx_tensors() failed\n", __func__); + gguf_free(ctx_gguf); + return false; + } + + try { + model.dense_w = checked_get_tensor(ctx, "dense/kernel:0"); + model.dense_b = checked_get_tensor(ctx, "dense/bias:0"); + + model.layer_norm_gamma = checked_get_tensor(ctx, "layer_normalization/gamma:0"); + model.layer_norm_beta = checked_get_tensor(ctx, "layer_normalization/beta:0"); + + model.dense_1_w = checked_get_tensor(ctx, "dense_1/kernel:0"); + model.dense_1_b = checked_get_tensor(ctx, "dense_1/bias:0"); + + model.dense_2_w = checked_get_tensor(ctx, "dense_2/kernel:0"); + model.dense_2_b = checked_get_tensor(ctx, "dense_2/bias:0"); + + model.layer_norm_1_gamma = checked_get_tensor(ctx, "layer_normalization_1/gamma:0"); + model.layer_norm_1_beta = checked_get_tensor(ctx, "layer_normalization_1/beta:0"); + + model.target_label_w = checked_get_tensor(ctx, "target_label/kernel:0"); + model.target_label_b = checked_get_tensor(ctx, "target_label/bias:0"); + } catch (const std::exception & e) { + fprintf(stderr, "%s: %s\n", __func__, e.what()); + gguf_free(ctx_gguf); + return false; + } + + FILE * f = fopen(fname.c_str(), "rb"); + if (!f) { + fprintf(stderr, "%s: fopen() failed\n", __func__); + gguf_free(ctx_gguf); + return false; + } + + const int n_tensors = gguf_get_n_tensors(ctx_gguf); + + for (int i = 0; i < n_tensors; i++) { + const char * name = gguf_get_tensor_name(ctx_gguf, i); + struct ggml_tensor * tensor = ggml_get_tensor(ctx, name); + size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i); + + //printf("%-30s: [%3ld, %3ld, %3ld, %3ld] %s\n", + // name, + // tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3], + // ggml_type_name(tensor->type)); + + std::vector buf(ggml_nbytes(tensor)); + if (fseek(f, offs, SEEK_SET) != 0) { + fprintf(stderr, "%s: fseek() failed\n", __func__); + gguf_free(ctx_gguf); + fclose(f); + return false; + } + + if (fread(buf.data(), 1, buf.size(), f) != buf.size()) { + fprintf(stderr, "%s: fread() failed\n", __func__); + gguf_free(ctx_gguf); + fclose(f); + return false; + } + + ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size()); + } + + fclose(f); + + gguf_free(ctx_gguf); + + return true; +} + +struct ggml_cgraph * magika_graph( + const magika_model & model, + const int n_files) { + + const auto & hparams = model.hparams; + + static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(); + static std::vector buf(buf_size); + + struct ggml_init_params params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ buf.data(), + /*.no_alloc =*/ true, + }; + + struct ggml_context * ctx = ggml_init(params); + + struct ggml_cgraph * gf = ggml_new_graph(ctx); + + struct ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 257, 1536, n_files); // one-hot + ggml_set_name(input, "input"); + ggml_set_input(input); + + struct ggml_tensor * cur; + + // dense + cur = ggml_mul_mat(ctx, model.dense_w, input); + cur = ggml_add(ctx, cur, model.dense_b); // [128, 1536, n_files] + cur = ggml_gelu(ctx, cur); + + // reshape + cur = ggml_reshape_3d(ctx, cur, 512, 384, n_files); // [384, 512, n_files] + cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); + + // layer normalization + cur = ggml_norm(ctx, cur, hparams.f_norm_eps); + cur = ggml_mul(ctx, cur, model.layer_norm_gamma); // [384, 512, n_files] + cur = ggml_add(ctx, cur, model.layer_norm_beta); // [384, 512, n_files] + + // dense_1 + cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); + cur = ggml_mul_mat(ctx, model.dense_1_w, cur); + cur = ggml_add(ctx, cur, model.dense_1_b); // [256, 384, n_files] + cur = ggml_gelu(ctx, cur); + + // dense_2 + cur = ggml_mul_mat(ctx, model.dense_2_w, cur); + cur = ggml_add(ctx, cur, model.dense_2_b); // [256, 384, n_files] + cur = ggml_gelu(ctx, cur); + + // global_max_pooling1d + cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); // [384, 256, n_files] + cur = ggml_pool_1d(ctx, cur, GGML_OP_POOL_MAX, 384, 384, 0); // [1, 256, n_files] + cur = ggml_reshape_2d(ctx, cur, 256, n_files); // [256, n_files] + + // layer normalization 1 + cur = ggml_norm(ctx, cur, hparams.f_norm_eps); + cur = ggml_mul(ctx, cur, model.layer_norm_1_gamma); // [256, n_files] + cur = ggml_add(ctx, cur, model.layer_norm_1_beta); // [256, n_files] + + // target_label + cur = ggml_mul_mat(ctx, model.target_label_w, cur); + cur = ggml_add(ctx, cur, model.target_label_b); // [n_label, n_files] + cur = ggml_soft_max(ctx, cur); // [n_label, n_files] + ggml_set_name(cur, "target_label_probs"); + ggml_set_output(cur); + + ggml_build_forward_expand(gf, cur); + + return gf; +} + +bool magika_eval( + struct magika_model & model, + const std::vector & fnames) { + + const auto & hparams = model.hparams; + + static ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + + struct ggml_cgraph * gf = magika_graph(model, fnames.size()); + + if (!ggml_gallocr_alloc_graph(alloc, gf)) { + fprintf(stderr, "%s: ggml_gallocr_alloc_graph() failed\n", __func__); + return false; + } + + struct ggml_tensor * input = ggml_graph_get_tensor(gf, "input"); + + for (size_t i = 0; i < fnames.size(); i++) { + FILE * f = fopen(fnames[i].c_str(), "rb"); + if (!f) { + fprintf(stderr, "%s: fopen() failed\n", __func__); + return false; + } + fseek(f, 0, SEEK_END); + long fsize = ftell(f); + + // the buffer is padded with the padding_token if the file is smaller than the block size + std::vector buf(1536, hparams.padding_token); + std::vector read_buf(std::max(hparams.beg_size, std::max(hparams.mid_size, hparams.end_size))); + + // read beg + fseek(f, 0, SEEK_SET); + int n_read = fread(read_buf.data(), 1, hparams.beg_size, f); + for (int j = 0; j < n_read; j++) { + // pad at the end + buf[j] = read_buf[j]; + } + + // read mid + long mid_offs = std::max(0L, (fsize - hparams.mid_size) / 2); + fseek(f, mid_offs, SEEK_SET); + n_read = fread(read_buf.data(), 1, hparams.mid_size, f); + for (int j = 0; j < n_read; j++) { + // pad at both ends + long mid_idx = hparams.beg_size + (hparams.mid_size / 2) - n_read / 2 + j; + buf[mid_idx] = read_buf[j]; + } + + // read end + long end_offs = std::max(0L, fsize - hparams.end_size); + fseek(f, end_offs, SEEK_SET); + n_read = fread(read_buf.data(), 1, hparams.end_size, f); + for (int j = 0; j < n_read; j++) { + // pad at the beginning + int end_idx = hparams.beg_size + hparams.mid_size + hparams.end_size - n_read + j; + buf[end_idx] = read_buf[j]; + } + + fclose(f); + + const size_t inp_bytes = hparams.beg_size + hparams.mid_size + hparams.end_size; + + // convert to one-hot + std::vector one_hot(257*inp_bytes); + for (size_t j = 0; j < inp_bytes; j++) { + one_hot[257*j + buf[j]] = 1.0f; + } + + ggml_backend_tensor_set(input, one_hot.data(), 257*inp_bytes*i*sizeof(float), 257*inp_bytes*sizeof(float)); + } + + if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) { + fprintf(stderr, "%s: ggml_backend_graph_compute() failed\n", __func__); + return false; + } + + struct ggml_tensor * target_label_probs = ggml_graph_get_tensor(gf, "target_label_probs"); + + // print probabilities for the top labels of each file + for (size_t i = 0; i < fnames.size(); i++) { + std::vector probs(hparams.n_label); + ggml_backend_tensor_get(target_label_probs, probs.data(), hparams.n_label*i*sizeof(float), hparams.n_label*sizeof(float)); + + // sort the probabilities + std::vector idx(hparams.n_label); + std::iota(idx.begin(), idx.end(), 0); + std::sort(idx.begin(), idx.end(), [&probs](int i1, int i2) { return probs[i1] > probs[i2]; }); + + // print the top labels + const int top_n = 5; + printf("%-30s: ", fnames[i].c_str()); + for (int j = 0; j < top_n; j++) { + printf("%s (%.2f%%) ", magika_labels[idx[j]], probs[idx[j]]*100); + } + printf("\n"); + } + + return true; +} + +int main(int argc, const char ** argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s [ ...]\n", argv[0]); + return 1; + } + + const char * model_fname = argv[1]; + std::vector fnames; + for (int i = 2; i < argc; i++) { + fnames.push_back(argv[i]); + } + + magika_model model; + if (!magika_model_load(model_fname, model)) { + fprintf(stderr, "magika_model_load() failed\n"); + return 1; + } + + magika_eval(model, fnames); + + return 0; +} diff --git a/ggml/examples/mnist/.gitignore b/ggml/examples/mnist/.gitignore new file mode 100644 index 0000000..f7ee87f --- /dev/null +++ b/ggml/examples/mnist/.gitignore @@ -0,0 +1,3 @@ +data/ +*.gguf +*.ggml diff --git a/ggml/examples/mnist/CMakeLists.txt b/ggml/examples/mnist/CMakeLists.txt new file mode 100644 index 0000000..a4b5133 --- /dev/null +++ b/ggml/examples/mnist/CMakeLists.txt @@ -0,0 +1,58 @@ +# +# mnist-common + +set(TEST_TARGET mnist-common) +add_library(${TEST_TARGET} STATIC mnist-common.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common) + +# +# mnist-eval + +set(TEST_TARGET mnist-eval) +add_executable(${TEST_TARGET} mnist-eval.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common) + +# +# mnist-train + +set(TEST_TARGET mnist-train) +add_executable(${TEST_TARGET} mnist-train.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common) + + +# +# mnist-wasm +if (EMSCRIPTEN) + set(TARGET mnist) + + add_executable(${TARGET} mnist-common.cpp) + target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu) + + set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \ + --bind \ + -s FORCE_FILESYSTEM=1 \ + -s USE_PTHREADS=1 \ + -s PTHREAD_POOL_SIZE=10 \ + -s ASSERTIONS=1 \ + -s WASM=1 \ + -s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'setValue', 'getValue']\" \ + -s EXPORTED_FUNCTIONS=\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\" \ + -s ALLOW_MEMORY_GROWTH=1 \ + --preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \ + --preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \ + ") + + # Copy output to web directory + add_custom_command( + TARGET ${TARGET} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_BINARY_DIR}/bin/mnist.js + ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_BINARY_DIR}/bin/mnist.wasm + ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_BINARY_DIR}/bin/mnist.worker.js + ${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js + ) +endif() diff --git a/ggml/examples/mnist/README.md b/ggml/examples/mnist/README.md new file mode 100644 index 0000000..af80261 --- /dev/null +++ b/ggml/examples/mnist/README.md @@ -0,0 +1,206 @@ +# MNIST Examples for GGML + +This directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/). +All commands listed in this README assume the working directory to be `examples/mnist`. +Please note that training in GGML is a work-in-progress and not production ready. + +## Obtaining the data + +A description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/). +While it is also in principle possible to download the dataset from this website these downloads are frequently throttled and +it is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead. +The dataset will be downloaded automatically when running `mnist-train-fc.py`. + +## Fully connected network + +For our first example we will train a fully connected network. +To train a fully connected model in PyTorch and save it as a GGUF file, run: + +```bash +$ python3 mnist-train-fc.py mnist-fc-f32.gguf + +... + +Test loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14% + +Model tensors saved to mnist-fc-f32.gguf: +fc1.weight (500, 784) +fc1.bias (500,) +fc2.weight (10, 500) +fc2.bias (10,) +``` + +The training script includes an evaluation of the model on the test set. +To evaluate the model on the CPU using GGML, run: + +```bash +$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte + +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +__________________________________####__________________ +______________________________########__________________ +__________________________##########____________________ +______________________##############____________________ +____________________######________####__________________ +__________________________________####__________________ +__________________________________####__________________ +________________________________####____________________ +______________________________####______________________ +________________________##########______________________ +______________________########__####____________________ +________________________##__________##__________________ +____________________________________##__________________ +__________________________________##____________________ +__________________________________##____________________ +________________________________##______________________ +____________________________####________________________ +__________##____________######__________________________ +__________##############________________________________ +________________####____________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no +ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no +ggml_cuda_init: found 1 CUDA devices: + Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes +mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend +mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority): +mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor) +mnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf' +mnist_model_init_from_file: model arch is mnist-fc +mnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf +main: loaded model in 109.44 ms +mnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image +main: predicted digit is 3 +main: test_loss=0.066379+-0.009101 +main: test_acc=97.94+-0.14% +``` + +In addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image. +To train a fully connected model on the CPU using GGML run: + +``` bash +$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte +``` + +It can then be evaluated with the same binary as above. + +## Convolutional network + +To train a convolutional network using TensorFlow run: + +```bash +$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf + +... + +Test loss: 0.047947 +Test accuracy: 98.46% +GGUF model saved to 'mnist-cnn-f32.gguf' +``` + +The saved model can be evaluated on the CPU using the `mnist-eval` binary: + +```bash +$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte + +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +______________________________________##________________ +______________________________________##________________ +______________________________________##________________ +____________________________________##__________________ +__________________________________####__________________ +__________________________________##____________________ +________________________________##______________________ +______________________________##________________________ +____________________________####________________________ +____________________________##__________________________ +__________________________##____________________________ +________________________##______________________________ +______________________##________________________________ +____________________####________________________________ +____________________##__________________________________ +__________________##____________________________________ +________________##______________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +________________________________________________________ +ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no +ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no +ggml_cuda_init: found 1 CUDA devices: + Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes +mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend +mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority): +mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor) +mnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf' +mnist_model_init_from_file: model arch is mnist-cnn +mnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf +main: loaded model in 91.99 ms +mnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image +main: predicted digit is 1 +main: test_loss=0.047955+-0.007029 +main: test_acc=98.46+-0.12% +``` + +Like with the fully connected network the convolutional network can also be trained using GGML: + +``` bash +$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte +``` + +As always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`. + +## Hardware Acceleration + +Both the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations. +A specific backend can be selected by appending the above commands with a backend name. +The compute graphs then schedule the operations to preferentially use the specified backend. +Note that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance. + +## Web demo + +The evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation). +First, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`. +Copy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`. +Symlinking these files will *not* work! +Compile the code like so: + +```bash +$ cd ../../ +$ mkdir -p build-em +$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \ + -DCMAKE_C_FLAGS="-pthread -matomics -mbulk-memory" \ + -DCMAKE_CXX_FLAGS="-pthread -matomics -mbulk-memory" +$ make mnist +``` + +The compilation output is copied into `examples/mnist/web`. +To run it, you need an HTTP server. +For example: + +``` bash +$ python3 examples/mnist/server.py + +Serving directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000 +Application context root: http://localhost:8000/ +``` + +The web demo can then be accessed via the link printed on the console. +Simply draw a digit on the canvas and the model will try to predict what it's supposed to be. +Alternatively, click the "Random" button to retrieve a random digit from the test set. +Be aware that like all neural networks the one we trained is susceptible to distributional shift: +if the numbers you draw look different than the ones in the training set +(e.g. because they're not centered) the model will perform comparatively worse. +An online demo can be accessed [here](https://mnist.ggerganov.com). diff --git a/ggml/examples/mnist/mnist-common.cpp b/ggml/examples/mnist/mnist-common.cpp new file mode 100644 index 0000000..88f91f1 --- /dev/null +++ b/ggml/examples/mnist/mnist-common.cpp @@ -0,0 +1,496 @@ +#include "ggml.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-opt.h" + +#include "mnist-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) { + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "failed to open images file %s\n", fname.c_str()); + return false; + } + fin.seekg(16); + + uint8_t image[MNIST_NINPUT]; + struct ggml_tensor * images = ggml_opt_dataset_data(dataset); + float * buf = ggml_get_data_f32(images); + + GGML_ASSERT(images->ne[0] == MNIST_NINPUT); + for (int64_t iex = 0; iex < images->ne[1]; ++iex) { + fin.read((char *) image, sizeof(image)); + + for (int64_t i = 0; i < MNIST_NINPUT; ++i) { + buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1] + } + } + + return true; +} + +void mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) { + struct ggml_tensor * images = ggml_opt_dataset_data(dataset); + GGML_ASSERT(images->ne[0] == MNIST_NINPUT); + GGML_ASSERT(iex < images->ne[1]); + const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT; + + for (int64_t row = 0; row < MNIST_HW; row++) { + for (int64_t col = 0; col < MNIST_HW; col++) { + const int rgb = roundf(255.0f * image[row*MNIST_HW + col]); +#ifdef _WIN32 + fprintf(stream, "%s", rgb >= 220 ? "##" : "__"); // Represented via text. +#else + fprintf(stream, "\033[48;2;%d;%d;%dm \033[0m", rgb, rgb, rgb); // Represented via colored blocks. +#endif // _WIN32 + } + fprintf(stream, "\n"); + } +} + +bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) { + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + fprintf(stderr, "failed to open labels file %s\n", fname.c_str()); + return 0; + } + fin.seekg(8); + + uint8_t label; + struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset); + float * buf = ggml_get_data_f32(labels); + + GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES); + for (int64_t iex = 0; iex < labels->ne[1]; ++iex) { + fin.read((char *) &label, sizeof(label)); + + for (int64_t i = 0; i < MNIST_NCLASSES; ++i) { + buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f; + } + } + + return true; +} + +// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality: +bool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) { + FILE * f = ggml_fopen(fname, "rb"); + if (!f) { + return false; + } + + const size_t buf_size = 4*1024*1024; + void * buf = malloc(buf_size); + + const int n_tensors = gguf_get_n_tensors(ctx_gguf); + for (int i = 0; i < n_tensors; i++) { + const char * name = gguf_get_tensor_name(ctx_gguf, i); + + struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name); + if (!tensor) { + continue; + } + + const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i); + + if (fseek(f, offs, SEEK_SET) != 0) { + fclose(f); + free(buf); + return false; + } + + const size_t nbytes = ggml_nbytes(tensor); + for (size_t pos = 0; pos < nbytes; pos += buf_size) { + const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos; + + if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) { + fclose(f); + free(buf); + return false; + } + + ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy); + } + } + + fclose(f); + free(buf); + return true; +} + +mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) { + mnist_model model(backend, nbatch_logical, nbatch_physical); + fprintf(stderr, "%s: loading model weights from '%s'\n", __func__, fname.c_str()); + + struct gguf_context * ctx; + { + struct gguf_init_params params = { + /*.no_alloc =*/ true, + /*.ctx =*/ &model.ctx_gguf, + }; + ctx = gguf_init_from_file(fname.c_str(), params); + if (!ctx) { + fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__); + exit(1); + } + } + model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, "general.architecture")); + fprintf(stderr, "%s: model arch is %s\n", __func__, model.arch.c_str()); + + if (model.arch == "mnist-fc") { + model.fc1_weight = ggml_get_tensor(model.ctx_gguf, "fc1.weight"); + GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT); + GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN); + GGML_ASSERT(model.fc1_weight->ne[2] == 1); + GGML_ASSERT(model.fc1_weight->ne[3] == 1); + + model.fc1_bias = ggml_get_tensor(model.ctx_gguf, "fc1.bias"); + GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN); + GGML_ASSERT(model.fc1_bias->ne[1] == 1); + GGML_ASSERT(model.fc1_bias->ne[2] == 1); + GGML_ASSERT(model.fc1_bias->ne[3] == 1); + + model.fc2_weight = ggml_get_tensor(model.ctx_gguf, "fc2.weight"); + GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN); + GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES); + GGML_ASSERT(model.fc2_weight->ne[2] == 1); + GGML_ASSERT(model.fc2_weight->ne[3] == 1); + + model.fc2_bias = ggml_get_tensor(model.ctx_gguf, "fc2.bias"); + GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES); + GGML_ASSERT(model.fc2_bias->ne[1] == 1); + GGML_ASSERT(model.fc2_bias->ne[2] == 1); + GGML_ASSERT(model.fc2_bias->ne[3] == 1); + } else if (model.arch == "mnist-cnn") { + model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, "conv1.kernel"); + GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32); + GGML_ASSERT(model.conv1_kernel->ne[0] == 3); + GGML_ASSERT(model.conv1_kernel->ne[1] == 3); + GGML_ASSERT(model.conv1_kernel->ne[2] == 1); + GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB); + + model.conv1_bias = ggml_get_tensor(model.ctx_gguf, "conv1.bias"); + GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32); + GGML_ASSERT(model.conv1_bias->ne[0] == 1); + GGML_ASSERT(model.conv1_bias->ne[1] == 1); + GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB); + GGML_ASSERT(model.conv1_bias->ne[3] == 1); + + model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, "conv2.kernel"); + GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32); + GGML_ASSERT(model.conv2_kernel->ne[0] == 3); + GGML_ASSERT(model.conv2_kernel->ne[1] == 3); + GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB); + GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2); + + model.conv2_bias = ggml_get_tensor(model.ctx_gguf, "conv2.bias"); + GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32); + GGML_ASSERT(model.conv2_bias->ne[0] == 1); + GGML_ASSERT(model.conv2_bias->ne[1] == 1); + GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2); + GGML_ASSERT(model.conv2_bias->ne[3] == 1); + + model.dense_weight = ggml_get_tensor(model.ctx_gguf, "dense.weight"); + GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32); + GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2)); + GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES); + GGML_ASSERT(model.dense_weight->ne[2] == 1); + GGML_ASSERT(model.dense_weight->ne[3] == 1); + + model.dense_bias = ggml_get_tensor(model.ctx_gguf, "dense.bias"); + GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32); + GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES); + GGML_ASSERT(model.dense_bias->ne[1] == 1); + GGML_ASSERT(model.dense_bias->ne[2] == 1); + GGML_ASSERT(model.dense_bias->ne[3] == 1); + } else { + fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str()); + } + + model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]); + + if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) { + fprintf(stderr, "%s: loading weights from %s failed\n", __func__, fname.c_str()); + exit(1); + } + + // The space in ctx_gguf exactly fits the model weights, + // the images (which also need to be statically allocated) need to be put in a different context. + + model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical); + + ggml_set_name(model.images, "images"); + ggml_set_input(model.images); + + model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]); + + fprintf(stderr, "%s: successfully loaded weights from %s\n", __func__, fname.c_str()); + return model; +} + +mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) { + mnist_model model(backend, nbatch_logical, nbatch_physical); + model.arch = arch; + + std::random_device rd{}; + std::mt19937 gen{rd()}; + std::normal_distribution nd{0.0f, 1e-2f}; + std::vector init_tensors; + + if (model.arch == "mnist-fc") { + fprintf(stderr, "%s: initializing random weights for a fully connected model\n", __func__); + + model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN); + model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN); + model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES); + model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES); + + ggml_set_name(model.fc1_weight, "fc1.weight"); + ggml_set_name(model.fc1_bias, "fc1.bias"); + ggml_set_name(model.fc2_weight, "fc2.weight"); + ggml_set_name(model.fc2_bias, "fc2.bias"); + + init_tensors.push_back(model.fc1_weight); + init_tensors.push_back(model.fc1_bias); + init_tensors.push_back(model.fc2_weight); + init_tensors.push_back(model.fc2_bias); + } else if (model.arch == "mnist-cnn") { + model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB); + model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB); + model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2); + model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2); + model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES); + model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES); + + ggml_set_name(model.conv1_kernel, "conv1.kernel"); + ggml_set_name(model.conv1_bias, "conv1.bias"); + ggml_set_name(model.conv2_kernel, "conv2.kernel"); + ggml_set_name(model.conv2_bias, "conv2.bias"); + ggml_set_name(model.dense_weight, "dense.weight"); + ggml_set_name(model.dense_bias, "dense.bias"); + + init_tensors.push_back(model.conv1_kernel); + init_tensors.push_back(model.conv1_bias); + init_tensors.push_back(model.conv2_kernel); + init_tensors.push_back(model.conv2_bias); + init_tensors.push_back(model.dense_weight); + init_tensors.push_back(model.dense_bias); + } else { + fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str()); + } + + model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL); + ggml_set_name(model.images, "images"); + ggml_set_input(model.images); + + model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]); + + for (ggml_tensor * t : init_tensors) { + GGML_ASSERT(t->type == GGML_TYPE_F32); + const int64_t ne = ggml_nelements(t); + std::vector tmp(ne); + + for (int64_t i = 0; i < ne; ++i) { + tmp[i] = nd(gen); + } + ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t)); + } + + return model; +} + +void mnist_model_build(mnist_model & model) { + if (model.arch == "mnist-fc") { + ggml_set_param(model.fc1_weight); + ggml_set_param(model.fc1_bias); + ggml_set_param(model.fc2_weight); + ggml_set_param(model.fc2_bias); + + ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute, + ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images), + model.fc1_bias)); + model.logits = ggml_add(model.ctx_compute, + ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1), + model.fc2_bias); + } else if (model.arch == "mnist-cnn") { + ggml_set_param(model.conv1_kernel); + ggml_set_param(model.conv1_bias); + ggml_set_param(model.conv2_kernel); + ggml_set_param(model.conv2_bias); + ggml_set_param(model.dense_weight); + ggml_set_param(model.dense_bias); + + struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]); + + struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute, + ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1), + model.conv1_bias)); + GGML_ASSERT(conv1_out->ne[0] == MNIST_HW); + GGML_ASSERT(conv1_out->ne[1] == MNIST_HW); + GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB); + GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical); + + struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0); + GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2); + GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2); + GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB); + GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical); + + struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute, + ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1), + model.conv2_bias)); + GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2); + GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2); + GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2); + GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical); + + struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0); + GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4); + GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4); + GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2); + GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical); + + dense_in = ggml_reshape_2d(model.ctx_compute, + ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)), + (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical); + GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2)); + GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical); + GGML_ASSERT(dense_in->ne[2] == 1); + GGML_ASSERT(dense_in->ne[3] == 1); + + model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias); + } else { + GGML_ASSERT(false); + } + + ggml_set_name(model.logits, "logits"); + ggml_set_output(model.logits); + GGML_ASSERT(model.logits->type == GGML_TYPE_F32); + GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES); + GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical); + GGML_ASSERT(model.logits->ne[2] == 1); + GGML_ASSERT(model.logits->ne[3] == 1); +} + +ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) { + ggml_opt_result_t result = ggml_opt_result_init(); + + ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); + params.ctx_compute = model.ctx_compute; + params.inputs = model.images; + params.outputs = model.logits; + params.build_type = GGML_OPT_BUILD_TYPE_FORWARD; + ggml_opt_context_t opt_ctx = ggml_opt_init(params); + + { + const int64_t t_start_us = ggml_time_us(); + + ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr); + + const int64_t t_total_us = ggml_time_us() - t_start_us; + const double t_total_ms = 1e-3*t_total_us; + const int nex = ggml_opt_dataset_data(dataset)->ne[1]; + fprintf(stderr, "%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\n", + __func__, nex, t_total_ms, (double) t_total_us/nex); + } + + ggml_opt_free(opt_ctx); + + return result; +} + +void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) { + ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset, + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false); +} + +void mnist_model_save(mnist_model & model, const std::string & fname) { + printf("%s: saving model to '%s'\n", __func__, fname.c_str()); + + struct ggml_context * ggml_ctx; + { + struct ggml_init_params params = { + /*.mem_size =*/ 100 * 1024*1024, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + ggml_ctx = ggml_init(params); + } + + gguf_context * gguf_ctx = gguf_init_empty(); + gguf_set_val_str(gguf_ctx, "general.architecture", model.arch.c_str()); + + std::vector weights; + if (model.arch == "mnist-fc") { + weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias}; + } else if (model.arch == "mnist-cnn") { + weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias}; + } else { + GGML_ASSERT(false); + } + for (struct ggml_tensor * t : weights) { + struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t); + ggml_set_name(copy, t->name); + ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t)); + gguf_add_tensor(gguf_ctx, copy); + } + gguf_write_to_file(gguf_ctx, fname.c_str(), false); + + ggml_free(ggml_ctx); + gguf_free(gguf_ctx); +} + +#ifdef __cplusplus +extern "C" { +#endif + +int wasm_eval(uint8_t * digitPtr) { + std::vector digit(digitPtr, digitPtr + MNIST_NINPUT); + + ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1); + struct ggml_tensor * data = ggml_opt_dataset_data(dataset); + + float * buf = ggml_get_data_f32(data); + for (int i = 0; i < MNIST_NINPUT; ++i) { + buf[i] = digitPtr[i] / 255.0f; + } + ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed. + + mnist_model model = mnist_model_init_from_file("mnist-f32.gguf", "CPU", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1); + mnist_model_build(model); + ggml_opt_result_t result = mnist_model_eval(model, dataset); + + int32_t pred; + ggml_opt_result_pred(result, &pred); + + return pred; +} + +int wasm_random_digit(char * digitPtr) { + auto fin = std::ifstream("t10k-images-idx3-ubyte", std::ios::binary); + if (!fin) { + fprintf(stderr, "failed to open digits file\n"); + return 0; + } + srand(time(NULL)); + + // Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000) + fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST)); + fin.read(digitPtr, MNIST_NINPUT); + + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/ggml/examples/mnist/mnist-common.h b/ggml/examples/mnist/mnist-common.h new file mode 100644 index 0000000..a663b3b --- /dev/null +++ b/ggml/examples/mnist/mnist-common.h @@ -0,0 +1,166 @@ +#include +#include +#include +#include +#include +#include + +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "gguf.h" +#include "ggml-cpu.h" +#include "ggml-opt.h" + +#define MNIST_NTRAIN 60000 +#define MNIST_NTEST 10000 + +// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one. +// The logical batch size determines how many datapoints are used for a gradient update. +// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory. +#define MNIST_NBATCH_LOGICAL 1000 +#define MNIST_NBATCH_PHYSICAL 500 + +static_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, "MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0"); +static_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0"); +static_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0"); + +#define MNIST_HW 28 +#define MNIST_NINPUT (MNIST_HW*MNIST_HW) +#define MNIST_NCLASSES 10 + +#define MNIST_NHIDDEN 500 + +// NCB = number of channels base +#define MNIST_CNN_NCB 8 + +struct mnist_model { + std::string arch; + ggml_backend_sched_t backend_sched; + std::vector backends; + const int nbatch_logical; + const int nbatch_physical; + + struct ggml_tensor * images = nullptr; + struct ggml_tensor * logits = nullptr; + + struct ggml_tensor * fc1_weight = nullptr; + struct ggml_tensor * fc1_bias = nullptr; + struct ggml_tensor * fc2_weight = nullptr; + struct ggml_tensor * fc2_bias = nullptr; + + struct ggml_tensor * conv1_kernel = nullptr; + struct ggml_tensor * conv1_bias = nullptr; + struct ggml_tensor * conv2_kernel = nullptr; + struct ggml_tensor * conv2_bias = nullptr; + struct ggml_tensor * dense_weight = nullptr; + struct ggml_tensor * dense_bias = nullptr; + + struct ggml_context * ctx_gguf = nullptr; + struct ggml_context * ctx_static = nullptr; + struct ggml_context * ctx_compute = nullptr; + ggml_backend_buffer_t buf_gguf = nullptr; + ggml_backend_buffer_t buf_static = nullptr; + + mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical) + : nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) { + std::vector devices; + const int ncores_logical = std::thread::hardware_concurrency(); + const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2); + + // Add primary backend: + if (!backend_name.empty()) { + ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str()); + if (dev == nullptr) { + fprintf(stderr, "%s: ERROR: backend %s not found, available:\n", __func__, backend_name.c_str()); + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev_i = ggml_backend_dev_get(i); + fprintf(stderr, " - %s (%s)\n", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i)); + } + exit(1); + } + + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + GGML_ASSERT(backend); + + if (ggml_backend_is_cpu(backend)) { + ggml_backend_cpu_set_n_threads(backend, nthreads); + } + + backends.push_back(backend); + devices.push_back(dev); + } + + // Add all available backends as fallback. + // A "backend" is a stream on a physical device so there is no problem with adding multiple backends for the same device. + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + + ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); + GGML_ASSERT(backend); + + if (ggml_backend_is_cpu(backend)) { + ggml_backend_cpu_set_n_threads(backend, nthreads); + } + + backends.push_back(backend); + devices.push_back(dev); + } + + // The order of the backends passed to ggml_backend_sched_new determines which backend is given priority. + backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true); + fprintf(stderr, "%s: using %s (%s) as primary backend\n", + __func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0])); + if (backends.size() >= 2) { + fprintf(stderr, "%s: unsupported operations will be executed on the following fallback backends (in order of priority):\n", __func__); + for (size_t i = 1; i < backends.size(); ++i) { + fprintf(stderr, "%s: - %s (%s)\n", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i])); + } + } + + { + const size_t size_meta = 1024*ggml_tensor_overhead(); + struct ggml_init_params params = { + /*.mem_size =*/ size_meta, + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_static = ggml_init(params); + } + + { + // The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step). + const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead(); + struct ggml_init_params params = { + /*.mem_size =*/ size_meta, + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_compute = ggml_init(params); + } + } + + ~mnist_model() { + ggml_free(ctx_gguf); + ggml_free(ctx_static); + ggml_free(ctx_compute); + + ggml_backend_buffer_free(buf_gguf); + ggml_backend_buffer_free(buf_static); + ggml_backend_sched_free(backend_sched); + for (ggml_backend_t backend : backends) { + ggml_backend_free(backend); + } + } +}; + +bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset); +void mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex); +bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset); + +mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical); +mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical); +void mnist_model_build(mnist_model & model); +ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset); +void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split); +void mnist_model_save(mnist_model & model, const std::string & fname); diff --git a/ggml/examples/mnist/mnist-eval.cpp b/ggml/examples/mnist/mnist-eval.cpp new file mode 100644 index 0000000..3a45233 --- /dev/null +++ b/ggml/examples/mnist/mnist-eval.cpp @@ -0,0 +1,67 @@ +#include "ggml.h" +#include "ggml-opt.h" + +#include "mnist-common.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +int main(int argc, char ** argv) { + srand(time(NULL)); + ggml_time_init(); + + if (argc != 4 && argc != 5) { + fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]); + exit(1); + } + + ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL); + + if (!mnist_image_load(argv[2], dataset)) { + return 1; + } + if (!mnist_label_load(argv[3], dataset)) { + return 1; + } + + const int iex = rand() % MNIST_NTEST; + mnist_image_print(stdout, dataset, iex); + + const std::string backend = argc >= 5 ? argv[4] : ""; + + const int64_t t_start_us = ggml_time_us(); + mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL); + mnist_model_build(model); + const int64_t t_load_us = ggml_time_us() - t_start_us; + fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0); + + ggml_opt_result_t result_eval = mnist_model_eval(model, dataset); + + std::vector pred(MNIST_NTEST); + ggml_opt_result_pred(result_eval, pred.data()); + fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]); + + double loss; + double loss_unc; + ggml_opt_result_loss(result_eval, &loss, &loss_unc); + fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc); + + double accuracy; + double accuracy_unc; + ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc); + fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc); + + ggml_opt_result_free(result_eval); + + return 0; +} diff --git a/ggml/examples/mnist/mnist-train-cnn.py b/ggml/examples/mnist/mnist-train-cnn.py new file mode 100755 index 0000000..b91fe81 --- /dev/null +++ b/ggml/examples/mnist/mnist-train-cnn.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import sys +from time import time +import gguf +import numpy as np +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers + + +def train(model_path): + # Model / data parameters + num_classes = 10 + input_shape = (28, 28, 1) + + # Load the data and split it between train and test sets + (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() + + # Scale images to the [0, 1] range + x_train = x_train.astype("float32") / 255 + x_test = x_test.astype("float32") / 255 + x_train = np.expand_dims(x_train, -1) + x_test = np.expand_dims(x_test, -1) + print("x_train shape:", x_train.shape) + print(x_train.shape[0], "train samples") + print(x_test.shape[0], "test samples") + + # convert class vectors to binary class matrices + y_train = keras.utils.to_categorical(y_train, num_classes) + y_test = keras.utils.to_categorical(y_test, num_classes) + + model = keras.Sequential( + [ + keras.Input(shape=input_shape, dtype=tf.float32), + layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Flatten(), + layers.Dense(num_classes, activation="softmax", dtype=tf.float32), + ] + ) + + model.summary() + batch_size = 1000 + epochs = 30 + model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) + + t_start = time() + model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1) + print(f"Training took {time()-t_start:.2f}s") + + score = model.evaluate(x_test, y_test, verbose=0) + print(f"Test loss: {score[0]:.6f}") + print(f"Test accuracy: {100*score[1]:.2f}%") + + gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn") + + conv1_kernel = model.layers[0].weights[0].numpy() + conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1]) + gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3)) + + conv1_bias = model.layers[0].weights[1].numpy() + gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1)) + + conv2_kernel = model.layers[2].weights[0].numpy() + conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0]) + gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3)) + + conv2_bias = model.layers[2].weights[1].numpy() + gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1)) + + dense_weight = model.layers[-1].weights[0].numpy() + dense_weight = dense_weight.transpose() + gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16)) + + dense_bias = model.layers[-1].weights[1].numpy() + gguf_writer.add_tensor("dense.bias", dense_bias) + + gguf_writer.write_header_to_file() + gguf_writer.write_kv_data_to_file() + gguf_writer.write_tensors_to_file() + gguf_writer.close() + print(f"GGUF model saved to '{model_path}'") + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + train(sys.argv[1]) diff --git a/ggml/examples/mnist/mnist-train-fc.py b/ggml/examples/mnist/mnist-train-fc.py new file mode 100644 index 0000000..6d8abc5 --- /dev/null +++ b/ggml/examples/mnist/mnist-train-fc.py @@ -0,0 +1,131 @@ +import gguf +import numpy as np +import torch +import torch.nn as nn +import torchvision.datasets as dsets +import torchvision.transforms as transforms +from torch.autograd import Variable + +import sys +from time import time + +input_size = 784 # img_size = (28,28) ---> 28*28=784 in total +hidden_size = 500 # number of nodes at hidden layer +num_classes = 10 # number of output classes discrete range [0,9] +num_epochs = 30 # number of times which the entire dataset is passed throughout the model +batch_size = 1000 # the size of input data used for one iteration +lr = 1e-3 # size of step + + +class Net(nn.Module): + def __init__(self, input_size, hidden_size, num_classes): + super(Net, self).__init__() + self.fc1 = nn.Linear(input_size, hidden_size) + self.relu = nn.ReLU() + self.fc2 = nn.Linear(hidden_size, num_classes) + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + return out + + +def train(model_path): + train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True) + test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor()) + + assert len(train_data) == 60000 + assert len(test_data) == 10000 + + kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True) + train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test) + test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test) + + net = Net(input_size, hidden_size, num_classes) + + if torch.cuda.is_available(): + net.cuda() + + loss_function = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(net.parameters(), lr=lr) + + t_start = time() + for epoch in range(num_epochs): + loss_history = [] + ncorrect = 0 + + for i, (images, labels) in enumerate(train_gen): + images = Variable(images.view(-1, 28*28)) + labels = Variable(labels) + + if torch.cuda.is_available(): + images = images.cuda() + labels = labels.cuda() + + optimizer.zero_grad() + outputs = net(images) + loss = loss_function(outputs, labels) + + loss_history.append(loss.cpu().data) + _, predictions = torch.max(outputs, 1) + ncorrect += (predictions == labels).sum() + + loss.backward() + optimizer.step() + + if (i + 1)*batch_size % 10000 == 0: + loss_mean = np.mean(loss_history) + accuracy = ncorrect / ((i + 1) * batch_size) + print( + f"Epoch [{epoch+1:02d}/{num_epochs}], " + f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], " + f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%") + print() + print(f"Training took {time()-t_start:.2f}s") + + loss_history = [] + ncorrect = 0 + + for i, (images, labels) in enumerate(test_gen): + images = Variable(images.view(-1, 28*28)) + labels = Variable(labels) + + if torch.cuda.is_available(): + images = images.cuda() + labels = labels.cuda() + + outputs = net(images) + loss = loss_function(outputs, labels) + + loss_history.append(loss.cpu().data) + _, predictions = torch.max(outputs, 1) + ncorrect += (predictions == labels).sum().cpu().numpy() + + loss_mean = np.mean(loss_history) + loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1) + accuracy_mean = ncorrect / (len(test_gen) * batch_size) + accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size)) + print() + print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%") + + gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc") + + print() + print(f"Model tensors saved to {model_path}:") + for tensor_name in net.state_dict().keys(): + data = net.state_dict()[tensor_name].squeeze().cpu().numpy() + print(tensor_name, "\t", data.shape) + gguf_writer.add_tensor(tensor_name, data) + + gguf_writer.write_header_to_file() + gguf_writer.write_kv_data_to_file() + gguf_writer.write_tensors_to_file() + gguf_writer.close() + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + train(sys.argv[1]) diff --git a/ggml/examples/mnist/mnist-train.cpp b/ggml/examples/mnist/mnist-train.cpp new file mode 100644 index 0000000..b9df47b --- /dev/null +++ b/ggml/examples/mnist/mnist-train.cpp @@ -0,0 +1,39 @@ +#include "ggml-opt.h" +#include "mnist-common.h" + +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +int main(int argc, char ** argv) { + if (argc != 5 && argc != 6) { + fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]); + exit(0); + } + + // The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA. + // With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact). + // A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1. + ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10); + + if (!mnist_image_load(argv[3], dataset)) { + return 1; + } + if (!mnist_label_load(argv[4], dataset)) { + return 1; + } + + mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL); + + mnist_model_build(model); + + mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f); + + mnist_model_save(model, argv[2]); +} diff --git a/ggml/examples/mnist/server.py b/ggml/examples/mnist/server.py new file mode 100644 index 0000000..588b396 --- /dev/null +++ b/ggml/examples/mnist/server.py @@ -0,0 +1,36 @@ +import http.server +import socketserver +import os +import sys + +DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web')) +PORT = 8000 + +class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=DIRECTORY, **kwargs) + + def end_headers(self): + # Add required headers for SharedArrayBuffer + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + self.send_header("Cross-Origin-Embedder-Policy", "require-corp") + self.send_header("Access-Control-Allow-Origin", "*") + super().end_headers() + +# Enable address reuse +class CustomServer(socketserver.TCPServer): + allow_reuse_address = True + +try: + with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd: + print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}") + print(f"Application context root: http://localhost:{PORT}/") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nServer stopped.") + # Force complete exit + sys.exit(0) +except OSError as e: + print(f"Error: {e}") + sys.exit(1) diff --git a/ggml/examples/perf-metal/CMakeLists.txt b/ggml/examples/perf-metal/CMakeLists.txt new file mode 100644 index 0000000..a970551 --- /dev/null +++ b/ggml/examples/perf-metal/CMakeLists.txt @@ -0,0 +1,7 @@ +# +# perf-metal + +set(TEST_TARGET perf-metal) +add_executable(${TEST_TARGET} perf-metal.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml) + diff --git a/ggml/examples/perf-metal/perf-metal.cpp b/ggml/examples/perf-metal/perf-metal.cpp new file mode 100644 index 0000000..e004dc0 --- /dev/null +++ b/ggml/examples/perf-metal/perf-metal.cpp @@ -0,0 +1,152 @@ +// basic tool to experiment with the Metal backend +// +// 1. Get GPU trace of a dummy graph: +// +// rm -rf /tmp/perf-metal.gputrace +// make -j perf-metal && METAL_CAPTURE_ENABLED=1 ./bin/perf-metal +// open /tmp/perf-metal.gputrace +// +// https://github.com/ggerganov/llama.cpp/issues/9507 +// + +#include "ggml.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml-metal.h" + +#include +#include +#include + +int main(int argc, char ** argv) { + int n_op = 1024; + int n_iter = 128; + + if (argc > 1) { + n_op = std::atoi(argv[1]); + } + + if (argc > 2) { + n_iter = std::atoi(argv[2]); + } + + printf("%s: n_op = %d, n_iter = %d\n", __func__, n_op, n_iter); + + const int ne00 = 8; + const int ne01 = 8; + const int ne11 = 8; + + std::vector data0(ne00*ne01, 1.0f); + std::vector data1(ne00*ne01, 1.0f/ne00); + + ggml_backend_t backend = ggml_backend_metal_init(); + if (!backend) { + fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__); + return 1; + } + + const size_t ctx_size = 2 * ggml_tensor_overhead(); + + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + struct ggml_context * ctx = ggml_init(params); + + struct ggml_tensor * t0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne01); + struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne11); + + ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend); + + ggml_backend_tensor_set(t0, data0.data(), 0, ggml_nbytes(t0)); + ggml_backend_tensor_set(t1, data1.data(), 0, ggml_nbytes(t1)); + + struct ggml_cgraph * gf = NULL; + + struct ggml_context * ctx_cgraph = NULL; + + // create a dummy compute graph: + // + // x = mul_mat(t0, t1) + // x = x * 1.0f + // x = mul_mat(x, t1) + // x = x * 1.0f + // ... repeat n_op times ... + // + { + struct ggml_init_params params0 = { + /*.mem_size =*/ 4*n_op*ggml_tensor_overhead() + ggml_graph_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ctx_cgraph = ggml_init(params0); + + gf = ggml_new_graph_custom(ctx_cgraph, 4*n_op, false); + + struct ggml_tensor * cur = ggml_mul_mat(ctx_cgraph, t0, t1); + cur = ggml_scale(ctx_cgraph, cur, 1.0f); + + for (int i = 0; i < n_op - 1; i++) { + cur = ggml_mul_mat(ctx_cgraph, cur, t1); + cur = ggml_scale(ctx_cgraph, cur, 1.0f); + } + + cur = ggml_scale(ctx_cgraph, cur, 42.0f); + + ggml_build_forward_expand(gf, cur); + } + + printf("%s: graph nodes = %d\n", __func__, ggml_graph_n_nodes(gf)); + + ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + ggml_gallocr_alloc_graph(allocr, gf); + + { + // warm-up + ggml_backend_graph_compute(backend, gf); + + const int64_t t_start = ggml_time_us(); + + for (int iter = 0; iter < n_iter; iter++) { + ggml_backend_graph_compute(backend, gf); + } + + const int64_t t_end = ggml_time_us(); + + // actual trace + ggml_backend_metal_capture_next_compute(backend); + ggml_backend_graph_compute(backend, gf); + //std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace! + ggml_backend_metal_capture_next_compute(backend); + ggml_backend_graph_compute(backend, gf); + //std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace! + ggml_backend_metal_capture_next_compute(backend); + ggml_backend_graph_compute(backend, gf); + + printf("%s: time = %f ms\n", __func__, (t_end - t_start) / 1000.0 / n_iter); + } + + { + struct ggml_tensor * res = ggml_graph_node(gf, -1); + + std::vector data(res->ne[0] * res->ne[1], 0.0f); + + ggml_backend_tensor_get(res, data.data(), 0, ggml_nbytes(res)); + + for (int i1 = 0; i1 < res->ne[1]; i1++) { + for (int i0 = 0; i0 < res->ne[0]; i0++) { + printf("%f ", data[i1*res->ne[0] + i0]); + } + printf("\n"); + } + } + + ggml_free(ctx_cgraph); + ggml_gallocr_free(allocr); + ggml_free(ctx); + ggml_backend_buffer_free(buffer); + ggml_backend_free(backend); + + return 0; +} diff --git a/ggml/examples/prompts/dolly-v2.txt b/ggml/examples/prompts/dolly-v2.txt new file mode 100644 index 0000000..ecdb0b7 --- /dev/null +++ b/ggml/examples/prompts/dolly-v2.txt @@ -0,0 +1,100 @@ +Hello World! => 12092,3645,2 +I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476 +The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449 +"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574 +'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464 +"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449 +The book costs $19.99 => 510,1984,4815,370,746,15,1525 +"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449 +Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2 +C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32 +W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2 +H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32 +I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15 +Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32 +The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15 +I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15 +The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15 +She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15 +We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15 +He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15 +They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15 +The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15 +I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15 +We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15 +The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15 +She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15 +He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15 +The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15 +I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15 +The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15 +They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15 +She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15 +We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15 +He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15 +The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15 +I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15 +They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15 +She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15 +He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15 +The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15 +I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15 +They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15 +She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15 +We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15 +The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15 +He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15 +The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15 +I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15 +They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15 +She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15 +We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15 +The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15 +He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15 +The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15 +I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15 +They are renovating their house. => 3726,403,30074,839,616,2419,15 +She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15 +We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15 +The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15 +He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15 +The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15 +I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15 +They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15 +She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15 +We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15 +The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15 +He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15 +The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15 +I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15 +They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15 +She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15 +We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15 +The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15 +He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15 +The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15 +I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15 +They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15 +She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15 +We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15 +The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15 +He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15 +The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15 +I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15 +They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15 +She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15 +We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15 +The traffic is congest => 510,7137,310,25801 +The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15 +I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15 +She plays the piano beautifully. => 2993,7120,253,18542,27839,15 +The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15 +I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15 +He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15 +The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15 +She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15 +The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15 +We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15 +He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15 +The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15 +She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15 diff --git a/ggml/examples/prompts/gpt-2-chinese.txt b/ggml/examples/prompts/gpt-2-chinese.txt new file mode 100644 index 0000000..919829d --- /dev/null +++ b/ggml/examples/prompts/gpt-2-chinese.txt @@ -0,0 +1 @@ +请问洗手间在哪里？ => 6435,7309,3819,2797,7313,1762,1525,7027,8043 diff --git a/ggml/examples/prompts/gpt-2.txt b/ggml/examples/prompts/gpt-2.txt new file mode 100644 index 0000000..a2ed931 --- /dev/null +++ b/ggml/examples/prompts/gpt-2.txt @@ -0,0 +1,100 @@ +Hello World! => 15496,2159,0 +I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474 +The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526 +"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496 +'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637 +"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526 +The book costs $19.99 => 464,1492,3484,720,1129,13,2079 +"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526 +Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0 +C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30 +W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0 +H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30 +I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13 +Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30 +The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13 +I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13 +The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13 +She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13 +We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13 +He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13 +They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13 +The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13 +I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13 +We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13 +The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13 +She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13 +He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13 +The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13 +I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13 +The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13 +They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13 +She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13 +We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13 +He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13 +The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13 +I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13 +They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13 +She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13 +He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13 +The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13 +I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13 +They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13 +She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13 +We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13 +The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13 +He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13 +The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13 +I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13 +They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13 +She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13 +We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13 +The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13 +He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13 +The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13 +I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13 +They are renovating their house. => 2990,389,24317,803,511,2156,13 +She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13 +We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13 +The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13 +He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13 +The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13 +I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13 +They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13 +She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13 +We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13 +The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13 +He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13 +The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13 +I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13 +They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13 +She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13 +We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13 +The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13 +He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13 +The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13 +I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13 +They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13 +She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13 +We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13 +The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13 +He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13 +The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13 +I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13 +They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13 +She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13 +We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13 +The traffic is congest => 464,4979,318,22791 +The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13 +I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13 +She plays the piano beautifully. => 3347,5341,262,19132,21104,13 +The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13 +I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13 +He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13 +The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13 +She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13 +The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13 +We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13 +He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13 +The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13 +She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13 diff --git a/ggml/examples/prompts/gpt-j.txt b/ggml/examples/prompts/gpt-j.txt new file mode 100644 index 0000000..a2ed931 --- /dev/null +++ b/ggml/examples/prompts/gpt-j.txt @@ -0,0 +1,100 @@ +Hello World! => 15496,2159,0 +I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474 +The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526 +"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496 +'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637 +"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526 +The book costs $19.99 => 464,1492,3484,720,1129,13,2079 +"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526 +Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0 +C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30 +W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0 +H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30 +I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13 +Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30 +The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13 +I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13 +The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13 +She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13 +We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13 +He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13 +They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13 +The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13 +I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13 +We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13 +The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13 +She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13 +He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13 +The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13 +I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13 +The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13 +They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13 +She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13 +We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13 +He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13 +The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13 +I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13 +They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13 +She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13 +He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13 +The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13 +I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13 +They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13 +She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13 +We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13 +The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13 +He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13 +The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13 +I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13 +They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13 +She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13 +We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13 +The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13 +He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13 +The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13 +I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13 +They are renovating their house. => 2990,389,24317,803,511,2156,13 +She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13 +We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13 +The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13 +He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13 +The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13 +I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13 +They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13 +She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13 +We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13 +The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13 +He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13 +The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13 +I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13 +They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13 +She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13 +We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13 +The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13 +He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13 +The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13 +I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13 +They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13 +She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13 +We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13 +The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13 +He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13 +The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13 +I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13 +They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13 +She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13 +We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13 +The traffic is congest => 464,4979,318,22791 +The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13 +I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13 +She plays the piano beautifully. => 3347,5341,262,19132,21104,13 +The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13 +I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13 +He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13 +The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13 +She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13 +The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13 +We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13 +He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13 +The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13 +She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13 diff --git a/ggml/examples/prompts/gpt-neox-japanese.txt b/ggml/examples/prompts/gpt-neox-japanese.txt new file mode 100644 index 0000000..c39df16 --- /dev/null +++ b/ggml/examples/prompts/gpt-neox-japanese.txt @@ -0,0 +1 @@ +明日の天気はどうですか。 => 263,7353,268,18461,271,1722,18405,265 diff --git a/ggml/examples/prompts/gpt-neox.txt b/ggml/examples/prompts/gpt-neox.txt new file mode 100644 index 0000000..ecdb0b7 --- /dev/null +++ b/ggml/examples/prompts/gpt-neox.txt @@ -0,0 +1,100 @@ +Hello World! => 12092,3645,2 +I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476 +The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449 +"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574 +'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464 +"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449 +The book costs $19.99 => 510,1984,4815,370,746,15,1525 +"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449 +Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2 +C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32 +W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2 +H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32 +I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15 +Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32 +The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15 +I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15 +The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15 +She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15 +We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15 +He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15 +They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15 +The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15 +I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15 +We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15 +The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15 +She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15 +He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15 +The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15 +I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15 +The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15 +They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15 +She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15 +We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15 +He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15 +The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15 +I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15 +They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15 +She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15 +He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15 +The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15 +I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15 +They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15 +She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15 +We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15 +The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15 +He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15 +The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15 +I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15 +They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15 +She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15 +We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15 +The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15 +He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15 +The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15 +I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15 +They are renovating their house. => 3726,403,30074,839,616,2419,15 +She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15 +We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15 +The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15 +He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15 +The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15 +I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15 +They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15 +She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15 +We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15 +The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15 +He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15 +The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15 +I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15 +They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15 +She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15 +We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15 +The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15 +He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15 +The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15 +I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15 +They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15 +She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15 +We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15 +The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15 +He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15 +The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15 +I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15 +They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15 +She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15 +We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15 +The traffic is congest => 510,7137,310,25801 +The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15 +I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15 +She plays the piano beautifully. => 2993,7120,253,18542,27839,15 +The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15 +I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15 +He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15 +The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15 +She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15 +The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15 +We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15 +He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15 +The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15 +She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15 diff --git a/ggml/examples/prompts/polyglot-ko.txt b/ggml/examples/prompts/polyglot-ko.txt new file mode 100644 index 0000000..41fa008 --- /dev/null +++ b/ggml/examples/prompts/polyglot-ko.txt @@ -0,0 +1,3 @@ +이것은 테스트 이다. => 12271,296,6474,28037,17 +걱정할 필요 없다. => 18311,482,1062,550,267,17 +버그는 언젠가 고쳐진다. => 6904,272,8575,10381,1765,17 diff --git a/ggml/examples/prompts/replit.txt b/ggml/examples/prompts/replit.txt new file mode 100644 index 0000000..7b5ffcf --- /dev/null +++ b/ggml/examples/prompts/replit.txt @@ -0,0 +1,100 @@ +Hello World! => 6466,147,2317,350 +I can't believe it's already Friday!" => 286,512,172,185,13392,393,172,155,3239,147,29249,8537 +The URL for the website is https://www.example.com." => 505,5635,250,170,11745,235,147,303,262,552,148,811,148,241,148,161 +"She said, 'I love to travel.'" => 161,10386,4089,150,206,286,8440,194,147,12363,148,172,161 +'The temperature is 25.5°C.' => 172,505,147,9502,235,147,20022,8516,228,148,172 +"Let's meet at 2:30 p.m. in the park." => 161,8997,172,155,17120,536,147,162,5245,147,207,148,204,148,219,170,147,17664,148,161 +The book costs $19.99 => 505,147,2277,17494,236,166,11824 +"John's favorite color is blue." => 161,7475,172,155,147,11105,147,349,235,17046,148,161 +Th@nk y0u f0r y0ur h3lp! => 6309,240,9019,147,237,159,247,147,202,159,223,147,237,159,2458,147,226,171,3899,350 +C@n I g3t a c0ffee, pl3@se? => 228,240,211,398,147,267,171,185,216,147,196,159,13360,163,150,147,1287,171,240,155,163,272 +W0w! Th@t's @m@zing! => 450,159,274,350,147,6309,240,185,172,155,268,204,240,301,248,350 +H0w 4re y0u t0d@y? => 304,159,274,320,440,147,237,159,247,147,185,159,182,240,237,272 +I l0ve t0 tr@vel @r0und the w0rld. => 286,997,159,1290,147,185,159,147,490,240,3893,268,223,159,3981,170,147,274,159,223,2833,148 +Wh@t's y0ur f@v0rite m0vie? => 450,226,240,185,172,155,147,237,159,2458,147,202,240,252,159,5961,163,147,204,159,24373,272 +The cat is sleeping on the mat. => 505,147,1604,235,147,3987,248,347,170,147,1297,148 +I need to buy some groceries for dinner. => 286,1645,194,147,8068,1499,147,10022,1037,10023,250,147,182,2749,148 +The sun is shining brightly in the sky. => 505,147,5852,235,147,7304,2967,147,215,649,391,219,170,147,7310,148 +She is reading a book in the park. => 10386,235,9838,216,147,2277,219,170,147,17664,148 +We went for a walk on the beach yesterday. => 3250,10825,250,216,147,8156,347,170,294,5371,147,28830,148 +He plays the guitar like a pro. => 5301,7084,155,170,147,4604,2214,1425,216,3474,148 +They are going to the movies tonight. => 18815,429,6552,194,170,147,15877,194,7907,148 +The flowers are blooming in the garden. => 505,147,22953,155,429,147,10411,2799,248,219,170,147,22140,148 +I enjoy listening to classical music. => 286,23162,15876,248,194,239,4251,147,7395,148 +We need to buy groceries for the week. => 3250,1645,194,147,8068,147,10022,1037,10023,250,170,9238,148 +The dog is chasing its tail in circles. => 505,147,6540,235,147,196,916,248,1602,147,5129,219,147,4095,155,148 +She is wearing a beautiful red dress. => 10386,235,147,16427,248,216,147,23447,147,1160,147,14592,148 +He is a talented actor in Hollywood. => 5301,235,216,147,29750,246,147,5112,219,147,16924,391,10477,148 +The children are playing in the playground. => 505,7934,429,7084,248,219,170,7084,12055,148 +I'm going to visit my grandparents this weekend. => 286,172,204,6552,194,9939,1247,147,11806,12019,291,9238,314,148 +The coffee tastes bitter without sugar. => 505,147,21526,147,20931,155,5145,1430,1988,147,28759,148 +They are planning a surprise party for her. => 18815,429,147,23661,216,147,29240,147,7344,250,1869,148 +She sings like an angel on stage. => 10386,147,155,6502,1425,426,147,26028,347,12685,148 +We should take a vacation to relax. => 3250,936,4654,216,147,15388,946,194,1998,2744,148 +He is studying medicine at the university. => 5301,235,7959,248,147,20742,1668,536,170,147,8025,148 +The rain is pouring heavily outside. => 505,147,6885,235,5306,248,1189,5451,391,8096,148 +I enjoy watching romantic movies. => 286,23162,147,3355,248,147,26080,4140,147,15877,148 +They are celebrating their anniversary today. => 18815,429,147,30000,5841,1669,147,24734,5464,1770,13386,148 +She dances gracefully to the music. => 10386,147,182,1626,155,147,267,8771,8001,194,170,147,7395,148 +He is an excellent basketball player. => 5301,235,426,147,12300,675,185,147,26646,5132,6294,148 +The baby is sleeping soundly in the crib. => 505,147,23597,235,147,3987,248,12642,391,219,170,147,7696,215,148 +I need to finish my homework before dinner. => 286,1645,194,147,6717,1247,147,1071,2722,2643,147,182,2749,148 +They are organizing a charity event next month. => 18815,429,147,16442,248,216,1054,1511,1663,2399,12821,148 +She is cooking a delicious meal for us. => 10386,235,147,20453,248,216,3936,23455,147,26658,250,147,539,148 +We should go hiking in the mountains. => 3250,936,4242,147,2254,5357,219,170,147,204,18028,155,148 +The car broke down on the way to work. => 505,7553,147,510,10036,4288,347,170,3699,194,1916,148 +He loves playing video games in his free time. => 5301,8440,155,7084,248,8722,147,11281,219,1439,4002,801,148 +The birds are chirping in the trees. => 505,147,13043,155,429,147,3904,223,4639,219,170,5311,155,148 +I want to learn how to play the piano. => 286,1857,194,14167,2496,194,7084,170,147,207,23635,148 +They are building a new shopping mall in the city. => 18815,429,11038,216,277,147,22184,147,204,609,219,170,147,2416,148 +She is writing a novel in her spare time. => 10386,235,3242,216,147,25814,219,1869,6772,2382,801,148 +We are going to the zoo this Saturday. => 3250,429,6552,194,170,147,25101,291,147,31426,148 +The cake looks delicious with chocolate frosting. => 505,147,24422,16303,3936,23455,312,147,5619,533,2239,147,202,3973,3431,148 +He is a talented painter who sells his artwork. => 5301,235,216,147,29750,246,147,9226,279,2888,13004,155,1439,12234,2722,148 +The students are studying for their exams. => 505,15707,429,7959,248,250,1669,147,12398,155,148 +I enjoy swimming in the ocean. => 286,23162,147,4729,8528,248,219,170,147,26193,148 +They are renovating their house. => 18815,429,991,10724,3643,1669,13788,148 +She is practicing yoga to stay healthy. => 10386,235,147,18453,248,147,5063,1186,194,15344,147,28550,148 +We should plant flowers in the garden. => 3250,936,147,9212,147,22953,155,219,170,147,22140,148 +The traffic is heavy during rush hour. => 505,147,11097,235,147,22232,4340,147,22319,147,5686,148 +He is a skilled chef who creates amazing dishes. => 5301,235,216,147,8891,246,9784,202,2888,13720,147,28880,147,23852,383,148 +The baby is crawling on the floor. => 505,147,23597,235,147,22120,248,347,170,147,5895,148 +I need to buy a new pair of shoes. => 286,1645,194,147,8068,216,277,12632,210,147,155,21953,155,148 +They are going on a road trip across the country. => 18815,429,6552,347,216,147,6362,147,11395,9762,170,11305,148 +She is playing the piano beautifully. => 10386,235,7084,248,170,147,207,23635,147,23447,391,148 +We are going to a concert tomorrow night. => 3250,429,6552,194,216,1710,4391,29524,12716,148 +The cake tastes delicious with vanilla frosting. => 505,147,24422,147,20931,155,3936,23455,312,5535,7476,147,202,3973,3431,148 +He is a dedicated teacher who inspires his students. => 5301,235,216,326,8298,3460,147,9675,2888,147,28801,155,1439,15707,148 +The students are participating in a science fair. => 505,15707,429,147,30961,3643,219,216,147,10587,147,7636,148 +I enjoy hiking in the mountains. => 286,23162,147,2254,5357,219,170,147,204,18028,155,148 +They are organizing a beach cleanup next weekend. => 18815,429,147,16442,248,216,294,5371,147,10401,2399,9238,314,148 +She is taking photographs of nature. => 10386,235,147,12345,147,4709,1547,155,210,147,211,8603,148 +We should try a new restaurant in town. => 3250,936,147,746,216,277,147,11007,219,147,10200,148 +The traffic is moving slowly on the highway. => 505,147,11097,235,147,8601,147,9880,391,347,170,5976,3330,148 +He is a talented singer with a beautiful voice. => 5301,235,216,147,29750,246,147,155,248,279,312,216,147,23447,147,9316,148 +The baby is laughing and giggling. => 505,147,23597,235,147,23066,248,221,147,2341,3631,2869,148 +I need to do laundry and wash my clothes. => 286,1645,194,543,960,3981,2154,221,147,27589,1247,147,22141,383,148 +They are planning a trip to Europe. => 18815,429,147,23661,216,147,11395,194,13131,148 +She is learning how to play the guitar. => 10386,235,11754,2496,194,7084,170,147,4604,2214,148 +We are going to a museum this Sunday. => 3250,429,6552,194,216,147,204,433,1177,291,147,29111,148 +The coffee smells amazing in the morning. => 505,147,21526,31454,155,147,28880,219,170,20701,148 +He is a hardworking farmer who grows crops. => 5301,235,216,8524,14992,147,16679,279,2888,147,6044,155,147,8650,155,148 +The students are presenting their research projects. => 505,15707,429,5130,248,1669,13217,14235,148 +I enjoy playing soccer with my friends. => 286,23162,7084,248,147,9351,5318,312,1247,147,5347,155,148 +They are volunteering at a local shelter. => 18815,429,147,5238,7478,163,12798,536,216,2491,2905,1359,279,148 +She is practicing martial arts for self-defense. => 10386,235,147,18453,248,147,3261,185,4381,12234,155,250,623,153,29896,148 +We should try a new recipe for dinner. => 3250,936,147,746,216,277,147,9851,250,147,182,2749,148 +The traffic is congest => 505,147,11097,235,1710,14169 +The sun is shining brightly today. => 505,147,5852,235,147,7304,2967,147,215,649,391,13386,148 +I enjoy reading books in my free time. => 286,23162,9838,147,9670,219,1247,4002,801,148 +She plays the piano beautifully. => 10386,7084,155,170,147,207,23635,147,23447,391,148 +The cat chased the mouse around the room. => 505,147,1604,147,196,916,246,170,12551,6890,170,9654,148 +I love eating pizza with extra cheese. => 286,8440,147,163,3643,147,207,8403,312,8230,9784,383,163,148 +He always wears a hat wherever he goes. => 5301,5418,147,16427,155,216,147,4879,2171,2433,1189,16177,148 +The flowers in the garden are blooming. => 505,147,22953,155,219,170,147,22140,429,147,10411,2799,248,148 +She danced gracefully on the stage. => 10386,13378,12408,147,267,8771,8001,347,170,12685,148 +The dog barked loudly in the park. => 505,147,6540,147,973,293,246,147,30182,391,219,170,147,17664,148 +We went swimming in the ocean yesterday. => 3250,10825,147,4729,8528,248,219,170,147,26193,147,28830,148 +He speaks fluent French and Spanish. => 5301,147,13285,155,147,21677,147,254,17590,221,147,31519,148 +The train arrived at the station on time. => 505,147,872,147,20712,182,536,170,147,7184,347,801,148 +She cooked a delicious meal for her family. => 10386,147,20453,246,216,3936,23455,147,26658,250,1869,147,2002,148 diff --git a/ggml/examples/prompts/starcoder.txt b/ggml/examples/prompts/starcoder.txt new file mode 100644 index 0000000..03a5b22 --- /dev/null +++ b/ggml/examples/prompts/starcoder.txt @@ -0,0 +1,100 @@ +Hello World! => 8279,10896,19 +I can't believe it's already Friday!" => 59,883,1330,13710,561,1182,3425,506,25674,11555 +The URL for the website is https://www.example.com." => 1318,3834,436,322,9575,438,1678,555,1499,32,2763,32,508,3107 +"She said, 'I love to travel.'" => 20,25387,9884,30,330,59,14290,372,25283,29329 +'The temperature is 25.5°C.' => 25,1318,13587,438,225,36,39,32,39,23767,53,4564 +"Let's meet at 2:30 p.m. in the park." => 20,9809,1182,18450,821,225,36,44,37,34,298,32,95,32,328,322,880,93,3107 +The book costs $19.99 => 1318,7618,25950,398,35,43,32,43,43 +"John's favorite color is blue." => 20,19693,1182,27448,1963,438,10087,3107 +Th@nk y0u f0r y0ur h3lp! => 1027,50,19877,533,34,103,296,34,100,533,34,305,420,37,1915,19 +C@n I g3t a c0ffee, pl3@se? => 53,50,96,439,485,37,102,312,281,34,21298,30,1278,37,50,277,49 +W0w! Th@t's @m@zing! => 73,34,105,19,947,50,102,1182,477,95,50,26768,19 +H0w 4re y0u t0d@y? => 58,34,105,225,38,268,533,34,103,273,34,86,50,107,49 +I l0ve t0 tr@vel @r0und the w0rld. => 59,456,34,587,273,34,554,50,1203,477,100,34,642,322,341,34,100,1381,32 +Wh@t's y0ur f@v0rite m0vie? => 2444,50,102,1182,533,34,305,296,50,104,34,1049,345,34,104,1075,49 +The cat is sleeping on the mat. => 1318,10501,438,9368,299,544,322,2491,32 +I need to buy some groceries for dinner. => 59,1849,372,16968,1629,20234,85,6958,436,343,3369,32 +The sun is shining brightly in the sky. => 1318,15323,438,787,19068,38231,631,328,322,26718,32 +She is reading a book in the park. => 25387,438,9175,312,7618,328,322,880,93,32 +We went for a walk on the beach yesterday. => 3122,14236,436,312,13503,544,322,526,867,39485,32 +He plays the guitar like a pro. => 1331,41271,322,3932,19931,2124,312,534,32 +They are going to the movies tonight. => 31805,884,6783,372,322,27889,26076,694,32 +The flowers are blooming in the garden. => 1318,7290,483,884,323,18466,299,328,322,485,22461,32 +I enjoy listening to classical music. => 59,31567,20498,372,443,1578,17522,32 +We need to buy groceries for the week. => 3122,1849,372,16968,20234,85,6958,436,322,8209,32 +The dog is chasing its tail in circles. => 1318,27435,438,663,9949,2819,13203,328,46428,32 +She is wearing a beautiful red dress. => 25387,438,996,6992,312,36493,3346,343,714,32 +He is a talented actor in Hollywood. => 1331,438,312,273,9556,318,16038,328,48228,631,21118,32 +The children are playing in the playground. => 1318,5713,884,19788,328,322,4654,1749,32 +I'm going to visit my grandparents this weekend. => 59,3464,6783,372,7725,1672,33162,19277,458,40618,32 +The coffee tastes bitter without sugar. => 1318,36917,273,633,307,3493,391,2876,309,18628,32 +They are planning a surprise party for her. => 31805,884,26116,312,6178,9251,15270,436,7791,32 +She sings like an angel on stage. => 25387,309,2052,2124,600,600,17691,544,10019,32 +We should take a vacation to relax. => 3122,1395,4818,312,29164,367,372,41972,32 +He is studying medicine at the university. => 1331,438,14866,299,32388,482,821,322,707,9190,32 +The rain is pouring heavily outside. => 1318,36987,438,9202,299,46003,2801,11127,32 +I enjoy watching romantic movies. => 59,31567,37652,26045,7268,27889,32 +They are celebrating their anniversary today. => 31805,884,48278,839,1741,3623,23921,5810,672,11610,32 +She dances gracefully to the music. => 25387,343,3151,31376,4938,372,322,17522,32 +He is an excellent basketball player. => 1331,438,600,39203,48400,11653,4362,32 +The baby is sleeping soundly in the crib. => 1318,323,17156,438,9368,299,9934,631,328,322,281,7972,32 +I need to finish my homework before dinner. => 59,1849,372,11361,1672,6765,1007,2670,343,3369,32 +They are organizing a charity event next month. => 31805,884,10558,6183,312,1351,543,1692,2354,6811,32 +She is cooking a delicious meal for us. => 25387,438,23682,299,312,409,406,2406,597,279,436,1770,32 +We should go hiking in the mountains. => 3122,1395,1983,420,1546,299,328,322,10874,1907,32 +The car broke down on the way to work. => 1318,6346,43289,2835,544,322,3352,372,1389,32 +He loves playing video games in his free time. => 1331,598,4954,19788,6027,19705,328,6697,3741,1133,32 +The birds are chirping in the trees. => 1318,8424,3210,884,663,476,7075,328,322,23453,32 +I want to learn how to play the piano. => 59,2637,372,7350,2624,372,4654,322,298,25757,32 +They are building a new shopping mall in the city. => 31805,884,9038,312,537,40692,345,464,328,322,11297,32 +She is writing a novel in her spare time. => 25387,438,4127,312,32913,328,7791,1869,586,1133,32 +We are going to the zoo this Saturday. => 3122,884,6783,372,322,1288,604,458,358,30288,32 +The cake looks delicious with chocolate frosting. => 1318,281,1062,7780,409,406,2406,623,10408,27589,296,20932,299,32 +He is a talented painter who sells his artwork. => 1331,438,312,273,9556,318,42300,6560,10800,101,6697,5549,1007,32 +The students are studying for their exams. => 1318,16512,884,14866,299,436,3623,538,1462,32 +I enjoy swimming in the ocean. => 59,31567,2535,449,6714,328,322,337,18857,32 +They are renovating their house. => 31805,884,316,15007,1741,3623,17075,32 +She is practicing yoga to stay healthy. => 25387,438,11808,11636,533,40067,372,20005,44538,32 +We should plant flowers in the garden. => 3122,1395,26795,7290,483,328,322,485,22461,32 +The traffic is heavy during rush hour. => 1318,16391,438,32389,5929,540,1372,12021,32 +He is a skilled chef who creates amazing dishes. => 1331,438,312,3001,12088,44051,6560,9585,36986,1214,4279,32 +The baby is crawling on the floor. => 1318,323,17156,438,281,1294,2920,544,322,17648,32 +I need to buy a new pair of shoes. => 59,1849,372,16968,312,537,6092,432,787,37764,32 +They are going on a road trip across the country. => 31805,884,6783,544,312,24122,19337,10160,322,10769,32 +She is playing the piano beautifully. => 25387,438,19788,322,298,25757,526,4846,325,514,107,32 +We are going to a concert tomorrow night. => 3122,884,6783,372,312,457,6989,31841,19212,32 +The cake tastes delicious with vanilla frosting. => 1318,281,1062,273,633,307,409,406,2406,623,44653,296,20932,299,32 +He is a dedicated teacher who inspires his students. => 1331,438,312,23112,30877,6560,26194,8017,6697,16512,32 +The students are participating in a science fair. => 1318,16512,884,24623,1741,328,312,27536,19375,32 +I enjoy hiking in the mountains. => 59,31567,420,1546,299,328,322,10874,1907,32 +They are organizing a beach cleanup next weekend. => 31805,884,10558,6183,312,526,867,13144,2354,40618,32 +She is taking photographs of nature. => 25387,438,15137,15110,23626,432,24406,32 +We should try a new restaurant in town. => 3122,1395,1596,312,537,43719,328,38212,32 +The traffic is moving slowly on the highway. => 1318,16391,438,14089,12899,631,544,322,3857,3073,32 +He is a talented singer with a beautiful voice. => 1331,438,312,273,9556,318,309,10118,623,312,36493,20309,32 +The baby is laughing and giggling. => 1318,323,17156,438,2317,2943,299,461,485,365,36088,32 +I need to do laundry and wash my clothes. => 59,1849,372,745,2317,642,994,461,341,917,1672,7375,46948,32 +They are planning a trip to Europe. => 31805,884,26116,312,19337,372,27268,32 +She is learning how to play the guitar. => 25387,438,9608,2624,372,4654,322,3932,19931,32 +We are going to a museum this Sunday. => 3122,884,6783,372,312,345,539,378,458,358,28036,32 +The coffee smells amazing in the morning. => 1318,36917,309,42153,101,36986,328,322,33768,32 +He is a hardworking farmer who grows crops. => 1331,438,312,6784,13578,9019,2302,6560,485,2138,25170,1069,32 +The students are presenting their research projects. => 1318,16512,884,5024,299,3623,13234,8528,32 +I enjoy playing soccer with my friends. => 59,31567,19788,22682,10035,623,1672,22523,32 +They are volunteering at a local shelter. => 31805,884,3920,45585,8637,821,312,2196,309,2542,391,32 +She is practicing martial arts for self-defense. => 25387,438,11808,11636,345,502,564,5549,101,436,630,31,43694,32 +We should try a new recipe for dinner. => 3122,1395,1596,312,537,15233,436,343,3369,32 +The traffic is congest => 1318,16391,438,457,2776 +The sun is shining brightly today. => 1318,15323,438,787,19068,38231,631,11610,32 +I enjoy reading books in my free time. => 59,31567,9175,21739,328,1672,3741,1133,32 +She plays the piano beautifully. => 25387,41271,322,298,25757,526,4846,325,514,107,32 +The cat chased the mouse around the room. => 1318,10501,663,16109,322,8459,6835,322,8355,32 +I love eating pizza with extra cheese. => 59,14290,484,1741,47630,623,6717,8277,30315,32 +He always wears a hat wherever he goes. => 1331,5182,996,4177,312,25793,2154,424,938,13107,32 +The flowers in the garden are blooming. => 1318,7290,483,328,322,485,22461,884,323,18466,299,32 +She danced gracefully on the stage. => 25387,343,6087,31376,4938,544,322,10019,32 +The dog barked loudly in the park. => 1318,27435,323,1087,318,598,836,631,328,322,880,93,32 +We went swimming in the ocean yesterday. => 3122,14236,2535,449,6714,328,322,337,18857,39485,32 +He speaks fluent French and Spanish. => 1331,24498,101,38055,43652,461,14911,1708,32 +The train arrived at the station on time. => 1318,5683,2099,32114,821,322,18662,544,1133,32 +She cooked a delicious meal for her family. => 25387,23682,318,312,409,406,2406,597,279,436,7791,13872,32 diff --git a/ggml/examples/prompts/test-cases.txt b/ggml/examples/prompts/test-cases.txt new file mode 100644 index 0000000..4d0bdbf --- /dev/null +++ b/ggml/examples/prompts/test-cases.txt @@ -0,0 +1,110 @@ +# test case format +# : + +English: Hello World! +English: I can't believe it's already Friday!" +English: The URL for the website is https://www.example.com." +English: "She said, 'I love to travel.'" +English: 'The temperature is 25.5°C.' +English: "Let's meet at 2:30 p.m. in the park." +English: The book costs $19.99 +English: "John's favorite color is blue." +English: Th@nk y0u f0r y0ur h3lp! +English: C@n I g3t a c0ffee, pl3@se? +English: W0w! Th@t's @m@zing! +English: H0w 4re y0u t0d@y? +English: I l0ve t0 tr@vel @r0und the w0rld. +English: Wh@t's y0ur f@v0rite m0vie? +English: The cat is sleeping on the mat. +English: I need to buy some groceries for dinner. +English: The sun is shining brightly in the sky. +English: She is reading a book in the park. +English: We went for a walk on the beach yesterday. +English: He plays the guitar like a pro. +English: They are going to the movies tonight. +English: The flowers are blooming in the garden. +English: I enjoy listening to classical music. +English: We need to buy groceries for the week. +English: The dog is chasing its tail in circles. +English: She is wearing a beautiful red dress. +English: He is a talented actor in Hollywood. +English: The children are playing in the playground. +English: I'm going to visit my grandparents this weekend. +English: The coffee tastes bitter without sugar. +English: They are planning a surprise party for her. +English: She sings like an angel on stage. +English: We should take a vacation to relax. +English: He is studying medicine at the university. +English: The rain is pouring heavily outside. +English: I enjoy watching romantic movies. +English: They are celebrating their anniversary today. +English: She dances gracefully to the music. +English: He is an excellent basketball player. +English: The baby is sleeping soundly in the crib. +English: I need to finish my homework before dinner. +English: They are organizing a charity event next month. +English: She is cooking a delicious meal for us. +English: We should go hiking in the mountains. +English: The car broke down on the way to work. +English: He loves playing video games in his free time. +English: The birds are chirping in the trees. +English: I want to learn how to play the piano. +English: They are building a new shopping mall in the city. +English: She is writing a novel in her spare time. +English: We are going to the zoo this Saturday. +English: The cake looks delicious with chocolate frosting. +English: He is a talented painter who sells his artwork. +English: The students are studying for their exams. +English: I enjoy swimming in the ocean. +English: They are renovating their house. +English: She is practicing yoga to stay healthy. +English: We should plant flowers in the garden. +English: The traffic is heavy during rush hour. +English: He is a skilled chef who creates amazing dishes. +English: The baby is crawling on the floor. +English: I need to buy a new pair of shoes. +English: They are going on a road trip across the country. +English: She is playing the piano beautifully. +English: We are going to a concert tomorrow night. +English: The cake tastes delicious with vanilla frosting. +English: He is a dedicated teacher who inspires his students. +English: The students are participating in a science fair. +English: I enjoy hiking in the mountains. +English: They are organizing a beach cleanup next weekend. +English: She is taking photographs of nature. +English: We should try a new restaurant in town. +English: The traffic is moving slowly on the highway. +English: He is a talented singer with a beautiful voice. +English: The baby is laughing and giggling. +English: I need to do laundry and wash my clothes. +English: They are planning a trip to Europe. +English: She is learning how to play the guitar. +English: We are going to a museum this Sunday. +English: The coffee smells amazing in the morning. +English: He is a hardworking farmer who grows crops. +English: The students are presenting their research projects. +English: I enjoy playing soccer with my friends. +English: They are volunteering at a local shelter. +English: She is practicing martial arts for self-defense. +English: We should try a new recipe for dinner. +English: The traffic is congest +English: The sun is shining brightly today. +English: I enjoy reading books in my free time. +English: She plays the piano beautifully. +English: The cat chased the mouse around the room. +English: I love eating pizza with extra cheese. +English: He always wears a hat wherever he goes. +English: The flowers in the garden are blooming. +English: She danced gracefully on the stage. +English: The dog barked loudly in the park. +English: We went swimming in the ocean yesterday. +English: He speaks fluent French and Spanish. +English: The train arrived at the station on time. +English: She cooked a delicious meal for her family. +Korean: 이것은 테스트 이다. +Korean: 걱정할 필요 없다. +Korean: 버그는 언젠가 고쳐진다. +Japanese: 明日の天気はどうですか。 +Chinese: 请问洗手间在哪里？ +Emoji: I'm feeling 😄 today! +Unicode: ◑ ▢ ▣ ◱ \ No newline at end of file diff --git a/ggml/examples/prompts/tokenize_huggingface.py b/ggml/examples/prompts/tokenize_huggingface.py new file mode 100644 index 0000000..627771f --- /dev/null +++ b/ggml/examples/prompts/tokenize_huggingface.py @@ -0,0 +1,65 @@ +import os +from transformers import AutoTokenizer + +os.environ['TOKENIZERS_PARALLELISM'] = "false" + +list_repo_hf = ["databricks/dolly-v2-3b", # dolly-v2 (3b, 7b, 12b models share the same tokenizer) + "gpt2", # gpt-2 (gpt2-xl, gpt2-large share the same tokenizer) + "uer/gpt2-chinese-cluecorpussmall", # gpt-2-chinese + "EleutherAI/gpt-j-6b", # gpt-j + "EleutherAI/gpt-neox-20b", # gpt-neox + "EleutherAI/polyglot-ko-1.3b", # gpt-neox (polyglot-ko 5.8b and 12.8b share the same tokenizer") + "rinna/japanese-gpt-neox-3.6b", # gpt-neox + # mpt-7b (uses gpt-neox-20b tokenizer) + "replit/replit-code-v1-3b", # replit + "bigcode/starcoder", # starcoder (huggingface-cli login required) + "openai/whisper-tiny" # whisper (base, large, large-v2 share the same tokenizer) + ] + +repo2ggml = {"databricks/dolly-v2-3b" : "dolly-v2", + "gpt2" : "gpt-2", + "uer/gpt2-chinese-cluecorpussmall" : "gpt-2-chinese", + "EleutherAI/gpt-j-6b" : "gpt-j", + "EleutherAI/gpt-neox-20b" : "gpt-neox", + "EleutherAI/polyglot-ko-1.3b" : "polyglot-ko", + "rinna/japanese-gpt-neox-3.6b" : "gpt-neox-japanese", + "replit/replit-code-v1-3b" : "replit", + "bigcode/starcoder" : "starcoder", + "openai/whisper-tiny" : "whisper"} + +repo2language = {"databricks/dolly-v2-3b" : "english", + "gpt2" : "english", + "uer/gpt2-chinese-cluecorpussmall" : "chinese", + "EleutherAI/gpt-j-6b" : "english", + "EleutherAI/gpt-neox-20b" : "english", + "EleutherAI/polyglot-ko-1.3b" : "korean", + "rinna/japanese-gpt-neox-3.6b" : "japanese", + "replit/replit-code-v1-3b" : "english", + "bigcode/starcoder" : "english", + "openai/whisper-tiny" : "english"} + +delimeter = ": " +test_sentences = [] +with open("test-cases.txt", "r") as f: + lines = [l.rstrip() for l in f.readlines()] + for l in lines: + if delimeter in l: + language = l[:l.index(delimeter)] + sentence = l[l.index(delimeter) + len(delimeter):] + test_sentences.append((language.lower(), sentence)) + +for repo in list_repo_hf: + + target_language = repo2language[repo] + + tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True) + + tokens_hf = [] + for language, sentence in test_sentences: + if language == target_language: + tokens = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentence)) + tokens_hf.append((sentence, tokens)) + + save_txt = repo2ggml[repo] + ".txt" + with open(save_txt, "w") as f: + f.writelines([sentence + " => " + ",".join(str(t) for t in tokens) + "\n" for sentence, tokens in tokens_hf]) diff --git a/ggml/examples/prompts/whisper.txt b/ggml/examples/prompts/whisper.txt new file mode 100644 index 0000000..a8f1caa --- /dev/null +++ b/ggml/examples/prompts/whisper.txt @@ -0,0 +1,100 @@ +Hello World! => 15947,3937,0 +I can't believe it's already Friday!" => 40,393,380,1697,309,311,1217,6984,2963 +The URL for the website is https://www.example.com." => 2278,12905,337,220,3322,3144,307,34426,21492,17919,13,3121,335,781,13,1112,889 +"She said, 'I love to travel.'" => 1,9526,848,11,922,40,959,220,1353,220,17227,779,28763 +'The temperature is 25.5°C.' => 6,2278,220,18275,610,1503,307,3552,13,20,11782,34,4443 +"Let's meet at 2:30 p.m. in the park." => 1,8373,311,1677,412,568,25,3446,280,13,76,13,294,220,3322,3884,889 +The book costs $19.99 => 2278,1446,5497,1848,3405,13,8494 +"John's favorite color is blue." => 1,16938,311,2954,2017,307,3344,889 +Th@nk y0u f0r y0ur h3lp! => 2434,31,77,74,288,15,84,283,15,81,288,15,374,276,18,75,79,0 +C@n I g3t a c0ffee, pl3@se? => 34,31,77,286,290,18,83,257,269,15,4617,11,499,18,31,405,30 +W0w! Th@t's @m@zing! => 54,15,86,0,334,31,83,311,10428,76,31,8781,0 +H0w 4re y0u t0d@y? => 39,15,86,1017,265,288,15,84,220,83,15,67,31,88,30 +I l0ve t0 tr@vel @r0und the w0rld. => 40,287,15,303,220,83,15,220,6903,31,779,10428,81,15,997,220,3322,261,15,81,348,13 +Wh@t's y0ur f@v0rite m0vie? => 2471,31,83,311,288,15,374,283,31,85,15,35002,275,15,12702,30 +The cat is sleeping on the mat. => 2278,3857,307,8296,322,220,3322,3803,13 +I need to buy some groceries for dinner. => 40,643,220,1353,2256,512,31391,337,6148,13 +The sun is shining brightly in the sky. => 2278,3295,307,18269,47418,294,220,3322,5443,13 +She is reading a book in the park. => 9526,307,3760,257,1446,294,220,3322,3884,13 +We went for a walk on the beach yesterday. => 4360,1437,337,257,1792,322,220,3322,7534,5186,13 +He plays the guitar like a pro. => 5205,5749,220,3322,7531,411,257,447,13 +They are going to the movies tonight. => 8829,366,516,220,1353,220,3322,6233,220,1756,397,13 +The flowers are blooming in the garden. => 2278,8085,366,45294,294,220,3322,7431,13 +I enjoy listening to classical music. => 40,2103,4764,220,1353,13735,1318,13 +We need to buy groceries for the week. => 4360,643,220,1353,2256,31391,337,220,3322,1243,13 +The dog is chasing its tail in circles. => 2278,3000,307,17876,1080,220,14430,294,13040,13 +She is wearing a beautiful red dress. => 9526,307,4769,257,2238,2182,5231,13 +He is a talented actor in Hollywood. => 5205,307,257,220,32831,6003,8747,294,11628,13 +The children are playing in the playground. => 2278,2227,366,2433,294,220,3322,24646,13 +I'm going to visit my grandparents this weekend. => 40,478,516,220,1353,3441,452,21876,220,11176,6711,13 +The coffee tastes bitter without sugar. => 2278,4982,220,83,40246,13871,1553,5076,13 +They are planning a surprise party for her. => 8829,366,5038,257,6365,3595,337,720,13 +She sings like an angel on stage. => 9526,23250,411,364,14250,322,3233,13 +We should take a vacation to relax. => 4360,820,220,27612,257,12830,220,1353,5789,13 +He is studying medicine at the university. => 5205,307,7601,7195,412,220,3322,5454,13 +The rain is pouring heavily outside. => 2278,4830,307,20450,10950,2380,13 +I enjoy watching romantic movies. => 40,2103,1976,13590,6233,13 +They are celebrating their anniversary today. => 8829,366,15252,220,3322,347,12962,220,83,378,320,13 +She dances gracefully to the music. => 9526,28322,10042,2277,220,1353,220,3322,1318,13 +He is an excellent basketball player. => 5205,307,364,7103,11767,4256,13 +The baby is sleeping soundly in the crib. => 2278,3186,307,8296,1626,356,294,220,3322,47163,13 +I need to finish my homework before dinner. => 40,643,220,1353,2413,452,14578,949,6148,13 +They are organizing a charity event next month. => 8829,366,17608,257,16863,2280,958,1618,13 +She is cooking a delicious meal for us. => 9526,307,6361,257,4809,6791,337,505,13 +We should go hiking in the mountains. => 4360,820,352,23784,294,220,3322,10233,13 +The car broke down on the way to work. => 2278,1032,6902,760,322,220,3322,636,220,1353,589,13 +He loves playing video games in his free time. => 5205,6752,2433,960,2813,294,702,1737,220,3766,13 +The birds are chirping in the trees. => 2278,9009,366,36682,294,220,3322,220,3599,279,13 +I want to learn how to play the piano. => 40,528,220,1353,1466,577,220,1353,862,220,3322,9211,13 +They are building a new shopping mall in the city. => 8829,366,2390,257,777,8688,16026,294,220,3322,2307,13 +She is writing a novel in her spare time. => 9526,307,3579,257,7613,294,720,13798,220,3766,13 +We are going to the zoo this Saturday. => 4360,366,516,220,1353,220,3322,25347,220,11176,8803,13 +The cake looks delicious with chocolate frosting. => 2278,5908,1542,4809,365,6215,37048,13 +He is a talented painter who sells his artwork. => 5205,307,257,220,32831,6003,26619,567,20897,702,15829,13 +The students are studying for their exams. => 2278,1731,366,7601,337,220,3322,347,20514,13 +I enjoy swimming in the ocean. => 40,2103,11989,294,220,3322,7810,13 +They are renovating their house. => 8829,366,18845,990,220,3322,347,1782,13 +She is practicing yoga to stay healthy. => 9526,307,11350,15128,220,1353,1754,4627,13 +We should plant flowers in the garden. => 4360,820,3709,8085,294,220,3322,7431,13 +The traffic is heavy during rush hour. => 2278,220,17227,3341,307,4676,1830,9300,1773,13 +He is a skilled chef who creates amazing dishes. => 5205,307,257,19690,10530,567,7829,2243,10814,13 +The baby is crawling on the floor. => 2278,3186,307,32979,322,220,3322,4123,13 +I need to buy a new pair of shoes. => 40,643,220,1353,2256,257,777,6119,295,6654,13 +They are going on a road trip across the country. => 8829,366,516,322,257,3060,220,83,8400,2108,220,3322,1941,13 +She is playing the piano beautifully. => 9526,307,2433,220,3322,9211,16525,13 +We are going to a concert tomorrow night. => 4360,366,516,220,1353,257,8543,220,83,298,3162,1818,13 +The cake tastes delicious with vanilla frosting. => 2278,5908,220,83,40246,4809,365,17528,37048,13 +He is a dedicated teacher who inspires his students. => 5205,307,257,8374,220,975,4062,567,32566,702,1731,13 +The students are participating in a science fair. => 2278,1731,366,13950,294,257,3497,3143,13 +I enjoy hiking in the mountains. => 40,2103,23784,294,220,3322,10233,13 +They are organizing a beach cleanup next weekend. => 8829,366,17608,257,7534,40991,958,6711,13 +She is taking photographs of nature. => 9526,307,220,48625,17649,295,3687,13 +We should try a new restaurant in town. => 4360,820,220,83,627,257,777,6383,294,220,30401,13 +The traffic is moving slowly on the highway. => 2278,220,17227,3341,307,2684,5692,322,220,3322,17205,13 +He is a talented singer with a beautiful voice. => 5205,307,257,220,32831,6003,11564,365,257,2238,3177,13 +The baby is laughing and giggling. => 2278,3186,307,5059,293,290,24542,13 +I need to do laundry and wash my clothes. => 40,643,220,1353,360,19811,293,5675,452,5534,13 +They are planning a trip to Europe. => 8829,366,5038,257,220,83,8400,220,1353,3315,13 +She is learning how to play the guitar. => 9526,307,2539,577,220,1353,862,220,3322,7531,13 +We are going to a museum this Sunday. => 4360,366,516,220,1353,257,8441,220,11176,7776,13 +The coffee smells amazing in the morning. => 2278,4982,10036,2243,294,220,3322,2446,13 +He is a hardworking farmer who grows crops. => 5205,307,257,1152,22475,17891,567,13156,16829,13 +The students are presenting their research projects. => 2278,1731,366,15578,220,3322,347,2132,4455,13 +I enjoy playing soccer with my friends. => 40,2103,2433,15469,365,452,1855,13 +They are volunteering at a local shelter. => 8829,366,33237,412,257,2654,13341,13 +She is practicing martial arts for self-defense. => 9526,307,11350,20755,8609,337,2698,12,49268,13 +We should try a new recipe for dinner. => 4360,820,220,83,627,257,777,6782,337,6148,13 +The traffic is congest => 2278,220,17227,3341,307,31871 +The sun is shining brightly today. => 2278,3295,307,18269,47418,220,83,378,320,13 +I enjoy reading books in my free time. => 40,2103,3760,3642,294,452,1737,220,3766,13 +She plays the piano beautifully. => 9526,5749,220,3322,9211,16525,13 +The cat chased the mouse around the room. => 2278,3857,33091,220,3322,9719,926,220,3322,1808,13 +I love eating pizza with extra cheese. => 40,959,3936,8298,365,2857,5399,13 +He always wears a hat wherever he goes. => 5205,1009,20877,257,2385,8660,415,1709,13 +The flowers in the garden are blooming. => 2278,8085,294,220,3322,7431,366,45294,13 +She danced gracefully on the stage. => 9526,32909,10042,2277,322,220,3322,3233,13 +The dog barked loudly in the park. => 2278,3000,16202,292,22958,294,220,3322,3884,13 +We went swimming in the ocean yesterday. => 4360,1437,11989,294,220,3322,7810,5186,13 +He speaks fluent French and Spanish. => 5205,10789,40799,5522,293,8058,13 +The train arrived at the station on time. => 2278,220,83,7146,6678,412,220,3322,5214,322,220,3766,13 +She cooked a delicious meal for her family. => 9526,9267,257,4809,6791,337,720,1605,13 diff --git a/ggml/examples/python/README.md b/ggml/examples/python/README.md new file mode 100644 index 0000000..3cf5d24 --- /dev/null +++ b/ggml/examples/python/README.md @@ -0,0 +1,115 @@ +# Simple autogenerated Python bindings for ggml + +This folder contains: + +- Scripts to generate full Python bindings from ggml headers (+ stubs for autocompletion in IDEs) +- Some barebones utils (see [ggml/utils.py](./ggml/utils.py)): + - `ggml.utils.init` builds a context that's freed automatically when the pointer gets GC'd + - `ggml.utils.copy` **copies between same-shaped tensors (numpy or ggml), w/ automatic (de/re)quantization** + - `ggml.utils.numpy` returns a numpy view over a ggml tensor; if it's quantized, it returns a copy (requires `allow_copy=True`) +- Very basic examples (anyone wants to port [llama2.c](https://github.com/karpathy/llama2.c)?) + +Provided you set `GGML_LIBRARY=.../path/to/libggml_shared.so` (see instructions below), it's trivial to do some operations on quantized tensors: + +```python +# Make sure libllama.so is in your [DY]LD_LIBRARY_PATH, or set GGML_LIBRARY=.../libggml_shared.so + +from ggml import lib, ffi +from ggml.utils import init, copy, numpy +import numpy as np + +ctx = init(mem_size=12*1024*1024) +n = 256 +n_threads = 4 + +a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) +b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # Can't both be quantized +sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too! + +gf = ffi.new('struct ggml_cgraph*') +lib.ggml_build_forward_expand(gf, sum) + +copy(np.array([i for i in range(n)], np.float32), a) +copy(np.array([i*100 for i in range(n)], np.float32), b) + +lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads) + +print(numpy(a, allow_copy=True)) +# 0. 1.0439453 2.0878906 3.131836 4.1757812 5.2197266. ... +print(numpy(b)) +# 0. 100. 200. 300. 400. 500. ... +print(numpy(sum, allow_copy=True)) +# 0. 105.4375 210.875 316.3125 421.75 527.1875 ... +``` + +### Prerequisites + +You'll need a shared library of ggml to use the bindings. + +#### Build libggml_shared.so or libllama.so + +As of this writing the best is to use [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)'s generated `libggml_shared.so` or `libllama.so`, which you can build as follows: + +```bash +git clone https://github.com/ggerganov/llama.cpp +# On a CUDA-enabled system add -DLLAMA_CUDA=1 +# On a Mac add -DLLAMA_METAL=1 +cmake llama.cpp \ + -B llama_build \ + -DCMAKE_C_FLAGS=-Ofast \ + -DLLAMA_NATIVE=1 \ + -DLLAMA_LTO=1 \ + -DBUILD_SHARED_LIBS=1 \ + -DLLAMA_MPI=1 \ + -DLLAMA_BUILD_TESTS=0 \ + -DLLAMA_BUILD_EXAMPLES=0 +( cd llama_build && make -j ) + +# On Mac, this will be libggml_shared.dylib instead +export GGML_LIBRARY=$PWD/llama_build/libggml_shared.so +# Alternatively, you can just copy it to your system's lib dir, e.g /usr/local/lib +``` + +#### (Optional) Regenerate the bindings and stubs + +If you added or changed any signatures of the C API, you'll want to regenerate the bindings ([ggml/cffi.py](./ggml/cffi.py)) and stubs ([ggml/__init__.pyi](./ggml/__init__.pyi)). + +Luckily it's a one-liner using [regenerate.py](./regenerate.py): + +```bash +pip install -q cffi + +python regenerate.py +``` + +By default it assumes `llama.cpp` was cloned in ../../../llama.cpp (alongside the ggml folder). You can override this with: + +```bash +C_INCLUDE_DIR=$LLAMA_CPP_DIR python regenerate.py +``` + +You can also edit [api.h](./api.h) to control which files should be included in the generated bindings (defaults to `llama.cpp/ggml*.h`) + +In fact, if you wanted to only generate bindings for the current version of the `ggml` repo itself (instead of `llama.cpp`; you'd loose support for k-quants), you could run: + +```bash +API=../../include/ggml.h python regenerate.py +``` + +## Develop + +Run tests: + +```bash +pytest +``` + +### Alternatives + +This example's goal is to showcase [cffi](https://cffi.readthedocs.io/)-generated bindings that are trivial to use and update, but there are already alternatives in the wild: + +- https://github.com/abetlen/ggml-python: these bindings seem to be hand-written and use [ctypes](https://docs.python.org/3/library/ctypes.html). It has [high-quality API reference docs](https://ggml-python.readthedocs.io/en/latest/api-reference/#ggml.ggml) that can be used with these bindings too, but it doesn't expose Metal, CUDA, MPI or OpenCL calls, doesn't support transparent (de/re)quantization like this example does (see [ggml.utils](./ggml/utils.py) module), and won't pick up your local changes. + +- https://github.com/abetlen/llama-cpp-python: these expose the C++ `llama.cpp` interface, which this example cannot easily be extended to support (`cffi` only generates bindings of C libraries) + +- [pybind11](https://github.com/pybind/pybind11) and [nanobind](https://github.com/wjakob/nanobind) are two alternatives to cffi that support binding C++ libraries, but it doesn't seem either of them have an automatic generator (writing bindings is rather time-consuming). diff --git a/ggml/examples/python/api.h b/ggml/examples/python/api.h new file mode 100644 index 0000000..8d565bd --- /dev/null +++ b/ggml/examples/python/api.h @@ -0,0 +1,14 @@ +/* + List here all the headers you want to expose in the Python bindings, + then run `python regenerate.py` (see details in README.md) +*/ + +#include "ggml.h" +#include "ggml-metal.h" +#include "ggml-opencl.h" + +// Headers below are currently only present in the llama.cpp repository, comment them out if you don't have them. +#include "k_quants.h" +#include "ggml-alloc.h" +#include "ggml-cuda.h" +#include "ggml-mpi.h" \ No newline at end of file diff --git a/ggml/examples/python/example_add_quant.py b/ggml/examples/python/example_add_quant.py new file mode 100644 index 0000000..cecb44e --- /dev/null +++ b/ggml/examples/python/example_add_quant.py @@ -0,0 +1,25 @@ +from ggml import lib, ffi +from ggml.utils import init, copy, numpy +import numpy as np + +ctx = init(mem_size=12*1024*1024) # automatically freed when pointer is GC'd +n = 256 +n_threads = 4 + +a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) +b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # can't both be quantized +sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too! + +# See cffi's doc on how to allocate native memory: it's very simple! +# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface +gf = ffi.new('struct ggml_cgraph*') +lib.ggml_build_forward_expand(gf, sum) + +copy(np.array([i for i in range(n)], np.float32), a) +copy(np.array([i*100 for i in range(n)], np.float32), b) + +lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads) + +print(numpy(a, allow_copy=True)) +print(numpy(b)) +print(numpy(sum, allow_copy=True)) \ No newline at end of file diff --git a/ggml/examples/python/example_test_all_quants.py b/ggml/examples/python/example_test_all_quants.py new file mode 100644 index 0000000..8d3c966 --- /dev/null +++ b/ggml/examples/python/example_test_all_quants.py @@ -0,0 +1,68 @@ +from ggml import ffi, lib +from ggml.utils import init, numpy, copy +import numpy as np +from math import pi, cos, sin, ceil + +import matplotlib.pyplot as plt + +ctx = init(mem_size=100*1024*1024) # Will be auto-GC'd +n = 256 + +orig = np.array([ + [ + cos(j * 2 * pi / n) * (sin(i * 2 * pi / n)) + for j in range(n) + ] + for i in range(n) +], np.float32) +orig_tensor = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, n, n) +copy(orig, orig_tensor) + +quants = [ + type for type in range(lib.GGML_TYPE_COUNT) + if lib.ggml_is_quantized(type) and + type not in [lib.GGML_TYPE_Q8_1, lib.GGML_TYPE_Q8_K] # Apparently not supported +] +# quants = [lib.GGML_TYPE_Q2_K] # Test a single one + +def get_name(type): + name = lib.ggml_type_name(type) + return ffi.string(name).decode('utf-8') if name else '?' + +quants.sort(key=get_name) +quants.insert(0, None) +print(quants) + +ncols=4 +nrows = ceil(len(quants) / ncols) + +plt.figure(figsize=(ncols * 5, nrows * 5), layout='tight') + +for i, type in enumerate(quants): + plt.subplot(nrows, ncols, i + 1) + try: + if type == None: + plt.title('Original') + plt.imshow(orig) + else: + quantized_tensor = lib.ggml_new_tensor_2d(ctx, type, n, n) + copy(orig_tensor, quantized_tensor) + quantized = numpy(quantized_tensor, allow_copy=True) + d = quantized - orig + results = { + "l2": np.linalg.norm(d, 2), + "linf": np.linalg.norm(d, np.inf), + "compression": + round(lib.ggml_nbytes(orig_tensor) / + lib.ggml_nbytes(quantized_tensor), 1) + } + name = get_name(type) + print(f'{name}: {results}') + + plt.title(f'{name} ({results["compression"]}x smaller)') + plt.imshow(quantized, interpolation='nearest') + + except Exception as e: + print(f'Error: {e}') + +plt.show() \ No newline at end of file diff --git a/ggml/examples/python/ggml/__init__.py b/ggml/examples/python/ggml/__init__.py new file mode 100644 index 0000000..31a1910 --- /dev/null +++ b/ggml/examples/python/ggml/__init__.py @@ -0,0 +1,58 @@ +""" + Python bindings for the ggml library. + + Usage example: + + from ggml import lib, ffi + from ggml.utils import init, copy, numpy + import numpy as np + + ctx = init(mem_size=10*1024*1024) + n = 1024 + n_threads = 4 + + a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) + b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) + sum = lib.ggml_add(ctx, a, b) + + gf = ffi.new('struct ggml_cgraph*') + lib.ggml_build_forward_expand(gf, sum) + + copy(np.array([i for i in range(n)], np.float32), a) + copy(np.array([i*100 for i in range(n)], np.float32), b) + lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads) + + print(numpy(sum, allow_copy=True)) + + See https://cffi.readthedocs.io/en/latest/cdef.html for more on cffi. +""" + +try: + from ggml.cffi import ffi as ffi +except ImportError as e: + raise ImportError(f"Couldn't find ggml bindings ({e}). Run `python regenerate.py` or check your PYTHONPATH.") + +import os, platform + +__exact_library = os.environ.get("GGML_LIBRARY") +if __exact_library: + __candidates = [__exact_library] +elif platform.system() == "Windows": + __candidates = ["ggml_shared.dll", "llama.dll"] +else: + __candidates = ["libggml_shared.so", "libllama.so"] + if platform.system() == "Darwin": + __candidates += ["libggml_shared.dylib", "libllama.dylib"] + +for i, name in enumerate(__candidates): + try: + # This is where all the functions, enums and constants are defined + lib = ffi.dlopen(name) + except OSError: + if i < len(__candidates) - 1: + continue + raise OSError(f"Couldn't find ggml's shared library (tried names: {__candidates}). Add its directory to DYLD_LIBRARY_PATH (on Mac) or LD_LIBRARY_PATH, or define GGML_LIBRARY.") + +# This contains the cffi helpers such as new, cast, string, etc. +# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface +ffi = ffi diff --git a/ggml/examples/python/ggml/__init__.pyi b/ggml/examples/python/ggml/__init__.pyi new file mode 100644 index 0000000..5df2a88 --- /dev/null +++ b/ggml/examples/python/ggml/__init__.pyi @@ -0,0 +1,2406 @@ +# auto-generated file +import ggml.ffi as ffi +import numpy as np +class lib: + @property + def GGML_BACKEND_CPU(self) -> int: ... + @property + def GGML_BACKEND_GPU(self) -> int: ... + @property + def GGML_BACKEND_GPU_SPLIT(self) -> int: ... + @property + def GGML_FTYPE_ALL_F32(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_F16(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q2_K(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q3_K(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q4_0(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q4_1(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q4_1_SOME_F16(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q4_K(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q5_0(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q5_1(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q5_K(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q6_K(self) -> int: ... + @property + def GGML_FTYPE_MOSTLY_Q8_0(self) -> int: ... + @property + def GGML_FTYPE_UNKNOWN(self) -> int: ... + @property + def GGML_LINESEARCH_BACKTRACKING_ARMIJO(self) -> int: ... + @property + def GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE(self) -> int: ... + @property + def GGML_LINESEARCH_BACKTRACKING_WOLFE(self) -> int: ... + @property + def GGML_LINESEARCH_DEFAULT(self) -> int: ... + @property + def GGML_LINESEARCH_FAIL(self) -> int: ... + @property + def GGML_LINESEARCH_INVALID_PARAMETERS(self) -> int: ... + @property + def GGML_LINESEARCH_MAXIMUM_ITERATIONS(self) -> int: ... + @property + def GGML_LINESEARCH_MAXIMUM_STEP(self) -> int: ... + @property + def GGML_LINESEARCH_MINIMUM_STEP(self) -> int: ... + @property + def GGML_OBJECT_GRAPH(self) -> int: ... + @property + def GGML_OBJECT_TENSOR(self) -> int: ... + @property + def GGML_OBJECT_WORK_BUFFER(self) -> int: ... + @property + def GGML_OPT_ADAM(self) -> int: ... + @property + def GGML_OPT_DID_NOT_CONVERGE(self) -> int: ... + @property + def GGML_OPT_FAIL(self) -> int: ... + @property + def GGML_OPT_INVALID_WOLFE(self) -> int: ... + @property + def GGML_OPT_LBFGS(self) -> int: ... + @property + def GGML_OPT_NO_CONTEXT(self) -> int: ... + @property + def GGML_OPT_OK(self) -> int: ... + @property + def GGML_OP_ACC(self) -> int: ... + @property + def GGML_OP_ADD(self) -> int: ... + @property + def GGML_OP_ADD1(self) -> int: ... + @property + def GGML_OP_ALIBI(self) -> int: ... + @property + def GGML_OP_ARGMAX(self) -> int: ... + @property + def GGML_OP_CLAMP(self) -> int: ... + @property + def GGML_OP_CONT(self) -> int: ... + @property + def GGML_OP_CONV_1D(self) -> int: ... + @property + def GGML_OP_CONV_2D(self) -> int: ... + @property + def GGML_OP_COUNT(self) -> int: ... + @property + def GGML_OP_CPY(self) -> int: ... + @property + def GGML_OP_CROSS_ENTROPY_LOSS(self) -> int: ... + @property + def GGML_OP_CROSS_ENTROPY_LOSS_BACK(self) -> int: ... + @property + def GGML_OP_DIAG(self) -> int: ... + @property + def GGML_OP_DIAG_MASK_INF(self) -> int: ... + @property + def GGML_OP_DIAG_MASK_ZERO(self) -> int: ... + @property + def GGML_OP_DIV(self) -> int: ... + @property + def GGML_OP_DUP(self) -> int: ... + @property + def GGML_OP_FLASH_ATTN(self) -> int: ... + @property + def GGML_OP_FLASH_ATTN_BACK(self) -> int: ... + @property + def GGML_OP_FLASH_FF(self) -> int: ... + @property + def GGML_OP_GET_ROWS(self) -> int: ... + @property + def GGML_OP_GET_ROWS_BACK(self) -> int: ... + @property + def GGML_OP_LOG(self) -> int: ... + @property + def GGML_OP_MAP_BINARY(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM1(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM1_F32(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM2(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM2_F32(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM3(self) -> int: ... + @property + def GGML_OP_MAP_CUSTOM3_F32(self) -> int: ... + @property + def GGML_OP_MAP_UNARY(self) -> int: ... + @property + def GGML_OP_MEAN(self) -> int: ... + @property + def GGML_OP_MUL(self) -> int: ... + @property + def GGML_OP_MUL_MAT(self) -> int: ... + @property + def GGML_OP_NONE(self) -> int: ... + @property + def GGML_OP_NORM(self) -> int: ... + @property + def GGML_OP_OUT_PROD(self) -> int: ... + @property + def GGML_OP_PERMUTE(self) -> int: ... + @property + def GGML_OP_POOL_1D(self) -> int: ... + @property + def GGML_OP_POOL_2D(self) -> int: ... + @property + def GGML_OP_POOL_AVG(self) -> int: ... + @property + def GGML_OP_POOL_COUNT(self) -> int: ... + @property + def GGML_OP_POOL_MAX(self) -> int: ... + @property + def GGML_OP_REPEAT(self) -> int: ... + @property + def GGML_OP_REPEAT_BACK(self) -> int: ... + @property + def GGML_OP_RESHAPE(self) -> int: ... + @property + def GGML_OP_RMS_NORM(self) -> int: ... + @property + def GGML_OP_RMS_NORM_BACK(self) -> int: ... + @property + def GGML_OP_ROPE(self) -> int: ... + @property + def GGML_OP_ROPE_BACK(self) -> int: ... + @property + def GGML_OP_SCALE(self) -> int: ... + @property + def GGML_OP_SET(self) -> int: ... + @property + def GGML_OP_SILU_BACK(self) -> int: ... + @property + def GGML_OP_SOFT_MAX(self) -> int: ... + @property + def GGML_OP_SOFT_MAX_BACK(self) -> int: ... + @property + def GGML_OP_SQR(self) -> int: ... + @property + def GGML_OP_SQRT(self) -> int: ... + @property + def GGML_OP_SUB(self) -> int: ... + @property + def GGML_OP_SUM(self) -> int: ... + @property + def GGML_OP_SUM_ROWS(self) -> int: ... + @property + def GGML_OP_TRANSPOSE(self) -> int: ... + @property + def GGML_OP_UNARY(self) -> int: ... + @property + def GGML_OP_VIEW(self) -> int: ... + @property + def GGML_OP_WIN_PART(self) -> int: ... + @property + def GGML_OP_WIN_UNPART(self) -> int: ... + @property + def GGML_TASK_COMPUTE(self) -> int: ... + @property + def GGML_TASK_FINALIZE(self) -> int: ... + @property + def GGML_TASK_INIT(self) -> int: ... + @property + def GGML_TYPE_COUNT(self) -> int: ... + @property + def GGML_TYPE_F16(self) -> int: ... + @property + def GGML_TYPE_F32(self) -> int: ... + @property + def GGML_TYPE_I16(self) -> int: ... + @property + def GGML_TYPE_I32(self) -> int: ... + @property + def GGML_TYPE_I8(self) -> int: ... + @property + def GGML_TYPE_Q2_K(self) -> int: ... + @property + def GGML_TYPE_Q3_K(self) -> int: ... + @property + def GGML_TYPE_Q4_0(self) -> int: ... + @property + def GGML_TYPE_Q4_1(self) -> int: ... + @property + def GGML_TYPE_Q4_K(self) -> int: ... + @property + def GGML_TYPE_Q5_0(self) -> int: ... + @property + def GGML_TYPE_Q5_1(self) -> int: ... + @property + def GGML_TYPE_Q5_K(self) -> int: ... + @property + def GGML_TYPE_Q6_K(self) -> int: ... + @property + def GGML_TYPE_Q8_0(self) -> int: ... + @property + def GGML_TYPE_Q8_1(self) -> int: ... + @property + def GGML_TYPE_Q8_K(self) -> int: ... + @property + def GGML_UNARY_OP_ABS(self) -> int: ... + @property + def GGML_UNARY_OP_ELU(self) -> int: ... + @property + def GGML_UNARY_OP_GELU(self) -> int: ... + @property + def GGML_UNARY_OP_GELU_QUICK(self) -> int: ... + @property + def GGML_UNARY_OP_NEG(self) -> int: ... + @property + def GGML_UNARY_OP_RELU(self) -> int: ... + @property + def GGML_UNARY_OP_SGN(self) -> int: ... + @property + def GGML_UNARY_OP_SILU(self) -> int: ... + @property + def GGML_UNARY_OP_STEP(self) -> int: ... + @property + def GGML_UNARY_OP_TANH(self) -> int: ... + @property + def GGUF_TYPE_ARRAY(self) -> int: ... + @property + def GGUF_TYPE_BOOL(self) -> int: ... + @property + def GGUF_TYPE_COUNT(self) -> int: ... + @property + def GGUF_TYPE_FLOAT32(self) -> int: ... + @property + def GGUF_TYPE_INT16(self) -> int: ... + @property + def GGUF_TYPE_INT32(self) -> int: ... + @property + def GGUF_TYPE_INT8(self) -> int: ... + @property + def GGUF_TYPE_STRING(self) -> int: ... + @property + def GGUF_TYPE_UINT16(self) -> int: ... + @property + def GGUF_TYPE_UINT32(self) -> int: ... + @property + def GGUF_TYPE_UINT8(self) -> int: ... + def abort_callback(data: ffi.CData) -> bool: + """ + abort ggml_graph_compute when true + + bool (*abort_callback)(void * data); + """ + ... + def dequantize_row_q2_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """ + Dequantization + + void dequantize_row_q2_K(const block_q2_K * restrict x, float * restrict y, int k); + """ + ... + def dequantize_row_q3_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void dequantize_row_q3_K(const block_q3_K * restrict x, float * restrict y, int k);""" + ... + def dequantize_row_q4_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void dequantize_row_q4_K(const block_q4_K * restrict x, float * restrict y, int k);""" + ... + def dequantize_row_q5_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void dequantize_row_q5_K(const block_q5_K * restrict x, float * restrict y, int k);""" + ... + def dequantize_row_q6_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void dequantize_row_q6_K(const block_q6_K * restrict x, float * restrict y, int k);""" + ... + def dequantize_row_q8_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void dequantize_row_q8_K(const block_q8_K * restrict x, float * restrict y, int k);""" + ... + def ggml_abs(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_abs_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_acc(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + """ + ... + def ggml_acc_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + """ + ... + def ggml_add(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_add1(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_add1_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_add_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_alibi(ctx: ffi.CData, a: ffi.CData, n_past: int, n_head: int, bias_max: float) -> ffi.CData: + """ + alibi position embedding + in-place, returns view(a) + + struct ggml_tensor * ggml_alibi( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_head, + float bias_max); + """ + ... + def ggml_allocr_alloc(alloc: ffi.CData, tensor: ffi.CData) -> None: + """GGML_API void ggml_allocr_alloc(struct ggml_allocr * alloc, struct ggml_tensor * tensor);""" + ... + def ggml_allocr_alloc_graph(alloc: ffi.CData, graph: ffi.CData) -> int: + """GGML_API size_t ggml_allocr_alloc_graph(struct ggml_allocr * alloc, struct ggml_cgraph * graph);""" + ... + def ggml_allocr_free(alloc: ffi.CData) -> None: + """GGML_API void ggml_allocr_free(struct ggml_allocr * alloc);""" + ... + def ggml_allocr_is_measure(alloc: ffi.CData) -> bool: + """GGML_API bool ggml_allocr_is_measure(struct ggml_allocr * alloc);""" + ... + def ggml_allocr_new(data: ffi.CData, size: int, alignment: int) -> ffi.CData: + """GGML_API struct ggml_allocr * ggml_allocr_new(void * data, size_t size, size_t alignment);""" + ... + def ggml_allocr_new_measure(alignment: int) -> ffi.CData: + """GGML_API struct ggml_allocr * ggml_allocr_new_measure(size_t alignment);""" + ... + def ggml_allocr_reset(alloc: ffi.CData) -> None: + """GGML_API void ggml_allocr_reset(struct ggml_allocr * alloc);""" + ... + def ggml_allocr_set_parse_seq(alloc: ffi.CData, list: ffi.CData, n: int) -> None: + """ + tell the allocator to parse nodes following the order described in the list + you should call this if your graph are optimized to execute out-of-order + + GGML_API void ggml_allocr_set_parse_seq(struct ggml_allocr * alloc, int * list, int n); + """ + ... + def ggml_are_same_shape(t0: ffi.CData, t1: ffi.CData) -> bool: + """ GGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);""" + ... + def ggml_argmax(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + argmax along rows + + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_blck_size(type: int) -> int: + """ GGML_API int ggml_blck_size (enum ggml_type type);""" + ... + def ggml_build_backward(ctx: ffi.CData, gf: ffi.CData, keep: bool) -> ffi.CData: + """ GGML_API struct ggml_cgraph ggml_build_backward(struct ggml_context * ctx, struct ggml_cgraph * gf, bool keep);""" + ... + def ggml_build_forward(tensor: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_cgraph ggml_build_forward (struct ggml_tensor * tensor);""" + ... + def ggml_build_forward_ctx(ctx: ffi.CData, tensor: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_cgraph * ggml_build_forward_ctx(struct ggml_context * ctx, struct ggml_tensor * tensor);""" + ... + def ggml_build_forward_expand(cgraph: ffi.CData, tensor: ffi.CData) -> None: + """ GGML_API void ggml_build_forward_expand(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor);""" + ... + def ggml_cl_can_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> bool: + """bool ggml_cl_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);""" + ... + def ggml_cl_free_data(tensor: ffi.CData) -> None: + """void ggml_cl_free_data(const struct ggml_tensor* tensor);""" + ... + def ggml_cl_host_free(ptr: ffi.CData) -> None: + """void ggml_cl_host_free(void * ptr);""" + ... + def ggml_cl_host_malloc(size: int) -> ffi.CData: + """void * ggml_cl_host_malloc(size_t size);""" + ... + def ggml_cl_init() -> None: + """void ggml_cl_init(void);""" + ... + def ggml_cl_mul(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> None: + """void ggml_cl_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);""" + ... + def ggml_cl_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData, wdata: ffi.CData, wsize: int) -> None: + """void ggml_cl_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize);""" + ... + def ggml_cl_mul_mat_get_wsize(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> int: + """size_t ggml_cl_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);""" + ... + def ggml_cl_transform_tensor(data: ffi.CData, tensor: ffi.CData) -> None: + """void ggml_cl_transform_tensor(void * data, struct ggml_tensor * tensor);""" + ... + def ggml_clamp(ctx: ffi.CData, a: ffi.CData, min: float, max: float) -> ffi.CData: + """ + clamp + in-place, returns view(a) + + struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + """ + ... + def ggml_cont(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + make contiguous + + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_conv_1d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s0: int, p0: int, d0: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int s0, // stride + int p0, // padding + int d0); // dilation + """ + ... + def ggml_conv_1d_ph(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s: int, d: int) -> ffi.CData: + """ + conv_1d with padding = half + alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + + GGML_API struct ggml_tensor * ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int s, + int d); + """ + ... + def ggml_conv_2d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, s0: int, s1: int, p0: int, p1: int, d0: int, d1: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int s0, + int s1, + int p0, + int p1, + int d0, + int d1); + """ + ... + def ggml_cpu_has_arm_fma() -> int: + """ GGML_API int ggml_cpu_has_arm_fma (void);""" + ... + def ggml_cpu_has_avx() -> int: + """ GGML_API int ggml_cpu_has_avx (void);""" + ... + def ggml_cpu_has_avx2() -> int: + """ GGML_API int ggml_cpu_has_avx2 (void);""" + ... + def ggml_cpu_has_avx512() -> int: + """ GGML_API int ggml_cpu_has_avx512 (void);""" + ... + def ggml_cpu_has_avx512_vbmi() -> int: + """ GGML_API int ggml_cpu_has_avx512_vbmi(void);""" + ... + def ggml_cpu_has_avx512_vnni() -> int: + """ GGML_API int ggml_cpu_has_avx512_vnni(void);""" + ... + def ggml_cpu_has_blas() -> int: + """ GGML_API int ggml_cpu_has_blas (void);""" + ... + def ggml_cpu_has_clblast() -> int: + """ GGML_API int ggml_cpu_has_clblast (void);""" + ... + def ggml_cpu_has_cuda() -> int: + """ GGML_API int ggml_cpu_has_cuda (void);""" + ... + def ggml_cpu_has_f16c() -> int: + """ GGML_API int ggml_cpu_has_f16c (void);""" + ... + def ggml_cpu_has_fma() -> int: + """ GGML_API int ggml_cpu_has_fma (void);""" + ... + def ggml_cpu_has_fp16_va() -> int: + """ GGML_API int ggml_cpu_has_fp16_va (void);""" + ... + def ggml_cpu_has_gpublas() -> int: + """ GGML_API int ggml_cpu_has_gpublas (void);""" + ... + def ggml_cpu_has_neon() -> int: + """ GGML_API int ggml_cpu_has_neon (void);""" + ... + def ggml_cpu_has_sse3() -> int: + """ GGML_API int ggml_cpu_has_sse3 (void);""" + ... + def ggml_cpu_has_vsx() -> int: + """ GGML_API int ggml_cpu_has_vsx (void);""" + ... + def ggml_cpu_has_wasm_simd() -> int: + """ GGML_API int ggml_cpu_has_wasm_simd (void);""" + ... + def ggml_cpy(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + a -> b, return view(b) + + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_cross_entropy_loss(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_cross_entropy_loss_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c); + """ + ... + def ggml_cuda_assign_buffers(tensor: ffi.CData) -> None: + """GGML_API void ggml_cuda_assign_buffers(struct ggml_tensor * tensor);""" + ... + def ggml_cuda_assign_buffers_force_inplace(tensor: ffi.CData) -> None: + """GGML_API void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor);""" + ... + def ggml_cuda_assign_buffers_no_scratch(tensor: ffi.CData) -> None: + """GGML_API void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor);""" + ... + def ggml_cuda_can_mul_mat(src0: ffi.CData, src1: ffi.CData, dst: ffi.CData) -> bool: + """GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst);""" + ... + def ggml_cuda_compute_forward(params: ffi.CData, tensor: ffi.CData) -> bool: + """GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor);""" + ... + def ggml_cuda_free_data(tensor: ffi.CData) -> None: + """GGML_API void ggml_cuda_free_data(struct ggml_tensor * tensor);""" + ... + def ggml_cuda_free_scratch() -> None: + """GGML_API void ggml_cuda_free_scratch(void);""" + ... + def ggml_cuda_get_device_count() -> int: + """GGML_API int ggml_cuda_get_device_count(void);""" + ... + def ggml_cuda_get_device_description(device: int, description: ffi.CData, description_size: int) -> None: + """GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size);""" + ... + def ggml_cuda_host_free(ptr: ffi.CData) -> None: + """GGML_API void ggml_cuda_host_free(void * ptr);""" + ... + def ggml_cuda_host_malloc(size: int) -> ffi.CData: + """GGML_API void * ggml_cuda_host_malloc(size_t size);""" + ... + def ggml_cuda_set_main_device(main_device: int) -> None: + """GGML_API void ggml_cuda_set_main_device(int main_device);""" + ... + def ggml_cuda_set_mul_mat_q(mul_mat_q: bool) -> None: + """GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q);""" + ... + def ggml_cuda_set_scratch_size(scratch_size: int) -> None: + """GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size);""" + ... + def ggml_cuda_set_tensor_split(tensor_split: ffi.CData) -> None: + """GGML_API void ggml_cuda_set_tensor_split(const float * tensor_split);""" + ... + def ggml_cuda_transform_tensor(data: ffi.CData, tensor: ffi.CData) -> None: + """GGML_API void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor);""" + ... + def ggml_cycles() -> int: + """ GGML_API int64_t ggml_cycles(void);""" + ... + def ggml_cycles_per_ms() -> int: + """ GGML_API int64_t ggml_cycles_per_ms(void);""" + ... + def ggml_diag(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_diag_mask_inf(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData: + """ + set elements above the diagonal to -INF + + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + """ + ... + def ggml_diag_mask_inf_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + """ + ... + def ggml_diag_mask_zero(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData: + """ + set elements above the diagonal to 0 + + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + """ + ... + def ggml_diag_mask_zero_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + """ + ... + def ggml_div(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_div_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_dup(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_dup_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_dup_tensor(ctx: ffi.CData, src: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src);""" + ... + def ggml_element_size(tensor: ffi.CData) -> int: + """ GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);""" + ... + def ggml_elu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_elu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_flash_attn(ctx: ffi.CData, q: ffi.CData, k: ffi.CData, v: ffi.CData, masked: bool) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_flash_attn( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + bool masked); + """ + ... + def ggml_flash_attn_back(ctx: ffi.CData, q: ffi.CData, k: ffi.CData, v: ffi.CData, d: ffi.CData, masked: bool) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + """ + ... + def ggml_flash_ff(ctx: ffi.CData, a: ffi.CData, b0: ffi.CData, b1: ffi.CData, c0: ffi.CData, c1: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_flash_ff( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b0, + struct ggml_tensor * b1, + struct ggml_tensor * c0, + struct ggml_tensor * c1); + """ + ... + def ggml_format_name(tensor: ffi.CData, fmt: ffi.CData, *args2) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...);""" + ... + def ggml_fp16_to_fp32(x: np.float16) -> float: + """ + convert FP16 <-> FP32 + + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t x); + """ + ... + def ggml_fp16_to_fp32_row(x: ffi.CData, y: ffi.CData, n: int) -> None: + """ GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int n);""" + ... + def ggml_fp32_to_fp16(x: float) -> np.float16: + """ GGML_API ggml_fp16_t ggml_fp32_to_fp16(float x);""" + ... + def ggml_fp32_to_fp16_row(x: ffi.CData, y: ffi.CData, n: int) -> None: + """ GGML_API void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n);""" + ... + def ggml_free(ctx: ffi.CData) -> None: + """ GGML_API void ggml_free(struct ggml_context * ctx);""" + ... + def ggml_ftype_to_ggml_type(ftype: int) -> int: + """ + TODO: temporary until model loading of ggml examples is refactored + + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + """ + ... + def ggml_gelu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + TODO: double-check this computation is correct + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_gelu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_gelu_quick(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_gelu_quick_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_get_data(tensor: ffi.CData) -> ffi.CData: + """ GGML_API void * ggml_get_data (const struct ggml_tensor * tensor);""" + ... + def ggml_get_data_f32(tensor: ffi.CData) -> ffi.CData: + """ GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor);""" + ... + def ggml_get_f32_1d(tensor: ffi.CData, i: int) -> float: + """ GGML_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i);""" + ... + def ggml_get_i32_1d(tensor: ffi.CData, i: int) -> int: + """ GGML_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i);""" + ... + def ggml_get_max_tensor_size(ctx: ffi.CData) -> int: + """ GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx);""" + ... + def ggml_get_mem_buffer(ctx: ffi.CData) -> ffi.CData: + """ GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx);""" + ... + def ggml_get_mem_size(ctx: ffi.CData) -> int: + """ GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx);""" + ... + def ggml_get_name(tensor: ffi.CData) -> ffi.CData: + """ GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor);""" + ... + def ggml_get_no_alloc(ctx: ffi.CData) -> bool: + """ GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx);""" + ... + def ggml_get_rows(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_get_rows_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c); + """ + ... + def ggml_get_tensor(ctx: ffi.CData, name: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);""" + ... + def ggml_get_unary_op(tensor: ffi.CData) -> int: + """ GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor);""" + ... + def ggml_graph_compute(cgraph: ffi.CData, cplan: ffi.CData) -> int: + """ GGML_API int ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);""" + ... + def ggml_graph_compute_with_ctx(ctx: ffi.CData, cgraph: ffi.CData, n_threads: int) -> None: + """ + same as ggml_graph_compute() but the work data is allocated as a part of the context + note: the drawback of this API is that you must have ensured that the context has enough memory for the work data + + GGML_API void ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + """ + ... + def ggml_graph_dump_dot(gb: ffi.CData, gf: ffi.CData, filename: ffi.CData) -> None: + """ + dump the graph into a file using the dot format + + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * gf, const char * filename); + """ + ... + def ggml_graph_get_tensor(cgraph: ffi.CData, name: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_graph_get_tensor(struct ggml_cgraph * cgraph, const char * name);""" + ... + def ggml_graph_overhead() -> int: + """ GGML_API size_t ggml_graph_overhead(void);""" + ... + def ggml_graph_plan(cgraph: ffi.CData, n_threads: int) -> ffi.CData: + """ + ggml_graph_plan() has to be called before ggml_graph_compute() + when plan.work_size > 0, caller must allocate memory for plan.work_data + + GGML_API struct ggml_cplan ggml_graph_plan (struct ggml_cgraph * cgraph, int n_threads /*= GGML_DEFAULT_N_THREADS*/); + """ + ... + def ggml_graph_print(cgraph: ffi.CData) -> None: + """ + print info and performance information for the graph + + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + """ + ... + def ggml_graph_reset(cgraph: ffi.CData) -> None: + """ GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph);""" + ... + def ggml_init(params: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_context * ggml_init(struct ggml_init_params params);""" + ... + def ggml_init_cuda() -> None: + """GGML_API void ggml_init_cuda(void);""" + ... + def ggml_internal_get_type_traits(type: int) -> ffi.CData: + """ ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type);""" + ... + def ggml_is_contiguous(tensor: ffi.CData) -> bool: + """ GGML_API bool ggml_is_contiguous(const struct ggml_tensor * tensor);""" + ... + def ggml_is_numa() -> bool: + """ GGML_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node""" + ... + def ggml_is_permuted(tensor: ffi.CData) -> bool: + """ GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);""" + ... + def ggml_is_quantized(type: int) -> bool: + """ GGML_API bool ggml_is_quantized(enum ggml_type type);""" + ... + def ggml_is_transposed(tensor: ffi.CData) -> bool: + """ GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);""" + ... + def ggml_log(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_log_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_map_binary_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_binary_op_f32_t fun), + "use ggml_map_custom2 instead"); + """ + ... + def ggml_map_binary_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_binary_inplace_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_binary_op_f32_t fun), + "use ggml_map_custom2_inplace instead"); + """ + ... + def ggml_map_custom1(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom1_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_f32_t fun), + "use ggml_map_custom1 instead"); + """ + ... + def ggml_map_custom1_inplace(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom1_inplace_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom1_inplace_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_f32_t fun), + "use ggml_map_custom1_inplace instead"); + """ + ... + def ggml_map_custom2(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom2_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_f32_t fun), + "use ggml_map_custom2 instead"); + """ + ... + def ggml_map_custom2_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom2_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom2_inplace_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_f32_t fun), + "use ggml_map_custom2_inplace instead"); + """ + ... + def ggml_map_custom3(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom3_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_f32_t fun), + "use ggml_map_custom3 instead"); + """ + ... + def ggml_map_custom3_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData, n_tasks: int, userdata: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + """ + ... + def ggml_map_custom3_inplace_f32(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, c: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_custom3_inplace_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_f32_t fun), + "use ggml_map_custom3_inplace instead"); + """ + ... + def ggml_map_unary_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_unary_op_f32_t fun), + "use ggml_map_custom1 instead"); + """ + ... + def ggml_map_unary_inplace_f32(ctx: ffi.CData, a: ffi.CData, fun: ffi.CData) -> ffi.CData: + """ + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_map_unary_inplace_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_unary_op_f32_t fun), + "use ggml_map_custom1_inplace instead"); + """ + ... + def ggml_mean(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + mean along rows + + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_metal_add_buffer(ctx: ffi.CData, name: ffi.CData, data: ffi.CData, size: int, max_size: int) -> bool: + """ + creates a mapping between a host memory buffer and a device memory buffer + - make sure to map all buffers used in the graph before calling ggml_metal_graph_compute + - the mapping is used during computation to determine the arguments of the compute kernels + - you don't need to keep the host memory buffer allocated as it is never accessed by Metal + - max_size specifies the maximum size of a tensor and is used to create shared views such + that it is guaranteed that the tensor will fit in at least one of the views + + + bool ggml_metal_add_buffer( + struct ggml_metal_context * ctx, + const char * name, + void * data, + size_t size, + size_t max_size); + """ + ... + def ggml_metal_free(ctx: ffi.CData) -> None: + """void ggml_metal_free(struct ggml_metal_context * ctx);""" + ... + def ggml_metal_get_concur_list(ctx: ffi.CData) -> ffi.CData: + """ + output the concur_list for ggml_alloc + + int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx); + """ + ... + def ggml_metal_get_tensor(ctx: ffi.CData, t: ffi.CData) -> None: + """ + get data from the device into host memory + + void ggml_metal_get_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t); + """ + ... + def ggml_metal_graph_compute(ctx: ffi.CData, gf: ffi.CData) -> None: + """ + same as ggml_graph_compute but uses Metal + creates gf->n_threads command buffers in parallel + + void ggml_metal_graph_compute(struct ggml_metal_context * ctx, struct ggml_cgraph * gf); + """ + ... + def ggml_metal_graph_find_concurrency(ctx: ffi.CData, gf: ffi.CData, check_mem: bool) -> None: + """ + try to find operations that can be run concurrently in the graph + you should run it again if the topology of your graph changes + + void ggml_metal_graph_find_concurrency(struct ggml_metal_context * ctx, struct ggml_cgraph * gf, bool check_mem); + """ + ... + def ggml_metal_host_free(data: ffi.CData) -> None: + """void ggml_metal_host_free (void * data);""" + ... + def ggml_metal_host_malloc(n: int) -> ffi.CData: + """void * ggml_metal_host_malloc(size_t n);""" + ... + def ggml_metal_if_optimized(ctx: ffi.CData) -> int: + """ + if the graph has been optimized for concurrently dispatch, return length of the concur_list if optimized + + int ggml_metal_if_optimized(struct ggml_metal_context * ctx); + """ + ... + def ggml_metal_init(n_cb: int) -> ffi.CData: + """ + number of command buffers to use + + struct ggml_metal_context * ggml_metal_init(int n_cb); + """ + ... + def ggml_metal_set_n_cb(ctx: ffi.CData, n_cb: int) -> None: + """ + set the number of command buffers to use + + void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb); + """ + ... + def ggml_metal_set_tensor(ctx: ffi.CData, t: ffi.CData) -> None: + """ + set data from host memory into the device + + void ggml_metal_set_tensor(struct ggml_metal_context * ctx, struct ggml_tensor * t); + """ + ... + def ggml_mpi_backend_free() -> None: + """void ggml_mpi_backend_free(void);""" + ... + def ggml_mpi_backend_init() -> None: + """void ggml_mpi_backend_init(void);""" + ... + def ggml_mpi_eval_init(ctx_mpi: ffi.CData, n_tokens: ffi.CData, n_past: ffi.CData, n_threads: ffi.CData) -> None: + """ + void ggml_mpi_eval_init( + struct ggml_mpi_context * ctx_mpi, + int * n_tokens, + int * n_past, + int * n_threads); + """ + ... + def ggml_mpi_free(ctx: ffi.CData) -> None: + """void ggml_mpi_free(struct ggml_mpi_context * ctx);""" + ... + def ggml_mpi_graph_compute_post(ctx_mpi: ffi.CData, gf: ffi.CData, n_layers: int) -> None: + """ + void ggml_mpi_graph_compute_post( + struct ggml_mpi_context * ctx_mpi, + struct ggml_cgraph * gf, + int n_layers); + """ + ... + def ggml_mpi_graph_compute_pre(ctx_mpi: ffi.CData, gf: ffi.CData, n_layers: int) -> None: + """ + void ggml_mpi_graph_compute_pre( + struct ggml_mpi_context * ctx_mpi, + struct ggml_cgraph * gf, + int n_layers); + """ + ... + def ggml_mpi_init() -> ffi.CData: + """struct ggml_mpi_context * ggml_mpi_init(void);""" + ... + def ggml_mpi_rank(ctx: ffi.CData) -> int: + """int ggml_mpi_rank(struct ggml_mpi_context * ctx);""" + ... + def ggml_mul(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_mul_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_mul_mat(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + A: n columns, m rows + B: n columns, p rows (i.e. we transpose it internally) + result is m columns, p rows + + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_nbytes(tensor: ffi.CData) -> int: + """ GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);""" + ... + def ggml_nbytes_pad(tensor: ffi.CData) -> int: + """ GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN""" + ... + def ggml_nbytes_split(tensor: ffi.CData, nrows_split: int) -> int: + """ GGML_API size_t ggml_nbytes_split(const struct ggml_tensor * tensor, int nrows_split);""" + ... + def ggml_neg(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_neg_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_nelements(tensor: ffi.CData) -> int: + """ GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor);""" + ... + def ggml_new_f32(ctx: ffi.CData, value: float) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value);""" + ... + def ggml_new_graph(ctx: ffi.CData) -> ffi.CData: + """ + graph allocation in a context + + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); + """ + ... + def ggml_new_i32(ctx: ffi.CData, value: int) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value);""" + ... + def ggml_new_tensor(ctx: ffi.CData, type: int, n_dims: int, ne: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + """ + ... + def ggml_new_tensor_1d(ctx: ffi.CData, type: int, ne0: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + """ + ... + def ggml_new_tensor_2d(ctx: ffi.CData, type: int, ne0: int, ne1: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + """ + ... + def ggml_new_tensor_3d(ctx: ffi.CData, type: int, ne0: int, ne1: int, ne2: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + """ + ... + def ggml_new_tensor_4d(ctx: ffi.CData, type: int, ne0: int, ne1: int, ne2: int, ne3: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + """ + ... + def ggml_norm(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + normalize along rows + TODO: eps is hardcoded to 1e-5 for now + + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_norm_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_nrows(tensor: ffi.CData) -> int: + """ GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor);""" + ... + def ggml_numa_init() -> None: + """ GGML_API void ggml_numa_init(void); // call once for better performance on NUMA systems""" + ... + def ggml_op_name(op: int) -> ffi.CData: + """ GGML_API const char * ggml_op_name (enum ggml_op op);""" + ... + def ggml_op_symbol(op: int) -> ffi.CData: + """ GGML_API const char * ggml_op_symbol(enum ggml_op op);""" + ... + def ggml_opt(ctx: ffi.CData, params: ffi.CData, f: ffi.CData) -> int: + """ + optimize the function defined by the tensor f + + GGML_API enum ggml_opt_result ggml_opt( + struct ggml_context * ctx, + struct ggml_opt_params params, + struct ggml_tensor * f); + """ + ... + def ggml_opt_default_params(type: int) -> ffi.CData: + """ GGML_API struct ggml_opt_params ggml_opt_default_params(enum ggml_opt_type type);""" + ... + def ggml_opt_init(ctx: ffi.CData, opt: ffi.CData, params: ffi.CData, nx: int) -> None: + """ + initialize optimizer context + + GGML_API void ggml_opt_init( + struct ggml_context * ctx, + struct ggml_opt_context * opt, + struct ggml_opt_params params, + int64_t nx); + """ + ... + def ggml_opt_resume(ctx: ffi.CData, opt: ffi.CData, f: ffi.CData) -> int: + """ + continue optimizing the function defined by the tensor f + + GGML_API enum ggml_opt_result ggml_opt_resume( + struct ggml_context * ctx, + struct ggml_opt_context * opt, + struct ggml_tensor * f); + """ + ... + def ggml_opt_resume_g(ctx: ffi.CData, opt: ffi.CData, f: ffi.CData, gf: ffi.CData, gb: ffi.CData) -> int: + """ + continue optimizing the function defined by the tensor f + + GGML_API enum ggml_opt_result ggml_opt_resume_g( + struct ggml_context * ctx, + struct ggml_opt_context * opt, + struct ggml_tensor * f, + struct ggml_cgraph * gf, + struct ggml_cgraph * gb); + """ + ... + def ggml_out_prod(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + A: m columns, n rows, + B: p columns, n rows, + result is m columns, p rows + + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_permute(ctx: ffi.CData, a: ffi.CData, axis0: int, axis1: int, axis2: int, axis3: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + """ + ... + def ggml_pool_1d(ctx: ffi.CData, a: ffi.CData, op: int, k0: int, s0: int, p0: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + """ + ... + def ggml_pool_2d(ctx: ffi.CData, a: ffi.CData, op: int, k0: int, k1: int, s0: int, s1: int, p0: int, p1: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + int p0, + int p1); + """ + ... + def ggml_print_object(obj: ffi.CData) -> None: + """ GGML_API void ggml_print_object (const struct ggml_object * obj);""" + ... + def ggml_print_objects(ctx: ffi.CData) -> None: + """ GGML_API void ggml_print_objects(const struct ggml_context * ctx);""" + ... + def ggml_quantize_chunk(type: int, src: ffi.CData, dst: ffi.CData, start: int, n: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_chunk(enum ggml_type type, const float * src, void * dst, int start, int n, int64_t * hist);""" + ... + def ggml_quantize_q2_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ + Quantization with histogram collection + + size_t ggml_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist); + """ + ... + def ggml_quantize_q3_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """size_t ggml_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q4_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q4_1(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q4_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """size_t ggml_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q5_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q5_1(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q5_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """size_t ggml_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q6_K(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """size_t ggml_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_quantize_q8_0(src: ffi.CData, dst: ffi.CData, n: int, k: int, hist: ffi.CData) -> int: + """ GGML_API size_t ggml_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist);""" + ... + def ggml_relu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_relu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_repeat(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + if a is the same shape as b, and a is not parameter, return a + otherwise, return a new tensor: repeat(a) to fit in b + + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_repeat_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_reshape(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + return view(a), b specifies the new shape + TODO: when we start computing gradient, make a copy instead of view + + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_reshape_1d(ctx: ffi.CData, a: ffi.CData, ne0: int) -> ffi.CData: + """ + return view(a) + TODO: when we start computing gradient, make a copy instead of view + + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + """ + ... + def ggml_reshape_2d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + """ + ... + def ggml_reshape_3d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int) -> ffi.CData: + """ + return view(a) + TODO: when we start computing gradient, make a copy instead of view + + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + """ + ... + def ggml_reshape_4d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, ne3: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + """ + ... + def ggml_rms_norm(ctx: ffi.CData, a: ffi.CData, eps: float) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + """ + ... + def ggml_rms_norm_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + a - x + b - dy + TODO: update with configurable eps + + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_rms_norm_inplace(ctx: ffi.CData, a: ffi.CData, eps: float) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + """ + ... + def ggml_rope(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData: + """ + rotary position embedding + if mode & 1 == 1, skip n_past elements + if mode & 2 == 1, GPT-NeoX style + if mode & 4 == 1, ChatGLM style + TODO: avoid creating a new tensor every time + + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_dims, + int mode, + int n_ctx); + """ + ... + def ggml_rope_back(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData: + """ + rotary position embedding backward, i.e compute dx from dy + a - dy + + GGML_API struct ggml_tensor * ggml_rope_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_dims, + int mode, + int n_ctx); + """ + ... + def ggml_rope_custom(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int, freq_base: float, freq_scale: float) -> ffi.CData: + """ + custom RoPE + + GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_dims, + int mode, + int n_ctx, + float freq_base, + float freq_scale); + """ + ... + def ggml_rope_custom_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int, freq_base: float, freq_scale: float) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_dims, + int mode, + int n_ctx, + float freq_base, + float freq_scale); + """ + ... + def ggml_rope_inplace(ctx: ffi.CData, a: ffi.CData, n_past: int, n_dims: int, mode: int, n_ctx: int) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past, + int n_dims, + int mode, + int n_ctx); + """ + ... + def ggml_scale(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_scale_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_set(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData: + """ + b -> view(a,offset,nb1,nb2,3), return modified a + + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + """ + ... + def ggml_set_1d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); + """ + ... + def ggml_set_1d_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); + """ + ... + def ggml_set_2d(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, offset: int) -> ffi.CData: + """ + b -> view(a,offset,nb1,nb2,3), return modified a + + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); + """ + ... + def ggml_set_2d_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, offset: int) -> ffi.CData: + """ + b -> view(a,offset,nb1,nb2,3), return view(a) + + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); + """ + ... + def ggml_set_f32(tensor: ffi.CData, value: float) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value);""" + ... + def ggml_set_f32_1d(tensor: ffi.CData, i: int, value: float) -> None: + """ GGML_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value);""" + ... + def ggml_set_i32(tensor: ffi.CData, value: int) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value);""" + ... + def ggml_set_i32_1d(tensor: ffi.CData, i: int, value: int) -> None: + """ GGML_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value);""" + ... + def ggml_set_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData: + """ + b -> view(a,offset,nb1,nb2,3), return view(a) + + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + """ + ... + def ggml_set_name(tensor: ffi.CData, name: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name);""" + ... + def ggml_set_no_alloc(ctx: ffi.CData, no_alloc: bool) -> None: + """ GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc);""" + ... + def ggml_set_param(ctx: ffi.CData, tensor: ffi.CData) -> None: + """ + GGML_API void ggml_set_param( + struct ggml_context * ctx, + struct ggml_tensor * tensor); + """ + ... + def ggml_set_scratch(ctx: ffi.CData, scratch: ffi.CData) -> int: + """ GGML_API size_t ggml_set_scratch (struct ggml_context * ctx, struct ggml_scratch scratch);""" + ... + def ggml_set_zero(tensor: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);""" + ... + def ggml_sgn(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sgn_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_silu(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_silu_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + a - x + b - dy + + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_silu_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_soft_max(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_soft_max_back(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_soft_max_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_soft_max_back_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_soft_max_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_soft_max_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + in-place, returns view(a) + + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sqr(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sqr_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sqrt(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sqrt_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_step(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_step_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sub(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_sub_inplace(ctx: ffi.CData, a: ffi.CData, b: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + """ + ... + def ggml_sum(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + return scalar + + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_sum_rows(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_tanh(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_tanh_inplace(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_tensor_overhead() -> int: + """ + use this to compute the memory overhead of a tensor + + GGML_API size_t ggml_tensor_overhead(void); + """ + ... + def ggml_time_init() -> None: + """ GGML_API void ggml_time_init(void); // call this once at the beginning of the program""" + ... + def ggml_time_ms() -> int: + """ GGML_API int64_t ggml_time_ms(void);""" + ... + def ggml_time_us() -> int: + """ GGML_API int64_t ggml_time_us(void);""" + ... + def ggml_transpose(ctx: ffi.CData, a: ffi.CData) -> ffi.CData: + """ + alias for ggml_permute(ctx, a, 1, 0, 2, 3) + + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + """ + ... + def ggml_type_name(type: int) -> ffi.CData: + """ GGML_API const char * ggml_type_name(enum ggml_type type);""" + ... + def ggml_type_size(type: int) -> int: + """ GGML_API size_t ggml_type_size (enum ggml_type type); // size in bytes for all elements in a block""" + ... + def ggml_type_sizef(type: int) -> float: + """ GGML_API float ggml_type_sizef(enum ggml_type type); // ggml_type_size()/ggml_blck_size() as float""" + ... + def ggml_unary(ctx: ffi.CData, a: ffi.CData, op: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + """ + ... + def ggml_unary_inplace(ctx: ffi.CData, a: ffi.CData, op: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + """ + ... + def ggml_used_mem(ctx: ffi.CData) -> int: + """ GGML_API size_t ggml_used_mem(const struct ggml_context * ctx);""" + ... + def ggml_vec_dot_q2_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None: + """ + Dot product + + void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy); + """ + ... + def ggml_vec_dot_q3_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None: + """void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);""" + ... + def ggml_vec_dot_q4_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None: + """void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);""" + ... + def ggml_vec_dot_q5_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None: + """void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);""" + ... + def ggml_vec_dot_q6_K_q8_K(n: int, s: ffi.CData, vx: ffi.CData, vy: ffi.CData) -> None: + """void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, const void * restrict vx, const void * restrict vy);""" + ... + def ggml_view_1d(ctx: ffi.CData, a: ffi.CData, ne0: int, offset: int) -> ffi.CData: + """ + offset in bytes + + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + """ + ... + def ggml_view_2d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, nb1: int, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + """ + ... + def ggml_view_3d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, nb1: int, nb2: int, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + """ + ... + def ggml_view_4d(ctx: ffi.CData, a: ffi.CData, ne0: int, ne1: int, ne2: int, ne3: int, nb1: int, nb2: int, nb3: int, offset: int) -> ffi.CData: + """ + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + """ + ... + def ggml_view_tensor(ctx: ffi.CData, src: ffi.CData) -> ffi.CData: + """ GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, const struct ggml_tensor * src);""" + ... + def ggml_win_part(ctx: ffi.CData, a: ffi.CData, w: int) -> ffi.CData: + """ + partition into non-overlapping windows with padding if needed + example: + a: 768 64 64 1 + w: 14 + res: 768 14 14 25 + used in sam + + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + """ + ... + def ggml_win_unpart(ctx: ffi.CData, a: ffi.CData, w0: int, h0: int, w: int) -> ffi.CData: + """ + reverse of ggml_win_part + used in sam + + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + """ + ... + def gguf_add_tensor(ctx: ffi.CData, tensor: ffi.CData) -> None: + """ + manage tensor info + + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + """ + ... + def gguf_find_key(ctx: ffi.CData, key: ffi.CData) -> int: + """ GGML_API int gguf_find_key(struct gguf_context * ctx, const char * key);""" + ... + def gguf_find_tensor(ctx: ffi.CData, name: ffi.CData) -> int: + """ GGML_API int gguf_find_tensor (struct gguf_context * ctx, const char * name);""" + ... + def gguf_free(ctx: ffi.CData) -> None: + """ GGML_API void gguf_free(struct gguf_context * ctx);""" + ... + def gguf_get_alignment(ctx: ffi.CData) -> int: + """ GGML_API size_t gguf_get_alignment (struct gguf_context * ctx);""" + ... + def gguf_get_arr_data(ctx: ffi.CData, i: int) -> ffi.CData: + """ GGML_API const void * gguf_get_arr_data(struct gguf_context * ctx, int i);""" + ... + def gguf_get_arr_n(ctx: ffi.CData, i: int) -> int: + """ GGML_API int gguf_get_arr_n (struct gguf_context * ctx, int i);""" + ... + def gguf_get_arr_str(ctx: ffi.CData, key_id: int, i: int) -> ffi.CData: + """ GGML_API const char * gguf_get_arr_str (struct gguf_context * ctx, int key_id, int i);""" + ... + def gguf_get_arr_type(ctx: ffi.CData, i: int) -> int: + """ GGML_API enum gguf_type gguf_get_arr_type(struct gguf_context * ctx, int i);""" + ... + def gguf_get_data(ctx: ffi.CData) -> ffi.CData: + """ GGML_API void * gguf_get_data (struct gguf_context * ctx);""" + ... + def gguf_get_data_offset(ctx: ffi.CData) -> int: + """ GGML_API size_t gguf_get_data_offset(struct gguf_context * ctx);""" + ... + def gguf_get_key(ctx: ffi.CData, i: int) -> ffi.CData: + """ GGML_API const char * gguf_get_key (struct gguf_context * ctx, int i);""" + ... + def gguf_get_kv_type(ctx: ffi.CData, i: int) -> int: + """ GGML_API enum gguf_type gguf_get_kv_type (struct gguf_context * ctx, int i);""" + ... + def gguf_get_meta_data(ctx: ffi.CData, data: ffi.CData) -> None: + """ GGML_API void gguf_get_meta_data(struct gguf_context * ctx, void * data);""" + ... + def gguf_get_meta_size(ctx: ffi.CData) -> int: + """ + get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + + GGML_API size_t gguf_get_meta_size(struct gguf_context * ctx); + """ + ... + def gguf_get_n_kv(ctx: ffi.CData) -> int: + """ GGML_API int gguf_get_n_kv(struct gguf_context * ctx);""" + ... + def gguf_get_n_tensors(ctx: ffi.CData) -> int: + """ GGML_API int gguf_get_n_tensors (struct gguf_context * ctx);""" + ... + def gguf_get_tensor_name(ctx: ffi.CData, i: int) -> ffi.CData: + """ GGML_API char * gguf_get_tensor_name (struct gguf_context * ctx, int i);""" + ... + def gguf_get_tensor_offset(ctx: ffi.CData, i: int) -> int: + """ GGML_API size_t gguf_get_tensor_offset(struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_bool(ctx: ffi.CData, i: int) -> bool: + """ GGML_API bool gguf_get_val_bool(struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_f32(ctx: ffi.CData, i: int) -> float: + """ GGML_API float gguf_get_val_f32 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_i16(ctx: ffi.CData, i: int) -> int: + """ GGML_API int16_t gguf_get_val_i16 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_i32(ctx: ffi.CData, i: int) -> int: + """ GGML_API int32_t gguf_get_val_i32 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_i8(ctx: ffi.CData, i: int) -> int: + """ GGML_API int8_t gguf_get_val_i8 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_str(ctx: ffi.CData, i: int) -> ffi.CData: + """ GGML_API const char * gguf_get_val_str (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_u16(ctx: ffi.CData, i: int) -> int: + """ GGML_API uint16_t gguf_get_val_u16 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_u32(ctx: ffi.CData, i: int) -> int: + """ GGML_API uint32_t gguf_get_val_u32 (struct gguf_context * ctx, int i);""" + ... + def gguf_get_val_u8(ctx: ffi.CData, i: int) -> int: + """ + results are undefined if the wrong type is used for the key + + GGML_API uint8_t gguf_get_val_u8 (struct gguf_context * ctx, int i); + """ + ... + def gguf_get_version(ctx: ffi.CData) -> int: + """ GGML_API int gguf_get_version (struct gguf_context * ctx);""" + ... + def gguf_init_empty() -> ffi.CData: + """ GGML_API struct gguf_context * gguf_init_empty(void);""" + ... + def gguf_init_from_file(fname: ffi.CData, params: ffi.CData) -> ffi.CData: + """ GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params);""" + ... + def gguf_set_arr_data(ctx: ffi.CData, key: ffi.CData, type: int, data: ffi.CData, n: int) -> None: + """ GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, int n);""" + ... + def gguf_set_arr_str(ctx: ffi.CData, key: ffi.CData, data: ffi.CData, n: int) -> None: + """ GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, int n);""" + ... + def gguf_set_kv(ctx: ffi.CData, src: ffi.CData) -> None: + """ + set or add KV pairs from another context + + GGML_API void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src); + """ + ... + def gguf_set_tensor_data(ctx: ffi.CData, name: ffi.CData, data: ffi.CData, size: int) -> None: + """ GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data, size_t size);""" + ... + def gguf_set_tensor_type(ctx: ffi.CData, name: ffi.CData, type: int) -> None: + """ GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type);""" + ... + def gguf_set_val_bool(ctx: ffi.CData, key: ffi.CData, val: bool) -> None: + """ GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val);""" + ... + def gguf_set_val_f32(ctx: ffi.CData, key: ffi.CData, val: float) -> None: + """ GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val);""" + ... + def gguf_set_val_i16(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val);""" + ... + def gguf_set_val_i32(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val);""" + ... + def gguf_set_val_i8(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val);""" + ... + def gguf_set_val_str(ctx: ffi.CData, key: ffi.CData, val: ffi.CData) -> None: + """ GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val);""" + ... + def gguf_set_val_u16(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val);""" + ... + def gguf_set_val_u32(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val);""" + ... + def gguf_set_val_u8(ctx: ffi.CData, key: ffi.CData, val: int) -> None: + """ + overrides existing values or adds a new one + + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + """ + ... + def gguf_type_name(type: int) -> ffi.CData: + """ GGML_API const char * gguf_type_name(enum gguf_type type);""" + ... + def gguf_write_to_file(ctx: ffi.CData, fname: ffi.CData, only_meta: bool) -> None: + """ + write the entire context to a binary file + + GGML_API void gguf_write_to_file(struct gguf_context * ctx, const char * fname, bool only_meta); + """ + ... + def quantize_row_q2_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q2_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q2_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """ + Quantization + + void quantize_row_q2_K_reference(const float * restrict x, block_q2_K * restrict y, int k); + """ + ... + def quantize_row_q3_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q3_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q3_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q3_K_reference(const float * restrict x, block_q3_K * restrict y, int k);""" + ... + def quantize_row_q4_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q4_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q4_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q4_K_reference(const float * restrict x, block_q4_K * restrict y, int k);""" + ... + def quantize_row_q5_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q5_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q5_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q5_K_reference(const float * restrict x, block_q5_K * restrict y, int k);""" + ... + def quantize_row_q6_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q6_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q6_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q6_K_reference(const float * restrict x, block_q6_K * restrict y, int k);""" + ... + def quantize_row_q8_K(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q8_K(const float * restrict x, void * restrict y, int k);""" + ... + def quantize_row_q8_K_reference(x: ffi.CData, y: ffi.CData, k: int) -> None: + """void quantize_row_q8_K_reference(const float * restrict x, block_q8_K * restrict y, int k);""" + ... diff --git a/ggml/examples/python/ggml/cffi.py b/ggml/examples/python/ggml/cffi.py new file mode 100644 index 0000000..ecc1bed --- /dev/null +++ b/ggml/examples/python/ggml/cffi.py @@ -0,0 +1,11 @@ +# auto-generated file +import _cffi_backend + +ffi = _cffi_backend.FFI('ggml.cffi', + _version = 0x2601, + _types = b'\x00\x00\xB6\x0D\x00\x00\x09\x0B\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x2F\x03\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x31\x03\x00\x04\x3D\x03\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x32\x03\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x34\x03\x00\x03\xFE\x03\x00\x04\x53\x03\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x3D\x03\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x04\x3E\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\xB6\x0D\x00\x00\x00\x0F\x00\x02\xD0\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x04\x0B\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x0B\x0B\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x0F\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x16\x0D\x00\x00\x0B\x11\x00\x04\x38\x03\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x16\x0D\x00\x00\x0B\x11\x00\x00\x44\x11\x00\x00\x08\x11\x00\x04\x30\x03\x00\x00\x4B\x11\x00\x00\x00\x0F\x00\x04\x16\x0D\x00\x00\x0B\x11\x00\x00\x20\x09\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x0B\x00\x00\x00\x0F\x00\x01\x14\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x34\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x02\x7E\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\xF4\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\xF4\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\xF4\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\xF4\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x04\x18\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x02\xE9\x0D\x00\x00\x0E\x11\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x4B\x11\x00\x04\x33\x03\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x0E\x11\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x04\x35\x03\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x21\x11\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x22\x0D\x00\x00\x00\x0F\x00\x00\xDB\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\xDB\x0D\x00\x00\x00\x0F\x00\x03\xB0\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x03\xB5\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x10\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x4B\x0D\x00\x00\x0B\x11\x00\x00\x00\x0F\x00\x00\x4B\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x30\x0D\x00\x00\x0F\x11\x00\x00\x0B\x03\x00\x00\xB0\x11\x00\x00\x00\x0F\x00\x04\x30\x0D\x00\x00\x0B\x11\x00\x00\x4B\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x04\x30\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x0B\x0D\x00\x00\x1B\x09\x00\x00\x00\x0F\x00\x04\x33\x0D\x00\x00\x4B\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x0E\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x7F\x0D\x00\x00\x00\x0F\x00\x00\x50\x0D\x00\x00\x07\x0B\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x4B\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\xDB\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x01\x11\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x01\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x01\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x01\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x05\x0B\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x01\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0A\x0B\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0B\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x0B\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x03\x5C\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x03\x62\x03\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x02\xD8\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x03\x4F\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x08\x11\x00\x03\x54\x03\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x02\xD3\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x03\x44\x03\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x03\x48\x03\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x0B\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x08\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x08\x11\x00\x00\x0F\x11\x00\x00\x01\x0F\x00\x00\x08\x0D\x00\x00\x08\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x08\x0D\x00\x00\x08\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x21\x0D\x00\x00\x0F\x11\x00\x00\x24\x09\x00\x00\x00\x0F\x00\x00\x21\x0D\x00\x00\x00\x0F\x00\x03\xBA\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x03\xBF\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x01\x11\x00\x00\xF4\x03\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xDB\x03\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x02\x35\x11\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x02\x39\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x04\x11\x00\x00\x4B\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x0B\x11\x00\x00\x21\x09\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x04\x32\x03\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x21\x11\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x00\x0F\x00\x00\x6C\x0D\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x6C\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x10\x0D\x00\x02\x4B\x11\x00\x00\x00\x0F\x00\x00\x10\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x00\x10\x0D\x00\x00\x21\x11\x00\x00\x00\x0F\x00\x00\x10\x0D\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x02\xE1\x0D\x00\x00\x21\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xF8\x03\x00\x00\xF4\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xF9\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xFA\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xFB\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xFC\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x03\xFD\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0F\x11\x00\x00\x0F\x11\x00\x00\x07\x01\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xF8\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xF9\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xFA\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xFB\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xFC\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x03\xFD\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x00\x6C\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x35\x11\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x07\x01\x00\x03\xFE\x03\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x07\x01\x00\x02\x7E\x11\x00\x02\x35\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x07\x01\x00\x02\x7E\x11\x00\x02\x35\x11\x00\x02\x35\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x07\x01\x00\x02\x7E\x11\x00\x04\x53\x03\x00\x02\xE1\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x04\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x04\x11\x00\x00\x22\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x04\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x4B\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x4B\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x04\x30\x03\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\xF8\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\xF8\x11\x00\x02\xF8\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0B\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0B\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0B\x11\x00\x00\x4B\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0B\x11\x00\x00\x44\x11\x00\x00\x50\x11\x00\x00\x0B\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0B\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\x4B\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0E\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0E\x11\x00\x00\x4B\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0E\x11\x00\x00\x4B\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0E\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x7F\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x7F\x11\x00\x02\xE9\x11\x00\x02\xE9\x11\x00\x02\xE9\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x7F\x11\x00\x00\x4B\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x04\x37\x03\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x08\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x08\x11\x00\x00\x10\x11\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x01\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x0F\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x0F\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x34\x11\x00\x02\xE1\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x05\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x03\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x04\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x08\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x0F\x11\x00\x02\xE1\x11\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x15\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x21\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x21\x11\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x0A\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x6C\x03\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x10\x11\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x02\xE1\x11\x00\x02\x7E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x04\x53\x0D\x00\x00\x00\x0F\x00\x00\x24\x03\x00\x00\x0D\x09\x00\x00\x0E\x09\x00\x00\x0F\x09\x00\x00\x10\x09\x00\x00\x11\x09\x00\x00\x12\x09\x00\x00\x13\x09\x00\x00\x14\x09\x00\x00\x04\x09\x00\x00\x05\x09\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x08\x09\x00\x00\x09\x09\x00\x00\x0A\x09\x00\x00\x02\x01\x00\x03\xFE\x05\x00\x00\x00\x80\x00\x03\xFE\x05\x00\x00\x00\x10\x00\x03\xFE\x05\x00\x00\x00\xC0\x00\x03\xFE\x05\x00\x00\x00\x25\x00\x03\xFE\x05\x00\x00\x00\x28\x00\x03\xFE\x05\x00\x00\x00\x04\x00\x03\xFE\x05\x00\x00\x00\x38\x00\x03\xFE\x05\x00\x00\x00\x40\x00\x03\xFE\x05\x00\x00\x1F\xF0\x00\x03\xFE\x05\x00\x00\x00\x08\x00\x00\x00\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x06\x0B\x00\x00\x08\x0B\x00\x00\x0B\x09\x00\x00\x22\x05\x00\x00\x10\x00\x00\x00\x22\x05\x00\x00\x00\x08\x00\x00\x0F\x01\x00\x00\xDB\x05\x00\x00\x00\x04\x00\x00\x09\x01\x00\x03\xB0\x05\x00\x00\x00\x10\x00\x03\xB5\x05\x00\x00\x00\x10\x00\x03\xB5\x05\x00\x00\x01\x00\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\x03\x09\x00\x04\x2C\x03\x00\x00\x0C\x09\x00\x04\x2E\x03\x00\x00\x15\x09\x00\x00\x16\x09\x00\x00\x17\x09\x00\x00\x18\x09\x00\x00\x19\x09\x00\x00\x1A\x09\x00\x00\x1C\x09\x00\x00\x1D\x09\x00\x04\x37\x03\x00\x00\x1E\x09\x00\x00\x1F\x09\x00\x00\x08\x05\x00\x00\x10\x00\x00\x00\x08\x05\x00\x00\x00\x06\x00\x00\x22\x09\x00\x00\x23\x09\x00\x03\xBA\x03\x00\x03\xBA\x05\x00\x00\x00\x80\x00\x03\xBA\x05\x00\x00\x00\x0C\x00\x03\xBA\x05\x00\x00\x00\x10\x00\x03\xBA\x05\x00\x00\x00\x20\x00\x03\xBA\x05\x00\x00\x00\x40\x00\x00\x0C\x01\x00\x00\x11\x05\x00\x00\x00\x04\x00\x00\x10\x05\x00\x00\x20\x51\x00\x02\xC6\x03\x00\x02\xDE\x03\x00\x03\xE0\x03\x00\x03\xE7\x03\x00\x00\x00\x01', + _globals = (b'\xFF\xFF\xFF\x0BGGML_BACKEND_CPU',0,b'\xFF\xFF\xFF\x0BGGML_BACKEND_GPU',10,b'\xFF\xFF\xFF\x0BGGML_BACKEND_GPU_SPLIT',20,b'\xFF\xFF\xFF\x0BGGML_FTYPE_ALL_F32',0,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_F16',1,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q2_K',10,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q3_K',11,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q4_0',2,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q4_1',3,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q4_1_SOME_F16',4,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q4_K',12,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q5_0',8,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q5_1',9,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q5_K',13,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q6_K',14,b'\xFF\xFF\xFF\x0BGGML_FTYPE_MOSTLY_Q8_0',7,b'\xFF\xFF\xFF\x0BGGML_FTYPE_UNKNOWN',-1,b'\xFF\xFF\xFF\x1FGGML_GRAPH_SIZE',164520,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_BACKTRACKING_ARMIJO',0,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE',2,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_BACKTRACKING_WOLFE',1,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_DEFAULT',1,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_FAIL',-128,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_INVALID_PARAMETERS',-124,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_MAXIMUM_ITERATIONS',-125,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_MAXIMUM_STEP',-126,b'\xFF\xFF\xFF\x0BGGML_LINESEARCH_MINIMUM_STEP',-127,b'\xFF\xFF\xFF\x0BGGML_OBJECT_GRAPH',1,b'\xFF\xFF\xFF\x1FGGML_OBJECT_SIZE',32,b'\xFF\xFF\xFF\x0BGGML_OBJECT_TENSOR',0,b'\xFF\xFF\xFF\x0BGGML_OBJECT_WORK_BUFFER',2,b'\xFF\xFF\xFF\x0BGGML_OPT_ADAM',0,b'\xFF\xFF\xFF\x0BGGML_OPT_DID_NOT_CONVERGE',1,b'\xFF\xFF\xFF\x0BGGML_OPT_FAIL',4,b'\xFF\xFF\xFF\x0BGGML_OPT_INVALID_WOLFE',3,b'\xFF\xFF\xFF\x0BGGML_OPT_LBFGS',1,b'\xFF\xFF\xFF\x0BGGML_OPT_NO_CONTEXT',2,b'\xFF\xFF\xFF\x0BGGML_OPT_OK',0,b'\xFF\xFF\xFF\x0BGGML_OP_ACC',4,b'\xFF\xFF\xFF\x0BGGML_OP_ADD',2,b'\xFF\xFF\xFF\x0BGGML_OP_ADD1',3,b'\xFF\xFF\xFF\x0BGGML_OP_ALIBI',40,b'\xFF\xFF\xFF\x0BGGML_OP_ARGMAX',14,b'\xFF\xFF\xFF\x0BGGML_OP_CLAMP',41,b'\xFF\xFF\xFF\x0BGGML_OP_CONT',26,b'\xFF\xFF\xFF\x0BGGML_OP_CONV_1D',42,b'\xFF\xFF\xFF\x0BGGML_OP_CONV_2D',43,b'\xFF\xFF\xFF\x0BGGML_OP_COUNT',62,b'\xFF\xFF\xFF\x0BGGML_OP_CPY',25,b'\xFF\xFF\xFF\x0BGGML_OP_CROSS_ENTROPY_LOSS',60,b'\xFF\xFF\xFF\x0BGGML_OP_CROSS_ENTROPY_LOSS_BACK',61,b'\xFF\xFF\xFF\x0BGGML_OP_DIAG',33,b'\xFF\xFF\xFF\x0BGGML_OP_DIAG_MASK_INF',34,b'\xFF\xFF\xFF\x0BGGML_OP_DIAG_MASK_ZERO',35,b'\xFF\xFF\xFF\x0BGGML_OP_DIV',7,b'\xFF\xFF\xFF\x0BGGML_OP_DUP',1,b'\xFF\xFF\xFF\x0BGGML_OP_FLASH_ATTN',46,b'\xFF\xFF\xFF\x0BGGML_OP_FLASH_ATTN_BACK',48,b'\xFF\xFF\xFF\x0BGGML_OP_FLASH_FF',47,b'\xFF\xFF\xFF\x0BGGML_OP_GET_ROWS',31,b'\xFF\xFF\xFF\x0BGGML_OP_GET_ROWS_BACK',32,b'\xFF\xFF\xFF\x0BGGML_OP_LOG',10,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_BINARY',53,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM1',57,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM1_F32',54,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM2',58,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM2_F32',55,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM3',59,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_CUSTOM3_F32',56,b'\xFF\xFF\xFF\x0BGGML_OP_MAP_UNARY',52,b'\xFF\xFF\xFF\x0BGGML_OP_MEAN',13,b'\xFF\xFF\xFF\x0BGGML_OP_MUL',6,b'\xFF\xFF\xFF\x0BGGML_OP_MUL_MAT',21,b'\xFF\xFF\xFF\x0BGGML_OP_NONE',0,b'\xFF\xFF\xFF\x0BGGML_OP_NORM',18,b'\xFF\xFF\xFF\x0BGGML_OP_OUT_PROD',22,b'\xFF\xFF\xFF\x0BGGML_OP_PERMUTE',29,b'\xFF\xFF\xFF\x0BGGML_OP_POOL_1D',44,b'\xFF\xFF\xFF\x0BGGML_OP_POOL_2D',45,b'\xFF\xFF\xFF\x0BGGML_OP_POOL_AVG',1,b'\xFF\xFF\xFF\x0BGGML_OP_POOL_COUNT',2,b'\xFF\xFF\xFF\x0BGGML_OP_POOL_MAX',0,b'\xFF\xFF\xFF\x0BGGML_OP_REPEAT',15,b'\xFF\xFF\xFF\x0BGGML_OP_REPEAT_BACK',16,b'\xFF\xFF\xFF\x0BGGML_OP_RESHAPE',27,b'\xFF\xFF\xFF\x0BGGML_OP_RMS_NORM',19,b'\xFF\xFF\xFF\x0BGGML_OP_RMS_NORM_BACK',20,b'\xFF\xFF\xFF\x0BGGML_OP_ROPE',38,b'\xFF\xFF\xFF\x0BGGML_OP_ROPE_BACK',39,b'\xFF\xFF\xFF\x0BGGML_OP_SCALE',23,b'\xFF\xFF\xFF\x0BGGML_OP_SET',24,b'\xFF\xFF\xFF\x0BGGML_OP_SILU_BACK',17,b'\xFF\xFF\xFF\x0BGGML_OP_SOFT_MAX',36,b'\xFF\xFF\xFF\x0BGGML_OP_SOFT_MAX_BACK',37,b'\xFF\xFF\xFF\x0BGGML_OP_SQR',8,b'\xFF\xFF\xFF\x0BGGML_OP_SQRT',9,b'\xFF\xFF\xFF\x0BGGML_OP_SUB',5,b'\xFF\xFF\xFF\x0BGGML_OP_SUM',11,b'\xFF\xFF\xFF\x0BGGML_OP_SUM_ROWS',12,b'\xFF\xFF\xFF\x0BGGML_OP_TRANSPOSE',30,b'\xFF\xFF\xFF\x0BGGML_OP_UNARY',51,b'\xFF\xFF\xFF\x0BGGML_OP_VIEW',28,b'\xFF\xFF\xFF\x0BGGML_OP_WIN_PART',49,b'\xFF\xFF\xFF\x0BGGML_OP_WIN_UNPART',50,b'\xFF\xFF\xFF\x0BGGML_TASK_COMPUTE',1,b'\xFF\xFF\xFF\x0BGGML_TASK_FINALIZE',2,b'\xFF\xFF\xFF\x0BGGML_TASK_INIT',0,b'\xFF\xFF\xFF\x1FGGML_TENSOR_SIZE',288,b'\xFF\xFF\xFF\x0BGGML_TYPE_COUNT',19,b'\xFF\xFF\xFF\x0BGGML_TYPE_F16',1,b'\xFF\xFF\xFF\x0BGGML_TYPE_F32',0,b'\xFF\xFF\xFF\x0BGGML_TYPE_I16',17,b'\xFF\xFF\xFF\x0BGGML_TYPE_I32',18,b'\xFF\xFF\xFF\x0BGGML_TYPE_I8',16,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q2_K',10,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q3_K',11,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q4_0',2,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q4_1',3,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q4_K',12,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q5_0',6,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q5_1',7,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q5_K',13,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q6_K',14,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q8_0',8,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q8_1',9,b'\xFF\xFF\xFF\x0BGGML_TYPE_Q8_K',15,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_ABS',0,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_ELU',5,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_GELU',7,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_GELU_QUICK',8,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_NEG',2,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_RELU',6,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_SGN',1,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_SILU',9,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_STEP',3,b'\xFF\xFF\xFF\x0BGGML_UNARY_OP_TANH',4,b'\xFF\xFF\xFF\x0BGGUF_TYPE_ARRAY',9,b'\xFF\xFF\xFF\x0BGGUF_TYPE_BOOL',7,b'\xFF\xFF\xFF\x0BGGUF_TYPE_COUNT',10,b'\xFF\xFF\xFF\x0BGGUF_TYPE_FLOAT32',6,b'\xFF\xFF\xFF\x0BGGUF_TYPE_INT16',3,b'\xFF\xFF\xFF\x0BGGUF_TYPE_INT32',5,b'\xFF\xFF\xFF\x0BGGUF_TYPE_INT8',1,b'\xFF\xFF\xFF\x0BGGUF_TYPE_STRING',8,b'\xFF\xFF\xFF\x0BGGUF_TYPE_UINT16',2,b'\xFF\xFF\xFF\x0BGGUF_TYPE_UINT32',4,b'\xFF\xFF\xFF\x0BGGUF_TYPE_UINT8',0,b'\x00\x02\x9A\x23__assert_rtn',0,b'\x00\x02\x7C\x23dequantize_row_q2_K',0,b'\x00\x02\x81\x23dequantize_row_q3_K',0,b'\x00\x02\x86\x23dequantize_row_q4_K',0,b'\x00\x02\x8B\x23dequantize_row_q5_K',0,b'\x00\x02\x90\x23dequantize_row_q6_K',0,b'\x00\x02\x95\x23dequantize_row_q8_K',0,b'\x00\x00\xFA\x23ggml_abs',0,b'\x00\x00\xFA\x23ggml_abs_inplace',0,b'\x00\x01\xDD\x23ggml_acc',0,b'\x00\x01\xDD\x23ggml_acc_inplace',0,b'\x00\x01\x84\x23ggml_add',0,b'\x00\x01\x84\x23ggml_add1',0,b'\x00\x01\x84\x23ggml_add1_inplace',0,b'\x00\x01\x84\x23ggml_add_inplace',0,b'\x00\x01\x26\x23ggml_alibi',0,b'\x00\x02\xEC\x23ggml_allocr_alloc',0,b'\x00\x02\x42\x23ggml_allocr_alloc_graph',0,b'\x00\x02\xE4\x23ggml_allocr_free',0,b'\x00\x00\x03\x23ggml_allocr_is_measure',0,b'\x00\x00\xA2\x23ggml_allocr_new',0,b'\x00\x00\x9F\x23ggml_allocr_new_measure',0,b'\x00\x02\xE4\x23ggml_allocr_reset',0,b'\x00\x02\xE7\x23ggml_allocr_set_parse_seq',0,b'\x00\x00\x17\x23ggml_are_same_shape',0,b'\x00\x00\xFA\x23ggml_argmax',0,b'\x00\x00\x74\x23ggml_blck_size',0,b'\x00\x00\xB3\x23ggml_build_backward',0,b'\x00\x00\xB8\x23ggml_build_forward',0,b'\x00\x00\xAA\x23ggml_build_forward_ctx',0,b'\x00\x02\xF3\x23ggml_build_forward_expand',0,b'\x00\x00\x1B\x23ggml_cl_can_mul_mat',0,b'\x00\x03\x6B\x23ggml_cl_free_data',0,b'\x00\x03\xE0\x23ggml_cl_host_free',0,b'\x00\x02\x72\x23ggml_cl_host_malloc',0,b'\x00\x03\xEC\x23ggml_cl_init',0,b'\x00\x03\x78\x23ggml_cl_mul',0,b'\x00\x03\x7D\x23ggml_cl_mul_mat',0,b'\x00\x02\x54\x23ggml_cl_mul_mat_get_wsize',0,b'\x00\x03\xE3\x23ggml_cl_transform_tensor',0,b'\x00\x01\x1B\x23ggml_clamp',0,b'\x00\x00\xFA\x23ggml_cont',0,b'\x00\x01\x90\x23ggml_conv_1d',0,b'\x00\x01\x89\x23ggml_conv_1d_ph',0,b'\x00\x01\x98\x23ggml_conv_2d',0,b'\x00\x00\x90\x23ggml_cpu_has_arm_fma',0,b'\x00\x00\x90\x23ggml_cpu_has_avx',0,b'\x00\x00\x90\x23ggml_cpu_has_avx2',0,b'\x00\x00\x90\x23ggml_cpu_has_avx512',0,b'\x00\x00\x90\x23ggml_cpu_has_avx512_vbmi',0,b'\x00\x00\x90\x23ggml_cpu_has_avx512_vnni',0,b'\x00\x00\x90\x23ggml_cpu_has_blas',0,b'\x00\x00\x90\x23ggml_cpu_has_clblast',0,b'\x00\x00\x90\x23ggml_cpu_has_cublas',0,b'\x00\x00\x90\x23ggml_cpu_has_f16c',0,b'\x00\x00\x90\x23ggml_cpu_has_fma',0,b'\x00\x00\x90\x23ggml_cpu_has_fp16_va',0,b'\x00\x00\x90\x23ggml_cpu_has_gpublas',0,b'\x00\x00\x90\x23ggml_cpu_has_neon',0,b'\x00\x00\x90\x23ggml_cpu_has_sse3',0,b'\x00\x00\x90\x23ggml_cpu_has_vsx',0,b'\x00\x00\x90\x23ggml_cpu_has_wasm_simd',0,b'\x00\x01\x84\x23ggml_cpy',0,b'\x00\x01\x84\x23ggml_cross_entropy_loss',0,b'\x00\x01\xA3\x23ggml_cross_entropy_loss_back',0,b'\x00\x03\x41\x23ggml_cuda_assign_buffers',0,b'\x00\x03\x41\x23ggml_cuda_assign_buffers_force_inplace',0,b'\x00\x03\x41\x23ggml_cuda_assign_buffers_no_scratch',0,b'\x00\x00\x1B\x23ggml_cuda_can_mul_mat',0,b'\x00\x00\x06\x23ggml_cuda_compute_forward',0,b'\x00\x03\x41\x23ggml_cuda_free_data',0,b'\x00\x03\xEC\x23ggml_cuda_free_scratch',0,b'\x00\x00\x90\x23ggml_cuda_get_device_count',0,b'\x00\x02\xCE\x23ggml_cuda_get_device_description',0,b'\x00\x03\xE0\x23ggml_cuda_host_free',0,b'\x00\x02\x72\x23ggml_cuda_host_malloc',0,b'\x00\x02\xCB\x23ggml_cuda_set_main_device',0,b'\x00\x02\x79\x23ggml_cuda_set_mul_mat_q',0,b'\x00\x03\xD8\x23ggml_cuda_set_scratch_size',0,b'\x00\x02\xA0\x23ggml_cuda_set_tensor_split',0,b'\x00\x03\xE3\x23ggml_cuda_transform_tensor',0,b'\x00\x00\x95\x23ggml_cycles',0,b'\x00\x00\x95\x23ggml_cycles_per_ms',0,b'\x00\x00\xFA\x23ggml_diag',0,b'\x00\x01\x21\x23ggml_diag_mask_inf',0,b'\x00\x01\x21\x23ggml_diag_mask_inf_inplace',0,b'\x00\x01\x21\x23ggml_diag_mask_zero',0,b'\x00\x01\x21\x23ggml_diag_mask_zero_inplace',0,b'\x00\x01\x84\x23ggml_div',0,b'\x00\x01\x84\x23ggml_div_inplace',0,b'\x00\x00\xFA\x23ggml_dup',0,b'\x00\x00\xFA\x23ggml_dup_inplace',0,b'\x00\x02\x0B\x23ggml_dup_tensor',0,b'\x00\x02\x4D\x23ggml_element_size',0,b'\x00\x00\xFA\x23ggml_elu',0,b'\x00\x00\xFA\x23ggml_elu_inplace',0,b'\x00\x01\xA9\x23ggml_flash_attn',0,b'\x00\x01\xB0\x23ggml_flash_attn_back',0,b'\x00\x01\xB8\x23ggml_flash_ff',0,b'\x00\x02\x16\x23ggml_format_name',0,b'\x00\x00\x6B\x23ggml_fp16_to_fp32',0,b'\x00\x03\xDB\x23ggml_fp16_to_fp32_row',0,b'\x00\x02\x62\x23ggml_fp32_to_fp16',0,b'\x00\x02\xC1\x23ggml_fp32_to_fp16_row',0,b'\x00\x03\x03\x23ggml_free',0,b'\x00\x00\x53\x23ggml_ftype_to_ggml_type',0,b'\x00\x00\xFA\x23ggml_gelu',0,b'\x00\x00\xFA\x23ggml_gelu_inplace',0,b'\x00\x00\xFA\x23ggml_gelu_quick',0,b'\x00\x00\xFA\x23ggml_gelu_quick_inplace',0,b'\x00\x02\x6C\x23ggml_get_data',0,b'\x00\x00\x5D\x23ggml_get_data_f32',0,b'\x00\x00\x63\x23ggml_get_f32_1d',0,b'\x00\x00\x81\x23ggml_get_i32_1d',0,b'\x00\x02\x4A\x23ggml_get_max_tensor_size',0,b'\x00\x02\x69\x23ggml_get_mem_buffer',0,b'\x00\x02\x4A\x23ggml_get_mem_size',0,b'\x00\x00\x36\x23ggml_get_name',0,b'\x00\x00\x0A\x23ggml_get_no_alloc',0,b'\x00\x01\x84\x23ggml_get_rows',0,b'\x00\x01\xA3\x23ggml_get_rows_back',0,b'\x00\x00\xCE\x23ggml_get_tensor',0,b'\x00\x00\x56\x23ggml_get_unary_op',0,b'\x00\x00\x77\x23ggml_graph_compute',0,b'\x00\x03\x0A\x23ggml_graph_compute_with_ctx',0,b'\x00\x02\xFE\x23ggml_graph_dump_dot',0,b'\x00\x02\xFA\x23ggml_graph_export',0,b'\x00\x00\xCA\x23ggml_graph_get_tensor',0,b'\x00\x00\xAE\x23ggml_graph_import',0,b'\x00\x02\x60\x23ggml_graph_overhead',0,b'\x00\x00\xBE\x23ggml_graph_plan',0,b'\x00\x02\xF7\x23ggml_graph_print',0,b'\x00\x02\xF0\x23ggml_graph_reset',0,b'\x00\x00\xBB\x23ggml_init',0,b'\x00\x03\xEC\x23ggml_init_cublas',0,b'\x00\x00\x6E\x23ggml_internal_get_type_traits',0,b'\x00\x00\x14\x23ggml_is_contiguous',0,b'\x00\x00\x27\x23ggml_is_numa',0,b'\x00\x00\x14\x23ggml_is_permuted',0,b'\x00\x00\x00\x23ggml_is_quantized',0,b'\x00\x00\x14\x23ggml_is_transposed',0,b'\x00\x00\xFA\x23ggml_log',0,b'\x00\x00\xFA\x23ggml_log_inplace',0,b'\x00\x01\xE6\x23ggml_map_binary_f32',0,b'\x00\x01\xE6\x23ggml_map_binary_inplace_f32',0,b'\x00\x02\x04\x23ggml_map_custom1',0,b'\x00\x01\xFF\x23ggml_map_custom1_f32',0,b'\x00\x02\x04\x23ggml_map_custom1_inplace',0,b'\x00\x01\xFF\x23ggml_map_custom1_inplace_f32',0,b'\x00\x01\xF2\x23ggml_map_custom2',0,b'\x00\x01\xEC\x23ggml_map_custom2_f32',0,b'\x00\x01\xF2\x23ggml_map_custom2_inplace',0,b'\x00\x01\xEC\x23ggml_map_custom2_inplace_f32',0,b'\x00\x01\xC7\x23ggml_map_custom3',0,b'\x00\x01\xC0\x23ggml_map_custom3_f32',0,b'\x00\x01\xC7\x23ggml_map_custom3_inplace',0,b'\x00\x01\xC0\x23ggml_map_custom3_inplace_f32',0,b'\x00\x01\xFA\x23ggml_map_unary_f32',0,b'\x00\x01\xFA\x23ggml_map_unary_inplace_f32',0,b'\x00\x00\xFA\x23ggml_mean',0,b'\x00\x00\x0D\x23ggml_metal_add_buffer',0,b'\x00\x03\x1C\x23ggml_metal_free',0,b'\x00\x00\x71\x23ggml_metal_get_concur_list',0,b'\x00\x03\x2C\x23ggml_metal_get_tensor',0,b'\x00\x03\x23\x23ggml_metal_graph_compute',0,b'\x00\x03\x27\x23ggml_metal_graph_find_concurrency',0,b'\x00\x03\xE0\x23ggml_metal_host_free',0,b'\x00\x02\x72\x23ggml_metal_host_malloc',0,b'\x00\x00\x7B\x23ggml_metal_if_optimized',0,b'\x00\x00\xC2\x23ggml_metal_init',0,b'\x00\x03\x1F\x23ggml_metal_set_n_cb',0,b'\x00\x03\x2C\x23ggml_metal_set_tensor',0,b'\x00\x03\xEC\x23ggml_mpi_backend_free',0,b'\x00\x03\xEC\x23ggml_mpi_backend_init',0,b'\x00\x03\x33\x23ggml_mpi_eval_init',0,b'\x00\x03\x30\x23ggml_mpi_free',0,b'\x00\x03\x39\x23ggml_mpi_graph_compute_post',0,b'\x00\x03\x39\x23ggml_mpi_graph_compute_pre',0,b'\x00\x00\xC5\x23ggml_mpi_init',0,b'\x00\x00\x7E\x23ggml_mpi_rank',0,b'\x00\x01\x84\x23ggml_mul',0,b'\x00\x01\x84\x23ggml_mul_inplace',0,b'\x00\x01\x84\x23ggml_mul_mat',0,b'\x00\x02\x4D\x23ggml_nbytes',0,b'\x00\x02\x4D\x23ggml_nbytes_pad',0,b'\x00\x02\x50\x23ggml_nbytes_split',0,b'\x00\x00\xFA\x23ggml_neg',0,b'\x00\x00\xFA\x23ggml_neg_inplace',0,b'\x00\x00\x92\x23ggml_nelements',0,b'\x00\x00\xF2\x23ggml_new_f32',0,b'\x00\x00\xA7\x23ggml_new_graph',0,b'\x00\x00\xF6\x23ggml_new_i32',0,b'\x00\x00\xD2\x23ggml_new_tensor',0,b'\x00\x00\xD8\x23ggml_new_tensor_1d',0,b'\x00\x00\xDD\x23ggml_new_tensor_2d',0,b'\x00\x00\xE3\x23ggml_new_tensor_3d',0,b'\x00\x00\xEA\x23ggml_new_tensor_4d',0,b'\x00\x00\xFA\x23ggml_norm',0,b'\x00\x00\xFA\x23ggml_norm_inplace',0,b'\x00\x00\x92\x23ggml_nrows',0,b'\x00\x03\xEC\x23ggml_numa_init',0,b'\x00\x00\x2D\x23ggml_op_name',0,b'\x00\x00\x2D\x23ggml_op_symbol',0,b'\x00\x00\x4E\x23ggml_opt',0,b'\x00\x00\xC7\x23ggml_opt_default_params',0,b'\x00\x03\x0F\x23ggml_opt_init',0,b'\x00\x00\x42\x23ggml_opt_resume',0,b'\x00\x00\x47\x23ggml_opt_resume_g',0,b'\x00\x01\x84\x23ggml_out_prod',0,b'\x00\x01\x34\x23ggml_permute',0,b'\x00\x00\xFE\x23ggml_pool_1d',0,b'\x00\x01\x06\x23ggml_pool_2d',0,b'\x00\x03\x3E\x23ggml_print_object',0,b'\x00\x03\x19\x23ggml_print_objects',0,b'\x00\x02\x33\x23ggml_quantize_chunk',0,b'\x00\x02\x3B\x23ggml_quantize_q2_K',0,b'\x00\x02\x3B\x23ggml_quantize_q3_K',0,b'\x00\x02\x3B\x23ggml_quantize_q4_0',0,b'\x00\x02\x3B\x23ggml_quantize_q4_1',0,b'\x00\x02\x3B\x23ggml_quantize_q4_K',0,b'\x00\x02\x3B\x23ggml_quantize_q5_0',0,b'\x00\x02\x3B\x23ggml_quantize_q5_1',0,b'\x00\x02\x3B\x23ggml_quantize_q5_K',0,b'\x00\x02\x3B\x23ggml_quantize_q6_K',0,b'\x00\x02\x3B\x23ggml_quantize_q8_0',0,b'\x00\x00\xFA\x23ggml_relu',0,b'\x00\x00\xFA\x23ggml_relu_inplace',0,b'\x00\x01\x84\x23ggml_repeat',0,b'\x00\x01\x84\x23ggml_repeat_back',0,b'\x00\x01\x84\x23ggml_reshape',0,b'\x00\x01\x46\x23ggml_reshape_1d',0,b'\x00\x01\x4B\x23ggml_reshape_2d',0,b'\x00\x01\x51\x23ggml_reshape_3d',0,b'\x00\x01\x58\x23ggml_reshape_4d',0,b'\x00\x01\x16\x23ggml_rms_norm',0,b'\x00\x01\x84\x23ggml_rms_norm_back',0,b'\x00\x01\x16\x23ggml_rms_norm_inplace',0,b'\x00\x01\x34\x23ggml_rope',0,b'\x00\x01\x34\x23ggml_rope_back',0,b'\x00\x01\x3C\x23ggml_rope_custom',0,b'\x00\x01\x3C\x23ggml_rope_custom_inplace',0,b'\x00\x01\x34\x23ggml_rope_inplace',0,b'\x00\x01\x84\x23ggml_scale',0,b'\x00\x01\x84\x23ggml_scale_inplace',0,b'\x00\x01\xDD\x23ggml_set',0,b'\x00\x01\xD0\x23ggml_set_1d',0,b'\x00\x01\xD0\x23ggml_set_1d_inplace',0,b'\x00\x01\xD6\x23ggml_set_2d',0,b'\x00\x01\xD6\x23ggml_set_2d_inplace',0,b'\x00\x02\x1A\x23ggml_set_f32',0,b'\x00\x03\x6E\x23ggml_set_f32_1d',0,b'\x00\x02\x1E\x23ggml_set_i32',0,b'\x00\x03\x73\x23ggml_set_i32_1d',0,b'\x00\x01\xDD\x23ggml_set_inplace',0,b'\x00\x02\x12\x23ggml_set_name',0,b'\x00\x03\x06\x23ggml_set_no_alloc',0,b'\x00\x03\x15\x23ggml_set_param',0,b'\x00\x02\x46\x23ggml_set_scratch',0,b'\x00\x02\x0F\x23ggml_set_zero',0,b'\x00\x00\xFA\x23ggml_sgn',0,b'\x00\x00\xFA\x23ggml_sgn_inplace',0,b'\x00\x00\xFA\x23ggml_silu',0,b'\x00\x01\x84\x23ggml_silu_back',0,b'\x00\x00\xFA\x23ggml_silu_inplace',0,b'\x00\x00\xFA\x23ggml_soft_max',0,b'\x00\x01\x84\x23ggml_soft_max_back',0,b'\x00\x01\x84\x23ggml_soft_max_back_inplace',0,b'\x00\x00\xFA\x23ggml_soft_max_inplace',0,b'\x00\x00\xFA\x23ggml_sqr',0,b'\x00\x00\xFA\x23ggml_sqr_inplace',0,b'\x00\x00\xFA\x23ggml_sqrt',0,b'\x00\x00\xFA\x23ggml_sqrt_inplace',0,b'\x00\x00\xFA\x23ggml_step',0,b'\x00\x00\xFA\x23ggml_step_inplace',0,b'\x00\x01\x84\x23ggml_sub',0,b'\x00\x01\x84\x23ggml_sub_inplace',0,b'\x00\x00\xFA\x23ggml_sum',0,b'\x00\x00\xFA\x23ggml_sum_rows',0,b'\x00\x00\xFA\x23ggml_tanh',0,b'\x00\x00\xFA\x23ggml_tanh_inplace',0,b'\x00\x02\x60\x23ggml_tensor_overhead',0,b'\x00\x03\xEC\x23ggml_time_init',0,b'\x00\x00\x95\x23ggml_time_ms',0,b'\x00\x00\x95\x23ggml_time_us',0,b'\x00\x00\xFA\x23ggml_transpose',0,b'\x00\x00\x30\x23ggml_type_name',0,b'\x00\x02\x30\x23ggml_type_size',0,b'\x00\x00\x60\x23ggml_type_sizef',0,b'\x00\x01\x11\x23ggml_unary',0,b'\x00\x01\x11\x23ggml_unary_inplace',0,b'\x00\x02\x4A\x23ggml_used_mem',0,b'\x00\x02\xDE\x23ggml_vec_dot_q2_K_q8_K',0,b'\x00\x02\xDE\x23ggml_vec_dot_q3_K_q8_K',0,b'\x00\x02\xDE\x23ggml_vec_dot_q4_K_q8_K',0,b'\x00\x02\xDE\x23ggml_vec_dot_q5_K_q8_K',0,b'\x00\x02\xDE\x23ggml_vec_dot_q6_K_q8_K',0,b'\x00\x01\x7E\x23ggml_view_1d',0,b'\x00\x01\x76\x23ggml_view_2d',0,b'\x00\x01\x6C\x23ggml_view_3d',0,b'\x00\x01\x60\x23ggml_view_4d',0,b'\x00\x02\x0B\x23ggml_view_tensor',0,b'\x00\x01\x21\x23ggml_win_part',0,b'\x00\x01\x2D\x23ggml_win_unpart',0,b'\x00\x03\xCC\x23gguf_add_tensor',0,b'\x00\x00\x88\x23gguf_find_key',0,b'\x00\x00\x88\x23gguf_find_tensor',0,b'\x00\x03\x84\x23gguf_free',0,b'\x00\x02\x59\x23gguf_get_alignment',0,b'\x00\x02\x75\x23gguf_get_arr_data',0,b'\x00\x00\x8C\x23gguf_get_arr_n',0,b'\x00\x00\x3D\x23gguf_get_arr_str',0,b'\x00\x00\x59\x23gguf_get_arr_type',0,b'\x00\x02\x6F\x23gguf_get_data',0,b'\x00\x02\x59\x23gguf_get_data_offset',0,b'\x00\x00\x39\x23gguf_get_key',0,b'\x00\x00\x59\x23gguf_get_kv_type',0,b'\x00\x03\xD4\x23gguf_get_meta_data',0,b'\x00\x02\x59\x23gguf_get_meta_size',0,b'\x00\x00\x85\x23gguf_get_n_kv',0,b'\x00\x00\x85\x23gguf_get_n_tensors',0,b'\x00\x00\x29\x23gguf_get_tensor_name',0,b'\x00\x02\x5C\x23gguf_get_tensor_offset',0,b'\x00\x00\x20\x23gguf_get_val_bool',0,b'\x00\x00\x67\x23gguf_get_val_f32',0,b'\x00\x00\x97\x23gguf_get_val_i16',0,b'\x00\x00\x8C\x23gguf_get_val_i32',0,b'\x00\x00\x9B\x23gguf_get_val_i8',0,b'\x00\x00\x39\x23gguf_get_val_str',0,b'\x00\x02\x65\x23gguf_get_val_u16',0,b'\x00\x02\x2C\x23gguf_get_val_u32',0,b'\x00\x02\x28\x23gguf_get_val_u8',0,b'\x00\x00\x85\x23gguf_get_version',0,b'\x00\x02\x26\x23gguf_init_empty',0,b'\x00\x02\x22\x23gguf_init_from_file',0,b'\x00\x03\x9C\x23gguf_set_arr_data',0,b'\x00\x03\x8C\x23gguf_set_arr_str',0,b'\x00\x03\xD0\x23gguf_set_kv',0,b'\x00\x03\xC6\x23gguf_set_tensor_data',0,b'\x00\x03\x97\x23gguf_set_tensor_type',0,b'\x00\x03\x87\x23gguf_set_val_bool',0,b'\x00\x03\xA3\x23gguf_set_val_f32',0,b'\x00\x03\xAD\x23gguf_set_val_i16',0,b'\x00\x03\xA8\x23gguf_set_val_i32',0,b'\x00\x03\xB2\x23gguf_set_val_i8',0,b'\x00\x03\x92\x23gguf_set_val_str',0,b'\x00\x03\xC1\x23gguf_set_val_u16',0,b'\x00\x03\xBC\x23gguf_set_val_u32',0,b'\x00\x03\xB7\x23gguf_set_val_u8',0,b'\x00\x00\x33\x23gguf_type_name',0,b'\x00\x03\x87\x23gguf_write_to_file',0,b'\x00\x02\xC6\x23quantize_row_q2_K',0,b'\x00\x02\xA3\x23quantize_row_q2_K_reference',0,b'\x00\x02\xC6\x23quantize_row_q3_K',0,b'\x00\x02\xA8\x23quantize_row_q3_K_reference',0,b'\x00\x02\xC6\x23quantize_row_q4_K',0,b'\x00\x02\xAD\x23quantize_row_q4_K_reference',0,b'\x00\x02\xC6\x23quantize_row_q5_K',0,b'\x00\x02\xB2\x23quantize_row_q5_K_reference',0,b'\x00\x02\xC6\x23quantize_row_q6_K',0,b'\x00\x02\xB7\x23quantize_row_q6_K_reference',0,b'\x00\x02\xC6\x23quantize_row_q8_K',0,b'\x00\x02\xBC\x23quantize_row_q8_K_reference',0), + _struct_unions = ((b'\x00\x00\x04\x27\x00\x00\x00\x02$1',b'\x00\x00\x22\x11n_iter',b'\x00\x00\xF4\x11sched',b'\x00\x00\xF4\x11decay',b'\x00\x00\xF4\x11alpha',b'\x00\x00\xF4\x11beta1',b'\x00\x00\xF4\x11beta2',b'\x00\x00\xF4\x11eps',b'\x00\x00\xF4\x11eps_f',b'\x00\x00\xF4\x11eps_g'),(b'\x00\x00\x04\x28\x00\x00\x00\x02$2',b'\x00\x00\x22\x11m',b'\x00\x00\x22\x11n_iter',b'\x00\x00\x22\x11max_linesearch',b'\x00\x00\xF4\x11eps',b'\x00\x00\xF4\x11ftol',b'\x00\x00\xF4\x11wolfe',b'\x00\x00\xF4\x11min_step',b'\x00\x00\xF4\x11max_step',b'\x00\x04\x14\x11linesearch'),(b'\x00\x00\x04\x29\x00\x00\x00\x02$3',b'\x00\x00\x08\x11x',b'\x00\x00\x08\x11g1',b'\x00\x00\x08\x11g2',b'\x00\x00\x08\x11m',b'\x00\x00\x08\x11v',b'\x00\x00\x08\x11mh',b'\x00\x00\x08\x11vh',b'\x00\x00\x08\x11pf',b'\x00\x00\xF4\x11fx_best',b'\x00\x00\xF4\x11fx_prev',b'\x00\x00\x22\x11n_no_improvement'),(b'\x00\x00\x04\x2A\x00\x00\x00\x02$4',b'\x00\x00\x08\x11x',b'\x00\x00\x08\x11xp',b'\x00\x00\x08\x11g',b'\x00\x00\x08\x11gp',b'\x00\x00\x08\x11d',b'\x00\x00\x08\x11pf',b'\x00\x00\x08\x11lmal',b'\x00\x00\x08\x11lmys',b'\x00\x00\x08\x11lms',b'\x00\x00\x08\x11lmy',b'\x00\x00\xF4\x11fx_best',b'\x00\x00\xF4\x11step',b'\x00\x00\x22\x11j',b'\x00\x00\x22\x11k',b'\x00\x00\x22\x11end',b'\x00\x00\x22\x11n_no_improvement'),(b'\x00\x00\x03\xF7\x00\x00\x00\x03$__mbstate_t',b'\x00\x03\xFF\x11__mbstate8',b'\x00\x00\xDB\x11_mbstateL'),(b'\x00\x00\x03\xF8\x00\x00\x00\x02$block_q2_K',b'\x00\x04\x44\x11scales',b'\x00\x04\x48\x11qs',b'\x00\x00\x6C\x11d',b'\x00\x00\x6C\x11dmin'),(b'\x00\x00\x03\xF9\x00\x00\x00\x02$block_q3_K',b'\x00\x04\x46\x11hmask',b'\x00\x04\x48\x11qs',b'\x00\x04\x42\x11scales',b'\x00\x00\x6C\x11d'),(b'\x00\x00\x03\xFA\x00\x00\x00\x02$block_q4_K',b'\x00\x00\x6C\x11d',b'\x00\x00\x6C\x11dmin',b'\x00\x04\x42\x11scales',b'\x00\x04\x40\x11qs'),(b'\x00\x00\x03\xFB\x00\x00\x00\x02$block_q5_K',b'\x00\x00\x6C\x11d',b'\x00\x00\x6C\x11dmin',b'\x00\x04\x42\x11scales',b'\x00\x04\x46\x11qh',b'\x00\x04\x40\x11qs'),(b'\x00\x00\x03\xFC\x00\x00\x00\x02$block_q6_K',b'\x00\x04\x40\x11ql',b'\x00\x04\x48\x11qh',b'\x00\x04\x23\x11scales',b'\x00\x00\x6C\x11d'),(b'\x00\x00\x03\xFD\x00\x00\x00\x02$block_q8_K',b'\x00\x00\xF4\x11d',b'\x00\x04\x25\x11qs',b'\x00\x04\x21\x11bsums'),(b'\x00\x00\x04\x18\x00\x00\x00\x02$ggml_type_traits_t',b'\x00\x00\x0F\x11type_name',b'\x00\x00\x22\x11blck_size',b'\x00\x00\x11\x11type_size',b'\x00\x00\xB6\x11is_quantized',b'\x00\x04\x52\x11to_float',b'\x00\x04\x4F\x11from_float',b'\x00\x04\x4F\x11from_float_reference',b'\x00\x04\x50\x11vec_dot',b'\x00\x00\x01\x11vec_dot_type'),(b'\x00\x00\x04\x2C\x00\x00\x00\x02__darwin_pthread_handler_rec',b'\x00\x04\x51\x11__routine',b'\x00\x00\x10\x11__arg',b'\x00\x04\x2B\x11__next'),(b'\x00\x00\x03\xEF\x00\x00\x00\x02_opaque_pthread_attr_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x0B\x11__opaque'),(b'\x00\x00\x03\xF0\x00\x00\x00\x02_opaque_pthread_cond_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x07\x11__opaque'),(b'\x00\x00\x03\xF1\x00\x00\x00\x02_opaque_pthread_condattr_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x11\x11__opaque'),(b'\x00\x00\x03\xF2\x00\x00\x00\x02_opaque_pthread_mutex_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x0B\x11__opaque'),(b'\x00\x00\x03\xF3\x00\x00\x00\x02_opaque_pthread_mutexattr_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x11\x11__opaque'),(b'\x00\x00\x03\xF4\x00\x00\x00\x02_opaque_pthread_once_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x11\x11__opaque'),(b'\x00\x00\x03\xF5\x00\x00\x00\x02_opaque_pthread_rwlock_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x03\x11__opaque'),(b'\x00\x00\x03\xF6\x00\x00\x00\x02_opaque_pthread_rwlockattr_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x01\x11__opaque'),(b'\x00\x00\x04\x2E\x00\x00\x00\x02_opaque_pthread_t',b'\x00\x04\x20\x11__sig',b'\x00\x04\x2B\x11__cleanup_stack',b'\x00\x04\x0F\x11__opaque'),(b'\x00\x00\x04\x2F\x00\x00\x00\x10ggml_allocr',),(b'\x00\x00\x04\x30\x00\x00\x00\x02ggml_cgraph',b'\x00\x00\x22\x11n_nodes',b'\x00\x00\x22\x11n_leafs',b'\x00\x04\x39\x11nodes',b'\x00\x04\x39\x11grads',b'\x00\x04\x39\x11leafs',b'\x00\x04\x4D\x11visited_hash_table',b'\x00\x00\x22\x11perf_runs',b'\x00\x00\xDB\x11perf_cycles',b'\x00\x00\xDB\x11perf_time_us'),(b'\x00\x00\x04\x31\x00\x00\x00\x02ggml_compute_params',b'\x00\x04\x17\x11type',b'\x00\x00\x22\x11ith',b'\x00\x00\x22\x11nth',b'\x00\x00\x11\x11wsize',b'\x00\x00\x10\x11wdata'),(b'\x00\x00\x04\x32\x00\x00\x00\x10ggml_context',),(b'\x00\x00\x04\x33\x00\x00\x00\x02ggml_cplan',b'\x00\x00\x11\x11work_size',b'\x00\x04\x3F\x11work_data',b'\x00\x00\x22\x11n_threads',b'\x00\x04\x19\x11n_tasks',b'\x00\x03\xEE\x11abort_callback',b'\x00\x00\x10\x11abort_callback_data'),(b'\x00\x00\x00\xBC\x00\x00\x00\x02ggml_init_params',b'\x00\x00\x11\x11mem_size',b'\x00\x00\x10\x11mem_buffer',b'\x00\x00\xB6\x11no_alloc'),(b'\x00\x00\x04\x34\x00\x00\x00\x10ggml_metal_context',),(b'\x00\x00\x04\x35\x00\x00\x00\x10ggml_mpi_context',),(b'\x00\x00\x04\x37\x00\x00\x00\x02ggml_object',b'\x00\x00\x11\x11offs',b'\x00\x00\x11\x11size',b'\x00\x04\x36\x11next',b'\x00\x04\x15\x11type',b'\x00\x04\x09\x11padding'),(b'\x00\x00\x04\x38\x00\x00\x00\x02ggml_opt_context',b'\x00\x00\x0B\x11ctx',b'\x00\x00\x50\x11params',b'\x00\x00\x22\x11iter',b'\x00\x00\xDB\x11nx',b'\x00\x00\xB6\x11just_initialized',b'\x00\x04\x29\x11adam',b'\x00\x04\x2A\x11lbfgs'),(b'\x00\x00\x00\x50\x00\x00\x00\x02ggml_opt_params',b'\x00\x00\xC8\x11type',b'\x00\x00\x22\x11n_threads',b'\x00\x00\x22\x11past',b'\x00\x00\xF4\x11delta',b'\x00\x00\x22\x11max_no_improvement',b'\x00\x00\xB6\x11print_forward_graph',b'\x00\x00\xB6\x11print_backward_graph',b'\x00\x04\x27\x11adam',b'\x00\x04\x28\x11lbfgs'),(b'\x00\x00\x02\x48\x00\x00\x00\x02ggml_scratch',b'\x00\x00\x11\x11offs',b'\x00\x00\x11\x11size',b'\x00\x00\x10\x11data'),(b'\x00\x00\x04\x3D\x00\x00\x00\x02ggml_tensor',b'\x00\x00\x01\x11type',b'\x00\x04\x13\x11backend',b'\x00\x00\x22\x11n_dims',b'\x00\x04\x1E\x11ne',b'\x00\x04\x4B\x11nb',b'\x00\x00\x2E\x11op',b'\x00\x04\x1B\x11op_params',b'\x00\x00\xB6\x11is_param',b'\x00\x00\x08\x11grad',b'\x00\x04\x3B\x11src',b'\x00\x00\x22\x11perf_runs',b'\x00\x00\xDB\x11perf_cycles',b'\x00\x00\xDB\x11perf_time_us',b'\x00\x00\x10\x11data',b'\x00\x04\x0D\x11name',b'\x00\x00\x10\x11extra',b'\x00\x04\x09\x11padding'),(b'\x00\x00\x04\x3E\x00\x00\x00\x10gguf_context',),(b'\x00\x00\x02\x24\x00\x00\x00\x02gguf_init_params',b'\x00\x00\xB6\x11no_alloc',b'\x00\x00\xB0\x11ctx')), + _enums = (b'\x00\x00\x04\x13\x00\x00\x00\x16ggml_backend\x00GGML_BACKEND_CPU,GGML_BACKEND_GPU,GGML_BACKEND_GPU_SPLIT',b'\x00\x00\x00\x54\x00\x00\x00\x15ggml_ftype\x00GGML_FTYPE_UNKNOWN,GGML_FTYPE_ALL_F32,GGML_FTYPE_MOSTLY_F16,GGML_FTYPE_MOSTLY_Q4_0,GGML_FTYPE_MOSTLY_Q4_1,GGML_FTYPE_MOSTLY_Q4_1_SOME_F16,GGML_FTYPE_MOSTLY_Q8_0,GGML_FTYPE_MOSTLY_Q5_0,GGML_FTYPE_MOSTLY_Q5_1,GGML_FTYPE_MOSTLY_Q2_K,GGML_FTYPE_MOSTLY_Q3_K,GGML_FTYPE_MOSTLY_Q4_K,GGML_FTYPE_MOSTLY_Q5_K,GGML_FTYPE_MOSTLY_Q6_K',b'\x00\x00\x04\x14\x00\x00\x00\x16ggml_linesearch\x00GGML_LINESEARCH_DEFAULT,GGML_LINESEARCH_BACKTRACKING_ARMIJO,GGML_LINESEARCH_BACKTRACKING_WOLFE,GGML_LINESEARCH_BACKTRACKING_STRONG_WOLFE',b'\x00\x00\x04\x15\x00\x00\x00\x16ggml_object_type\x00GGML_OBJECT_TENSOR,GGML_OBJECT_GRAPH,GGML_OBJECT_WORK_BUFFER',b'\x00\x00\x00\x2E\x00\x00\x00\x16ggml_op\x00GGML_OP_NONE,GGML_OP_DUP,GGML_OP_ADD,GGML_OP_ADD1,GGML_OP_ACC,GGML_OP_SUB,GGML_OP_MUL,GGML_OP_DIV,GGML_OP_SQR,GGML_OP_SQRT,GGML_OP_LOG,GGML_OP_SUM,GGML_OP_SUM_ROWS,GGML_OP_MEAN,GGML_OP_ARGMAX,GGML_OP_REPEAT,GGML_OP_REPEAT_BACK,GGML_OP_SILU_BACK,GGML_OP_NORM,GGML_OP_RMS_NORM,GGML_OP_RMS_NORM_BACK,GGML_OP_MUL_MAT,GGML_OP_OUT_PROD,GGML_OP_SCALE,GGML_OP_SET,GGML_OP_CPY,GGML_OP_CONT,GGML_OP_RESHAPE,GGML_OP_VIEW,GGML_OP_PERMUTE,GGML_OP_TRANSPOSE,GGML_OP_GET_ROWS,GGML_OP_GET_ROWS_BACK,GGML_OP_DIAG,GGML_OP_DIAG_MASK_INF,GGML_OP_DIAG_MASK_ZERO,GGML_OP_SOFT_MAX,GGML_OP_SOFT_MAX_BACK,GGML_OP_ROPE,GGML_OP_ROPE_BACK,GGML_OP_ALIBI,GGML_OP_CLAMP,GGML_OP_CONV_1D,GGML_OP_CONV_2D,GGML_OP_POOL_1D,GGML_OP_POOL_2D,GGML_OP_FLASH_ATTN,GGML_OP_FLASH_FF,GGML_OP_FLASH_ATTN_BACK,GGML_OP_WIN_PART,GGML_OP_WIN_UNPART,GGML_OP_UNARY,GGML_OP_MAP_UNARY,GGML_OP_MAP_BINARY,GGML_OP_MAP_CUSTOM1_F32,GGML_OP_MAP_CUSTOM2_F32,GGML_OP_MAP_CUSTOM3_F32,GGML_OP_MAP_CUSTOM1,GGML_OP_MAP_CUSTOM2,GGML_OP_MAP_CUSTOM3,GGML_OP_CROSS_ENTROPY_LOSS,GGML_OP_CROSS_ENTROPY_LOSS_BACK,GGML_OP_COUNT',b'\x00\x00\x01\x01\x00\x00\x00\x16ggml_op_pool\x00GGML_OP_POOL_MAX,GGML_OP_POOL_AVG,GGML_OP_POOL_COUNT',b'\x00\x00\x04\x16\x00\x00\x00\x15ggml_opt_result\x00GGML_OPT_OK,GGML_OPT_DID_NOT_CONVERGE,GGML_OPT_NO_CONTEXT,GGML_OPT_INVALID_WOLFE,GGML_OPT_FAIL,GGML_LINESEARCH_FAIL,GGML_LINESEARCH_MINIMUM_STEP,GGML_LINESEARCH_MAXIMUM_STEP,GGML_LINESEARCH_MAXIMUM_ITERATIONS,GGML_LINESEARCH_INVALID_PARAMETERS',b'\x00\x00\x00\xC8\x00\x00\x00\x16ggml_opt_type\x00GGML_OPT_ADAM,GGML_OPT_LBFGS',b'\x00\x00\x04\x17\x00\x00\x00\x16ggml_task_type\x00GGML_TASK_INIT,GGML_TASK_COMPUTE,GGML_TASK_FINALIZE',b'\x00\x00\x00\x01\x00\x00\x00\x16ggml_type\x00GGML_TYPE_F32,GGML_TYPE_F16,GGML_TYPE_Q4_0,GGML_TYPE_Q4_1,GGML_TYPE_Q5_0,GGML_TYPE_Q5_1,GGML_TYPE_Q8_0,GGML_TYPE_Q8_1,GGML_TYPE_Q2_K,GGML_TYPE_Q3_K,GGML_TYPE_Q4_K,GGML_TYPE_Q5_K,GGML_TYPE_Q6_K,GGML_TYPE_Q8_K,GGML_TYPE_I8,GGML_TYPE_I16,GGML_TYPE_I32,GGML_TYPE_COUNT',b'\x00\x00\x01\x14\x00\x00\x00\x16ggml_unary_op\x00GGML_UNARY_OP_ABS,GGML_UNARY_OP_SGN,GGML_UNARY_OP_NEG,GGML_UNARY_OP_STEP,GGML_UNARY_OP_TANH,GGML_UNARY_OP_ELU,GGML_UNARY_OP_RELU,GGML_UNARY_OP_GELU,GGML_UNARY_OP_GELU_QUICK,GGML_UNARY_OP_SILU',b'\x00\x00\x00\x34\x00\x00\x00\x16gguf_type\x00GGUF_TYPE_UINT8,GGUF_TYPE_INT8,GGUF_TYPE_UINT16,GGUF_TYPE_INT16,GGUF_TYPE_UINT32,GGUF_TYPE_INT32,GGUF_TYPE_FLOAT32,GGUF_TYPE_BOOL,GGUF_TYPE_STRING,GGUF_TYPE_ARRAY,GGUF_TYPE_COUNT'), + _typenames = (b'\x00\x00\x00\xDB__darwin_blkcnt_t',b'\x00\x00\x00\x22__darwin_blksize_t',b'\x00\x00\x00\x11__darwin_clock_t',b'\x00\x00\x00\x22__darwin_ct_rune_t',b'\x00\x00\x00\x22__darwin_dev_t',b'\x00\x00\x03\xBF__darwin_fsblkcnt_t',b'\x00\x00\x03\xBF__darwin_fsfilcnt_t',b'\x00\x00\x03\xBF__darwin_gid_t',b'\x00\x00\x03\xBF__darwin_id_t',b'\x00\x00\x04\x4A__darwin_ino64_t',b'\x00\x00\x04\x4A__darwin_ino_t',b'\x00\x00\x04\x20__darwin_intptr_t',b'\x00\x00\x03\xBF__darwin_mach_port_name_t',b'\x00\x00\x03\xBF__darwin_mach_port_t',b'\x00\x00\x03\xF7__darwin_mbstate_t',b'\x00\x00\x00\x6C__darwin_mode_t',b'\x00\x00\x03\xBF__darwin_natural_t',b'\x00\x00\x00\xDB__darwin_off_t',b'\x00\x00\x00\x22__darwin_pid_t',b'\x00\x00\x03\xEF__darwin_pthread_attr_t',b'\x00\x00\x03\xF0__darwin_pthread_cond_t',b'\x00\x00\x03\xF1__darwin_pthread_condattr_t',b'\x00\x00\x00\x11__darwin_pthread_key_t',b'\x00\x00\x03\xF2__darwin_pthread_mutex_t',b'\x00\x00\x03\xF3__darwin_pthread_mutexattr_t',b'\x00\x00\x03\xF4__darwin_pthread_once_t',b'\x00\x00\x03\xF5__darwin_pthread_rwlock_t',b'\x00\x00\x03\xF6__darwin_pthread_rwlockattr_t',b'\x00\x00\x04\x2D__darwin_pthread_t',b'\x00\x00\x04\x20__darwin_ptrdiff_t',b'\x00\x00\x00\x22__darwin_rune_t',b'\x00\x00\x03\xBF__darwin_sigset_t',b'\x00\x00\x00\x11__darwin_size_t',b'\x00\x00\x03\xBF__darwin_socklen_t',b'\x00\x00\x04\x20__darwin_ssize_t',b'\x00\x00\x00\x22__darwin_suseconds_t',b'\x00\x00\x04\x20__darwin_time_t',b'\x00\x00\x03\xBF__darwin_uid_t',b'\x00\x00\x03\xBF__darwin_useconds_t',b'\x00\x00\x04\x05__darwin_uuid_string_t',b'\x00\x00\x04\x44__darwin_uuid_t',b'\x00\x00\x00\x22__darwin_wchar_t',b'\x00\x00\x00\x22__darwin_wint_t',b'\x00\x00\x03\xB0__int16_t',b'\x00\x00\x00\x22__int32_t',b'\x00\x00\x00\xDB__int64_t',b'\x00\x00\x03\xB5__int8_t',b'\x00\x00\x03\xF7__mbstate_t',b'\x00\x00\x00\x6C__uint16_t',b'\x00\x00\x03\xBF__uint32_t',b'\x00\x00\x04\x4A__uint64_t',b'\x00\x00\x03\xBA__uint8_t',b'\x00\x00\x03\xF8block_q2_K',b'\x00\x00\x03\xF9block_q3_K',b'\x00\x00\x03\xFAblock_q4_K',b'\x00\x00\x03\xFBblock_q5_K',b'\x00\x00\x03\xFCblock_q6_K',b'\x00\x00\x03\xFDblock_q8_K',b'\x00\x00\x01\xEAggml_binary_op_f32_t',b'\x00\x00\x02\x02ggml_custom1_op_f32_t',b'\x00\x00\x02\x07ggml_custom1_op_t',b'\x00\x00\x01\xF0ggml_custom2_op_f32_t',b'\x00\x00\x01\xF6ggml_custom2_op_t',b'\x00\x00\x01\xC5ggml_custom3_op_f32_t',b'\x00\x00\x01\xCCggml_custom3_op_t',b'\x00\x00\x00\x6Cggml_fp16_t',b'\x00\x00\x04\x4Fggml_from_float_t',b'\x00\x00\x04\x52ggml_to_float_t',b'\x00\x00\x04\x18ggml_type_traits_t',b'\x00\x00\x01\xFDggml_unary_op_f32_t',b'\x00\x00\x04\x50ggml_vec_dot_t',b'\x00\x00\x03\xB0int16_t',b'\x00\x00\x00\x22int32_t',b'\x00\x00\x00\xDBint64_t',b'\x00\x00\x03\xB5int8_t',b'\x00\x00\x03\xB0int_fast16_t',b'\x00\x00\x00\x22int_fast32_t',b'\x00\x00\x00\xDBint_fast64_t',b'\x00\x00\x03\xB5int_fast8_t',b'\x00\x00\x03\xB0int_least16_t',b'\x00\x00\x00\x22int_least32_t',b'\x00\x00\x00\xDBint_least64_t',b'\x00\x00\x03\xB5int_least8_t',b'\x00\x00\x04\x20intmax_t',b'\x00\x00\x04\x20intptr_t',b'\x00\x00\x04\x1Dmax_align_t',b'\x00\x00\x04\x20ptrdiff_t',b'\x00\x00\x00\xDBregister_t',b'\x00\x00\x00\x11rsize_t',b'\x00\x00\x00\x11size_t',b'\x00\x00\x04\x4Asyscall_arg_t',b'\x00\x00\x00\x6Cu_int16_t',b'\x00\x00\x03\xBFu_int32_t',b'\x00\x00\x04\x4Au_int64_t',b'\x00\x00\x03\xBAu_int8_t',b'\x00\x00\x00\x6Cuint16_t',b'\x00\x00\x03\xBFuint32_t',b'\x00\x00\x04\x4Auint64_t',b'\x00\x00\x03\xBAuint8_t',b'\x00\x00\x00\x6Cuint_fast16_t',b'\x00\x00\x03\xBFuint_fast32_t',b'\x00\x00\x04\x4Auint_fast64_t',b'\x00\x00\x03\xBAuint_fast8_t',b'\x00\x00\x00\x6Cuint_least16_t',b'\x00\x00\x03\xBFuint_least32_t',b'\x00\x00\x04\x4Auint_least64_t',b'\x00\x00\x03\xBAuint_least8_t',b'\x00\x00\x00\x11uintmax_t',b'\x00\x00\x00\x11uintptr_t',b'\x00\x00\x04\x4Auser_addr_t',b'\x00\x00\x00\xDBuser_long_t',b'\x00\x00\x00\xDBuser_off_t',b'\x00\x00\x04\x4Auser_size_t',b'\x00\x00\x00\xDBuser_ssize_t',b'\x00\x00\x00\xDBuser_time_t',b'\x00\x00\x04\x4Auser_ulong_t',b'\x00\x00\x00\x22wchar_t'), +) diff --git a/ggml/examples/python/ggml/ffi/__init__.pyi b/ggml/examples/python/ggml/ffi/__init__.pyi new file mode 100644 index 0000000..73117a1 --- /dev/null +++ b/ggml/examples/python/ggml/ffi/__init__.pyi @@ -0,0 +1,7 @@ +# Phony stubs. + +class CData: + pass + +class CType: + pass \ No newline at end of file diff --git a/ggml/examples/python/ggml/utils.py b/ggml/examples/python/ggml/utils.py new file mode 100644 index 0000000..7cea2bf --- /dev/null +++ b/ggml/examples/python/ggml/utils.py @@ -0,0 +1,182 @@ +""" + Common helpers for working with ggml + numpy +""" +from ggml import ffi, lib +from typing import Union, Optional +import numpy as np + +def init(mem_size: int, mem_buffer: ffi.CData = ffi.NULL, no_alloc: bool = False) -> ffi.CData: + """ + Initialize a ggml context, which will be freed automatically when the pointer is garbage collected. + """ + params = ffi.new('struct ggml_init_params*') + params.mem_size = mem_size + params.mem_buffer = mem_buffer + params.no_alloc = no_alloc + return ffi.gc(lib.ggml_init(params[0]), lib.ggml_free) + +TensorLike = Union[ffi.CData, np.ndarray] + +def copy(from_tensor: TensorLike, to_tensor: TensorLike, allow_requantize: bool = True): + """ + Copy the contents of one tensor to another, doing any necessary (de/re)quantization transparently. + Works across numpy & ggml tensors, but they must have the same shape (and be contiguous). + + Parameters + ---------- + from_tensor : TensorLike + The tensor to copy from (a numpy array or possibly-quantized ggml tensor) + to_tensor : TensorLike + The tensor to copy to (a numpy array or possibly-quantized ggml tensor) + allow_requantize : bool + If False, will throw an error if requantization is required (i.e. both from_tensor + and to_tensor are quantized with different quantization types) + """ + if id(from_tensor) == id(to_tensor): + return + + __expect_same_layout("source", from_tensor, "destination", to_tensor) + __check_shape_consistent_with_type(from_tensor) + __check_shape_consistent_with_type(to_tensor) + + from_type = __get_type(from_tensor) + to_type = __get_type(to_tensor) + + if from_type == to_type: + ffi.memmove(__get_data(to_tensor), __get_data(from_tensor), __get_nbytes(from_tensor)) + else: + assert allow_requantize or not lib.ggml_is_quantized(from_type) or not lib.ggml_is_quantized(to_type), \ + f"Requantizing from {__type_name(from_type)} to {__type_name(to_type)} is disabled. Force with allow_requantize=True" + + __set_floats(to_tensor, __get_floats(from_tensor)) + +def numpy(tensor: ffi.CData, allow_copy: Union[bool, np.ndarray] = False, allow_requantize=False) -> np.ndarray: + """ + Convert a ggml tensor to a numpy array. + If the tensor isn't quantized, the returned numpy array will be a view over its data. + + If it is quantized (and allow_copy is True), the copy will involve dequantization and the returned array will + be a copy of the original tensor (any changes to the numpy array won't then be reflected back to the tensor). + + Parameters + ---------- + tensor : ffi.CData + The tensor to convert to a numpy array + allow_copy : bool or np.ndarray + If False, will throw an error if the tensor is quantized (since dequantization requires extra memory). + If True, will dequantize the tensor and return a copy of the data in a new float32 numpy array. + If an np.ndarray, will copy the data into the given array (which must be the same shape as the tensor) when dequantization is needed + allow_requantize : bool + If allow_copy is a tensor with a different quantization type than the source tensor, will throw an error unless allow_requantize is True. + """ + shape = __get_shape(tensor) + + if lib.ggml_is_quantized(tensor.type): + if allow_copy == False: + raise ValueError(f"{__describe(tensor)} is quantized, conversion to numpy requires a copy (pass allow_copy=True; changes to the numpy array won't affect the original).") + elif isinstance(allow_copy, np.ndarray): + __expect_same_layout("source tensor", tensor, "dequantization output tensor", allow_copy) + destination = allow_copy + else: + destination = np.empty(shape, dtype=np.float32) + + copy(tensor, destination, allow_requantize=allow_requantize) + return destination + else: + dtype = __type_to_dtype(tensor.type) + if not dtype: + raise NotImplementedError(f'Cannot convert {__describe(tensor)} to numpy') + + assert __is_contiguous(tensor), f"Cannot convert {__describe(tensor)} to numpy (support contiguous tensors only)" + nbytes = lib.ggml_nelements(tensor) * lib.ggml_type_size(tensor.type) + array = np.frombuffer(ffi.buffer(lib.ggml_get_data(tensor), nbytes), dtype=dtype) + array.shape = shape + return array + +def __type_name(type: int) -> str: + name = lib.ggml_type_name(type) + return ffi.string(name).decode('utf-8') if name else None + +__k_quant_types = set([ + lib.GGML_TYPE_Q2_K, + lib.GGML_TYPE_Q3_K, + lib.GGML_TYPE_Q4_K, + lib.GGML_TYPE_Q5_K, + lib.GGML_TYPE_Q6_K, + lib.GGML_TYPE_Q8_K, +]) + +__type_to_dtype_dict = { + lib.GGML_TYPE_I8: np.int8, + lib.GGML_TYPE_I16: np.int16, + lib.GGML_TYPE_I32: np.int32, + lib.GGML_TYPE_F16: np.float16, + lib.GGML_TYPE_F32: np.float32, +} + +def __type_to_dtype(type: int) -> Optional[np.dtype]: return __type_to_dtype_dict.get(type) +def __dtype_to_type(dtype: np.dtype): + if dtype == np.float32: return lib.GGML_TYPE_F32 + elif dtype == np.float16: return lib.GGML_TYPE_F16 + elif dtype == np.int32: return lib.GGML_TYPE_I32 + elif dtype == np.int16: return lib.GGML_TYPE_I16 + elif dtype == np.int8: return lib.GGML_TYPE_I8 + else: raise ValueError(f"Unsupported dtype: {dtype}") + +def __describe(tensor: ffi.CType): return f'Tensor[{__type_name(__get_type(tensor))}, {__get_shape(tensor)}]' +def __get_type(tensor: TensorLike): return __dtype_to_type(tensor.dtype) if isinstance(tensor, np.ndarray) else tensor.type +def __get_shape(x: TensorLike): return x.shape if isinstance(x, np.ndarray) else tuple([x.ne[i] for i in range(x.n_dims)]) +def __get_strides(x: TensorLike): return x.strides if isinstance(x, np.ndarray) else tuple([x.nb[i] for i in range(x.n_dims)]) +def __get_data(x: TensorLike) -> ffi.CData: return ffi.from_buffer(x) if isinstance(x, np.ndarray) else lib.ggml_get_data(x) +def __get_nbytes(tensor: TensorLike): return tensor.nbytes if isinstance(tensor, np.ndarray) else lib.ggml_nbytes(tensor) +def __get_nelements(tensor: TensorLike): return tensor.size if isinstance(tensor, np.ndarray) else lib.ggml_nelements(tensor) +def __is_contiguous(tensor: TensorLike): return tensor.flags['C_CONTIGUOUS'] if isinstance(tensor, np.ndarray) else lib.ggml_is_contiguous(tensor) + +def __get_floats(tensor: TensorLike) -> ffi.CData: + data, type = __get_data(tensor), __get_type(tensor) + if type == lib.GGML_TYPE_F32: + return ffi.cast('float*', data) + else: + nelements = __get_nelements(tensor) + floats = ffi.new('float[]', nelements) + if type == lib.GGML_TYPE_F16: + lib.ggml_fp16_to_fp32_row(ffi.cast('uint16_t*', data), floats, nelements) + elif lib.ggml_is_quantized(type): + qtype = lib.ggml_internal_get_type_traits(type) + assert qtype.to_float, f"Type {__type_name(type)} is not supported by ggml" + qtype.to_float(data, floats, nelements) + else: + raise NotImplementedError(f'Cannot read floats from {__describe(tensor)}') + return floats + +def __set_floats(tensor: TensorLike, f32_data: ffi.CData) -> None: + data, type, nbytes = __get_data(tensor), __get_type(tensor), __get_nbytes(tensor) + if type == lib.GGML_TYPE_F32: + ffi.memmove(data, f32_data, nbytes) + else: + nelements = __get_nelements(tensor) + if type == lib.GGML_TYPE_F16: + lib.ggml_fp32_to_fp16_row(f32_data, ffi.cast('uint16_t*', data), nelements) + elif lib.ggml_is_quantized(type): + qtype = lib.ggml_internal_get_type_traits(type) + assert qtype.from_float, f"Type {__type_name(type)} is not supported by ggml" + qtype.from_float(f32_data, data, nelements) + else: + raise NotImplementedError(f'Cannot write floats to {__describe(tensor)}') + +def __expect_same_layout(name1: str, tensor1: TensorLike, name2: str, tensor2: TensorLike): + shape1, shape2 = __get_shape(tensor1), __get_shape(tensor2) + assert shape1 == shape2, f"Shape mismatch: {name1} has {shape1} but {name2} has {shape2}" + assert __is_contiguous(tensor1) and __is_contiguous(tensor2), f"Only contiguous tensors are supported (got {name1} with strides {__get_strides(tensor1)} and {name2} with strides {__get_strides(tensor2)})" + +def __check_shape_consistent_with_type(tensor: TensorLike): + type = __get_type(tensor) + if not lib.ggml_is_quantized(type): + return + shape = __get_shape(tensor) + + block_size = lib.ggml_blck_size(type) + assert not (block_size == 0 and type in __k_quant_types), f"Can't quantize, native library was not compiled with USE_K_QUANTS!" + assert block_size > 0, f"Invalid block size {block_size} for type {__type_name(type)}" + for i, d in enumerate(shape): + assert d % block_size == 0, f"Dimension {i} of {__describe(tensor)} is not divisible by {block_size}, required for quantization." diff --git a/ggml/examples/python/regenerate.py b/ggml/examples/python/regenerate.py new file mode 100644 index 0000000..08d84c0 --- /dev/null +++ b/ggml/examples/python/regenerate.py @@ -0,0 +1,42 @@ +# Generates bindings for the ggml library. +# +# cffi requires prior C preprocessing of the headers, and it uses pycparser which chokes on a couple of things +# so we help it a bit (e.g. replace sizeof expressions with their value, remove exotic syntax found in Darwin headers). +import os, sys, re, subprocess +import cffi +from stubs import generate_stubs + +API = os.environ.get('API', 'api.h') +CC = os.environ.get('CC') or 'gcc' +C_INCLUDE_DIR = os.environ.get('C_INCLUDE_DIR', '../../../llama.cpp') +CPPFLAGS = [ + "-I", C_INCLUDE_DIR, + '-D__fp16=uint16_t', # pycparser doesn't support __fp16 + '-D__attribute__(x)=', + '-D_Static_assert(x, m)=', +] + [x for x in os.environ.get('CPPFLAGS', '').split(' ') if x != ''] + +try: header = subprocess.run([CC, "-E", *CPPFLAGS, API], capture_output=True, text=True, check=True).stdout +except subprocess.CalledProcessError as e: print(f'{e.stderr}\n{e}', file=sys.stderr); raise + +header = '\n'.join([l for l in header.split('\n') if '__darwin_va_list' not in l]) # pycparser hates this + +# Replace constant size expressions w/ their value (compile & run a mini exe for each, because why not). +# First, extract anyting *inside* square brackets and anything that looks like a sizeof call. +for expr in set(re.findall(f'(?<=\\[)[^\\]]+(?=])|sizeof\\s*\\([^()]+\\)', header)): + if re.match(r'^(\d+|\s*)$', expr): continue # skip constants and empty bracket contents + subprocess.run([CC, "-o", "eval_size_expr", *CPPFLAGS, "-x", "c", "-"], text=True, check=True, + input=f'''#include + #include "{API}" + int main() {{ printf("%lu", (size_t)({expr})); }}''') + size = subprocess.run(["./eval_size_expr"], capture_output=True, text=True, check=True).stdout + print(f'Computed constexpr {expr} = {size}') + header = header.replace(expr, size) + +ffibuilder = cffi.FFI() +ffibuilder.cdef(header) +ffibuilder.set_source(f'ggml.cffi', None) # we're not compiling a native extension, as this quickly gets hairy +ffibuilder.compile(verbose=True) + +with open("ggml/__init__.pyi", "wt") as f: + f.write(generate_stubs(header)) \ No newline at end of file diff --git a/ggml/examples/python/stubs.py b/ggml/examples/python/stubs.py new file mode 100644 index 0000000..adf3d6c --- /dev/null +++ b/ggml/examples/python/stubs.py @@ -0,0 +1,128 @@ +""" + This generates .pyi stubs for the cffi Python bindings generated by regenerate.py +""" +import sys, re, itertools +sys.path.extend(['.', '..']) # for pycparser + +from pycparser import c_ast, parse_file, CParser +import pycparser.plyparser +from pycparser.c_ast import PtrDecl, TypeDecl, FuncDecl, EllipsisParam, IdentifierType, Struct, Enum, Typedef +from typing import Tuple + +__c_type_to_python_type = { + 'void': 'None', '_Bool': 'bool', + 'char': 'int', 'short': 'int', 'int': 'int', 'long': 'int', + 'ptrdiff_t': 'int', 'size_t': 'int', + 'int8_t': 'int', 'uint8_t': 'int', + 'int16_t': 'int', 'uint16_t': 'int', + 'int32_t': 'int', 'uint32_t': 'int', + 'int64_t': 'int', 'uint64_t': 'int', + 'float': 'float', 'double': 'float', + 'ggml_fp16_t': 'np.float16', +} + +def format_type(t: TypeDecl): + if isinstance(t, PtrDecl) or isinstance(t, Struct): + return 'ffi.CData' + if isinstance(t, Enum): + return 'int' + if isinstance(t, TypeDecl): + return format_type(t.type) + if isinstance(t, IdentifierType): + assert len(t.names) == 1, f'Expected a single name, got {t.names}' + return __c_type_to_python_type.get(t.names[0]) or 'ffi.CData' + return t.name + +class PythonStubFuncDeclVisitor(c_ast.NodeVisitor): + def __init__(self): + self.sigs = {} + self.sources = {} + + def get_source_snippet_lines(self, coord: pycparser.plyparser.Coord) -> Tuple[list[str], list[str]]: + if coord.file not in self.sources: + with open(coord.file, 'rt') as f: + self.sources[coord.file] = f.readlines() + source_lines = self.sources[coord.file] + ncomment_lines = len(list(itertools.takewhile(lambda i: re.search(r'^\s*(//|/\*)', source_lines[i]), range(coord.line - 2, -1, -1)))) + comment_lines = [l.strip() for l in source_lines[coord.line - 1 - ncomment_lines:coord.line - 1]] + decl_lines = [] + for line in source_lines[coord.line - 1:]: + decl_lines.append(line.rstrip()) + if (';' in line) or ('{' in line): break + return (comment_lines, decl_lines) + + def visit_Enum(self, node: Enum): + if node.values is not None: + for e in node.values.enumerators: + self.sigs[e.name] = f' @property\n def {e.name}(self) -> int: ...' + + def visit_Typedef(self, node: Typedef): + pass + + def visit_FuncDecl(self, node: FuncDecl): + ret_type = node.type + is_ptr = False + while isinstance(ret_type, PtrDecl): + ret_type = ret_type.type + is_ptr = True + + fun_name = ret_type.declname + if fun_name.startswith('__'): + return + + args = [] + argnames = [] + def gen_name(stem): + i = 1 + while True: + new_name = stem if i == 1 else f'{stem}{i}' + if new_name not in argnames: return new_name + i += 1 + + for a in node.args.params: + if isinstance(a, EllipsisParam): + arg_name = gen_name('args') + argnames.append(arg_name) + args.append('*' + gen_name('args')) + elif format_type(a.type) == 'None': + continue + else: + arg_name = a.name or gen_name('arg') + argnames.append(arg_name) + args.append(f'{arg_name}: {format_type(a.type)}') + + ret = format_type(ret_type if not is_ptr else node.type) + + comment_lines, decl_lines = self.get_source_snippet_lines(node.coord) + + lines = [f' def {fun_name}({", ".join(args)}) -> {ret}:'] + if len(comment_lines) == 0 and len(decl_lines) == 1: + lines += [f' """{decl_lines[0]}"""'] + else: + lines += [' """'] + lines += [f' {c.lstrip("/* ")}' for c in comment_lines] + if len(comment_lines) > 0: + lines += [''] + lines += [f' {d}' for d in decl_lines] + lines += [' """'] + lines += [' ...'] + self.sigs[fun_name] = '\n'.join(lines) + +def generate_stubs(header: str): + """ + Generates a .pyi Python stub file for the GGML API using C header files. + """ + + v = PythonStubFuncDeclVisitor() + v.visit(CParser().parse(header, "")) + + keys = list(v.sigs.keys()) + keys.sort() + + return '\n'.join([ + '# auto-generated file', + 'import ggml.ffi as ffi', + 'import numpy as np', + 'class lib:', + *[v.sigs[k] for k in keys] + ]) diff --git a/ggml/examples/python/test_tensor.py b/ggml/examples/python/test_tensor.py new file mode 100644 index 0000000..1a365fa --- /dev/null +++ b/ggml/examples/python/test_tensor.py @@ -0,0 +1,258 @@ +import pytest +from pytest import raises + +from ggml import lib, ffi +from ggml.utils import init, copy, numpy +import numpy as np +import numpy.testing as npt + +@pytest.fixture() +def ctx(): + print("setup") + yield init(mem_size=10*1024*1024) + print("teardown") + +class TestNumPy: + + # Single element + + def test_set_get_single_i32(self, ctx): + i = lib.ggml_new_i32(ctx, 42) + assert lib.ggml_get_i32_1d(i, 0) == 42 + assert numpy(i) == np.array([42], dtype=np.int32) + + def test_set_get_single_f32(self, ctx): + i = lib.ggml_new_f32(ctx, 4.2) + + epsilon = 0.000001 # Not sure why so large a difference?? + pytest.approx(lib.ggml_get_f32_1d(i, 0), 4.2, epsilon) + pytest.approx(numpy(i), np.array([4.2], dtype=np.float32), epsilon) + + def _test_copy_np_to_ggml(self, a: np.ndarray, t: ffi.CData): + a2 = a.copy() # Clone original + copy(a, t) + npt.assert_array_equal(numpy(t), a2) + + # I32 + + def test_copy_np_to_ggml_1d_i32(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_I32, 10) + a = np.arange(10, dtype=np.int32) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_2d_i32(self, ctx): + t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_I32, 2, 3) + a = np.arange(2 * 3, dtype=np.int32).reshape((2, 3)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_3d_i32(self, ctx): + t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_I32, 2, 3, 4) + a = np.arange(2 * 3 * 4, dtype=np.int32).reshape((2, 3, 4)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_i32(self, ctx): + t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_I32, 2, 3, 4, 5) + a = np.arange(2 * 3 * 4 * 5, dtype=np.int32).reshape((2, 3, 4, 5)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_n_i32(self, ctx): + dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash + pdims = ffi.new('int64_t[]', len(dims)) + for i, d in enumerate(dims): pdims[i] = d + t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_I32, len(dims), pdims) + a = np.arange(np.prod(dims), dtype=np.int32).reshape(tuple(pdims)) + self._test_copy_np_to_ggml(a, t) + + # F32 + + def test_copy_np_to_ggml_1d_f32(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10) + a = np.arange(10, dtype=np.float32) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_2d_f32(self, ctx): + t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3) + a = np.arange(2 * 3, dtype=np.float32).reshape((2, 3)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_3d_f32(self, ctx): + t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4) + a = np.arange(2 * 3 * 4, dtype=np.float32).reshape((2, 3, 4)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_f32(self, ctx): + t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5) + a = np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape((2, 3, 4, 5)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_n_f32(self, ctx): + dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash + pdims = ffi.new('int64_t[]', len(dims)) + for i, d in enumerate(dims): pdims[i] = d + t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F32, len(dims), pdims) + a = np.arange(np.prod(dims), dtype=np.float32).reshape(tuple(pdims)) + self._test_copy_np_to_ggml(a, t) + + # F16 + + def test_copy_np_to_ggml_1d_f16(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 10) + a = np.arange(10, dtype=np.float16) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_2d_f16(self, ctx): + t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F16, 2, 3) + a = np.arange(2 * 3, dtype=np.float16).reshape((2, 3)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_3d_f16(self, ctx): + t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F16, 2, 3, 4) + a = np.arange(2 * 3 * 4, dtype=np.float16).reshape((2, 3, 4)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_f16(self, ctx): + t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F16, 2, 3, 4, 5) + a = np.arange(2 * 3 * 4 * 5, dtype=np.float16).reshape((2, 3, 4, 5)) + self._test_copy_np_to_ggml(a, t) + + def test_copy_np_to_ggml_4d_n_f16(self, ctx): + dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash + pdims = ffi.new('int64_t[]', len(dims)) + for i, d in enumerate(dims): pdims[i] = d + t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F16, len(dims), pdims) + a = np.arange(np.prod(dims), dtype=np.float16).reshape(tuple(pdims)) + self._test_copy_np_to_ggml(a, t) + + # Mismatching shapes + + def test_copy_mismatching_shapes_1d(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10) + a = np.arange(10, dtype=np.float32) + copy(a, t) # OK + + a = a.reshape((5, 2)) + with raises(AssertionError): copy(a, t) + with raises(AssertionError): copy(t, a) + + def test_copy_mismatching_shapes_2d(self, ctx): + t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3) + a = np.arange(6, dtype=np.float32) + copy(a.reshape((2, 3)), t) # OK + + a = a.reshape((3, 2)) + with raises(AssertionError): copy(a, t) + with raises(AssertionError): copy(t, a) + + def test_copy_mismatching_shapes_3d(self, ctx): + t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4) + a = np.arange(24, dtype=np.float32) + copy(a.reshape((2, 3, 4)), t) # OK + + a = a.reshape((2, 4, 3)) + with raises(AssertionError): copy(a, t) + with raises(AssertionError): copy(t, a) + + def test_copy_mismatching_shapes_4d(self, ctx): + t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5) + a = np.arange(24*5, dtype=np.float32) + copy(a.reshape((2, 3, 4, 5)), t) # OK + + a = a.reshape((2, 3, 5, 4)) + with raises(AssertionError): copy(a, t) + with raises(AssertionError): copy(t, a) + + def test_copy_f16_to_f32(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1) + a = np.array([123.45], dtype=np.float16) + copy(a, t) + np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3) + + def test_copy_f32_to_f16(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 1) + a = np.array([123.45], dtype=np.float32) + copy(a, t) + np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3) + + def test_copy_f16_to_Q5_K(self, ctx): + n = 256 + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) + a = np.arange(n, dtype=np.float16) + copy(a, t) + np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05) + + def test_copy_Q5_K_to_f16(self, ctx): + n = 256 + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) + copy(np.arange(n, dtype=np.float32), t) + a = np.arange(n, dtype=np.float16) + copy(t, a) + np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05) + + def test_copy_i16_f32_mismatching_types(self, ctx): + t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1) + a = np.arange(1, dtype=np.int16) + with raises(NotImplementedError): copy(a, t) + with raises(NotImplementedError): copy(t, a) + +class TestTensorCopy: + + def test_copy_self(self, ctx): + t = lib.ggml_new_i32(ctx, 42) + copy(t, t) + assert lib.ggml_get_i32_1d(t, 0) == 42 + + def test_copy_1d(self, ctx): + t1 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10) + t2 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10) + a = np.arange(10, dtype=np.float32) + copy(a, t1) + copy(t1, t2) + assert np.allclose(a, numpy(t2)) + assert np.allclose(numpy(t1), numpy(t2)) + +class TestGraph: + + def test_add(self, ctx): + n = 256 + ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) + tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) + tsum = lib.ggml_add(ctx, ta, tb) + assert tsum.type == lib.GGML_TYPE_F32 + + gf = ffi.new('struct ggml_cgraph*') + lib.ggml_build_forward_expand(gf, tsum) + + a = np.arange(0, n, dtype=np.float32) + b = np.arange(n, 0, -1, dtype=np.float32) + copy(a, ta) + copy(b, tb) + + lib.ggml_graph_compute_with_ctx(ctx, gf, 1) + + assert np.allclose(numpy(tsum, allow_copy=True), a + b) + +class TestQuantization: + + def test_quantized_add(self, ctx): + n = 256 + ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n) + tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) + tsum = lib.ggml_add(ctx, ta, tb) + assert tsum.type == lib.GGML_TYPE_Q5_K + + gf = ffi.new('struct ggml_cgraph*') + lib.ggml_build_forward_expand(gf, tsum) + + a = np.arange(0, n, dtype=np.float32) + b = np.arange(n, 0, -1, dtype=np.float32) + copy(a, ta) + copy(b, tb) + + lib.ggml_graph_compute_with_ctx(ctx, gf, 1) + + unquantized_sum = a + b + sum = numpy(tsum, allow_copy=True) + + diff = np.linalg.norm(unquantized_sum - sum, np.inf) + assert diff > 4 + assert diff < 5 diff --git a/ggml/examples/sam/CMakeLists.txt b/ggml/examples/sam/CMakeLists.txt new file mode 100644 index 0000000..5033618 --- /dev/null +++ b/ggml/examples/sam/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# sam + +set(TEST_TARGET sam) +add_executable(${TEST_TARGET} sam.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common) + +# +# sam-quantize + +#set(TEST_TARGET sam-quantize) +#add_executable(${TEST_TARGET} quantize.cpp) +#target_link_libraries(${TEST_TARGET} PRIVATE ggml common) diff --git a/ggml/examples/sam/README.md b/ggml/examples/sam/README.md new file mode 100644 index 0000000..7354a4a --- /dev/null +++ b/ggml/examples/sam/README.md @@ -0,0 +1,95 @@ +# SAM.cpp + +Inference of Meta's [Segment Anything Model](https://github.com/facebookresearch/segment-anything/) in pure C/C++ + +## Description + +The example currently supports only the [ViT-B SAM model checkpoint](https://huggingface.co/facebook/sam-vit-base). + +## Next steps + +- [X] Reduce memory usage by utilizing the new ggml-alloc +- [X] Remove redundant graph nodes +- [ ] Make inference faster +- [X] Fix the difference in output masks compared to the PyTorch implementation +- [X] Filter masks based on stability score +- [ ] Add support for user input +- [ ] Support F16 for heavy F32 ops +- [ ] Test quantization +- [X] Support bigger model checkpoints +- [ ] GPU support + +## Quick start +Setup Python and build examples according to main README. + +```bash +# Download PTH model +wget -P examples/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth + +# Convert PTH model to ggml +python examples/sam/convert-pth-to-ggml.py examples/sam/sam_vit_b_01ec64.pth examples/sam/ 1 + +# run inference +./bin/sam -t 16 -i ../examples/sam/example.jpg -m ../examples/sam/ggml-model-f16.bin +``` + +## Downloading and converting the model checkpoints + +You can download a [model checkpoint](https://github.com/facebookresearch/segment-anything/tree/main#model-checkpoints) and convert it to `ggml` format using the script `convert-pth-to-ggml.py`: + +## Example output on M2 Ultra +``` + $ ▶ make -j sam && time ./bin/sam -t 8 -i img.jpg +[ 28%] Built target common +[ 71%] Built target ggml +[100%] Built target sam +main: seed = 1693224265 +main: loaded image 'img.jpg' (680 x 453) +sam_image_preprocess: scale = 0.664062 +main: preprocessed image (1024 x 1024) +sam_model_load: loading model from 'models/sam-vit-b/ggml-model-f16.bin' - please wait ... +sam_model_load: n_enc_state = 768 +sam_model_load: n_enc_layer = 12 +sam_model_load: n_enc_head = 12 +sam_model_load: n_enc_out_chans = 256 +sam_model_load: n_pt_embd = 4 +sam_model_load: ftype = 1 +sam_model_load: qntvr = 0 +operator(): ggml ctx size = 202.32 MB +sam_model_load: ...................................... done +sam_model_load: model size = 185.05 MB / num tensors = 304 +embd_img +dims: 64 64 256 1 f32 +First & Last 10 elements: +-0.05117 -0.06408 -0.07154 -0.06991 -0.07212 -0.07690 -0.07508 -0.07281 -0.07383 -0.06779 +0.01589 0.01775 0.02250 0.01675 0.01766 0.01661 0.01811 0.02051 0.02103 0.03382 +sum: 12736.272313 + +Skipping mask 0 with iou 0.705935 below threshold 0.880000 +Skipping mask 1 with iou 0.762136 below threshold 0.880000 +Mask 2: iou = 0.947081, stability_score = 0.955437, bbox (371, 436), (144, 168) + + +main: load time = 51.28 ms +main: total time = 2047.49 ms + +real 0m2.068s +user 0m16.343s +sys 0m0.214s +``` + +Input point is (414.375, 162.796875) (currently hardcoded) + +Input image: + +![llamas](https://user-images.githubusercontent.com/8558655/261301565-37b7bf4b-bf91-40cf-8ec1-1532316e1612.jpg) + +Output mask (mask_out_2.png in build folder): + +![mask_glasses](https://user-images.githubusercontent.com/8558655/263706800-47eeea30-1457-4c87-938b-8f11536c5aa7.png) + +## References + +- [ggml](https://github.com/ggerganov/ggml) +- [SAM](https://segment-anything.com/) +- [SAM demo](https://segment-anything.com/demo) diff --git a/ggml/examples/sam/convert-pth-to-ggml.py b/ggml/examples/sam/convert-pth-to-ggml.py new file mode 100644 index 0000000..0de422e --- /dev/null +++ b/ggml/examples/sam/convert-pth-to-ggml.py @@ -0,0 +1,147 @@ +# Convert a SAM model checkpoint to a ggml compatible file +# + +import sys +import torch +import struct +import numpy as np + +if len(sys.argv) < 3: + print("Usage: convert-pth-to-ggml.py file-model dir-output [ftype]\n") + print(" ftype == 0 -> float32") + print(" ftype == 1 -> float16") + sys.exit(1) + +# output in the same directory as the model +fname_model = sys.argv[1] +dir_out = sys.argv[2] +fname_out = dir_out + "/ggml-model.bin" + +# possible data types +# ftype == 0 -> float32 +# ftype == 1 -> float16 +# +# map from ftype to string +ftype_str = ["f32", "f16"] + +ftype = 1 +if len(sys.argv) > 3: + ftype = int(sys.argv[3]) + +if ftype < 0 or ftype > 1: + print("Invalid ftype: " + str(ftype)) + sys.exit(1) + +fname_out = fname_out.replace(".bin", "-" + ftype_str[ftype] + ".bin") + +# Default params are set to sam_vit_b checkpoint +n_enc_state = 768 +n_enc_layers = 12 +n_enc_heads = 12 +n_enc_out_chans = 256 +n_pt_embd = 4 + +model = torch.load(fname_model, map_location="cpu") +for k, v in model.items(): + print(k, v.shape) + if k == "image_encoder.blocks.0.norm1.weight": + n_enc_state = v.shape[0] + +if n_enc_state == 1024: # sam_vit_l + n_enc_layers = 24 + n_enc_heads = 16 +elif n_enc_state == 1280: # sam_vit_h + n_enc_layers = 32 + n_enc_heads = 16 + +hparams = { + "n_enc_state": n_enc_state, + "n_enc_layers": n_enc_layers, + "n_enc_heads": n_enc_heads, + "n_enc_out_chans": n_enc_out_chans, + "n_pt_embd": n_pt_embd, +} + +print(hparams) + +for k, v in model.items(): + print(k, v.shape) + +#exit() +#code.interact(local=locals()) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["n_enc_state"])) +fout.write(struct.pack("i", hparams["n_enc_layers"])) +fout.write(struct.pack("i", hparams["n_enc_heads"])) +fout.write(struct.pack("i", hparams["n_enc_out_chans"])) +fout.write(struct.pack("i", hparams["n_pt_embd"])) +fout.write(struct.pack("i", ftype)) + +for k, v in model.items(): + name = k + shape = v.shape + + if name[:19] == "prompt_encoder.mask": + continue + + print("Processing variable: " + name + " with shape: ", shape, " and type: ", v.dtype) + + #data = tf.train.load_variable(dir_model, name).squeeze() + #data = v.numpy().squeeze() + data = v.numpy() + n_dims = len(data.shape) + + # for efficiency - transpose some matrices + # "model/h.*/attn/c_attn/w" + # "model/h.*/attn/c_proj/w" + # "model/h.*/mlp/c_fc/w" + # "model/h.*/mlp/c_proj/w" + #if name[-14:] == "/attn/c_attn/w" or \ + # name[-14:] == "/attn/c_proj/w" or \ + # name[-11:] == "/mlp/c_fc/w" or \ + # name[-13:] == "/mlp/c_proj/w": + # print(" Transposing") + # data = data.transpose() + + dshape = data.shape + + # default type is fp16 + ftype_cur = 1 + if ftype == 0 or n_dims == 1 or \ + name == "image_encoder.pos_embed" or \ + name.startswith("prompt_encoder") or \ + name.startswith("mask_decoder.iou_token") or \ + name.startswith("mask_decoder.mask_tokens"): + print(" Converting to float32") + data = data.astype(np.float32) + ftype_cur = 0 + else: + print(" Converting to float16") + data = data.astype(np.float16) + + # reshape the 1D bias into a 4D tensor so we can use ggml_repeat + # keep it in F32 since the data is small + if name == "image_encoder.patch_embed.proj.bias": + data = data.reshape(1, data.shape[0], 1, 1) + n_dims = len(data.shape) + dshape = data.shape + + print(" New shape: ", dshape) + + # header + str = name.encode('utf-8') + fout.write(struct.pack("iii", n_dims, len(str), ftype_cur)) + for i in range(n_dims): + fout.write(struct.pack("i", dshape[n_dims - 1 - i])) + fout.write(str) + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/ggml/examples/sam/example.jpg b/ggml/examples/sam/example.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d5116e807055481d0eb50bb053b3f6da1fea0ad GIT binary patch literal 79236 zcmb5VWl$X7(*`=QxXThCxCMtmAR)osf-LUAg1fsV*ajAd1b1KDoe(^@YX}--Avi(r z{@#Dpt^4&()lSXM(ds@=_nD{rJS{$L07Oc1igEx73h=+{X$6o5&_QTuXdrat4ILdF z0}~Gm6L}Hh;$q_w6Oxb+6A}}XKBFcl1wSVvCZ=GZcuqq@M@L6W&dAJ2%S=s6NBdtV zD9EOm7?=cDSOm0SVleIh@9U`tAjU%JM=b@RFaoH=C?H~#ryl?f0HC0O{@>{$Cyfe1 zP7(PM3maLiLGvxEGO1JMr)2;SSre5QL=1=n zEg^8&T=~zrd{WPpZ?=6Jj1BF%L1X2NPg?j!R*0s&@ofFUj)RjT@NN zQ7zTuZKVio6iNWHauFTJ;*gvo8>l*zS|b-kmd`d+*-uHz=mR&812bzt(e1%9>Io7k z2A2hJVhO-Tg%XSrf)Vj$y}<@#lFU5ir=AWh6xqx_um!S#^eZbXbMdb8;OTRKb&*~3 zzCkWN?OZv>b6}*$UOO!0jlQ0+jyeEQx2hMw`c#=W<}85YRHIXGn2QgyDV4Wdyw^es zk|>wvPiQVGMzdu4VlFCK8}K$%2iX9MsRDRNhRDDWOS1ui4YP-_HOpo`$*8H5bc zB&vkY=~>b11jE6R5YnEjO8O~_^zUpa{A>zESb6l5s4_ZHMOc;n$QJYL#*O(6rD#!b z;H5{Lk ztQ>g$Q%6S~LkFUxjwxy{SUv*K+gmiEJNm1mKoE(N z-C|xo7^RH`PB1IsE`cUIg<^mjg9H2ko9dv*p#!$E6l4tsFWCUpb2)SeZVK=>;Pn*8 zRH$t8o}2^`r@u`Pd}~JmaD*d@_>f8gxE;9Jo*^ZSvAn+gvOy9l*bbfV(|JA!!xaha z@O;$dmH!?P`MBxo$f^JmT^@@DS`QO52`R^g-_W4)?ki9(C zP_^9iN&*X!$Ei#PC)*4mUKzBTGB9++HUMOxG!3l=Q%ZmY+}@E#qd`j@^R2;@fe{?r zzy&J>PF4g>pE7tK0$V0&=vo(6Pbr|M*Ya0SVNBzc(n&^hm2OI1IfIksqA84F8|cB2 z(vCJ^!5FzBm$$R1k)Eb;Q&NzVN>DxxDvp#D5UtKxPcesJF?9?O6dP|zE7f>ahmxcl+oegxoBZijzv=E zhAU@9J=l)BO(ZF}hAT>_X7*;b#*H-%lO(u?xJn3QqjW%l?Fx>dovj*KpE^;DCt1;N zN?9>nWS5Y4v|856tz7xvJEYw5R+Qt3ISu|(ieu3b7NU|<*%9to2+D+8O&O!x;7o6z zdCfyll@i8mpb{ki)junOP9{iTrR1?IZz6FZ1Gj;5BEg0oO47h$>xhI6ChC9fc5D=K zAs7ZEe98$Dqss9_j3Lnf6snGVfb+VY*(Y~s0CNoBY z0ScrQn~w;=1T8nn62tw}0iDuVLOAddc>yj2bG7TAz4O6Xv7zZ4&pFUvLQo87N=uQ( z*+D1LP5_M32+oI+A%DO!4n)>gmMuTzbyzaQK*ClQj1r#I@${aX2#F}94GR4qi~t6Evmh-q zo3Kg1yy;moE)qMWGMoQtuZbKsJ4uQw^DNHMfm`S$#&Sb+Wn}qCkW3*BjjTNw16VOwK^EEOzQ-nm^G~nX%i7>k?{FI0 znzJcQZ=)$q5*wqblNF&*>(nYGNJDL~k$TVtBOQc{`eq7aV_XWrj1+JyYRG@uK|+%_1a=!-^v_A*AZ6r2TD~YWR4n>4tRrNL{HO#8+5dF&4OyLw-L2sp zQe9@qSKt5Z38~{e3Z$7Mz4*@phKB<$>`jv2v)O>O|AR`8u#Qfo-x?~Dd6ZWkSh^AQ zv)Cn>O%MkX2}By@Y+4$n4gaT?{{Qsy->rvjQXQkG2-s}MPnMLR2#dk_LB}Bop5&7U z=i~IS&8lM{4VWVs!j_ANE0^E&1?d>fs|8GTdh$_21kj1dYLJQofZZrw0e^c_%@87_ zwF-g92MW(fOoFA;cV>&vDh)mwZ!s|mF_Z3}U;Ufa;HeG!rQXt*{O|iZ16%5=ha+nW zX1|wv;!3veYWZy!Yp&ziH7C%;kK_CV!ZZHAzQ+}{K z#IMP%gcluoEquKUGmHHZO!9i>Vxp7nwt-yl&Vsrw06Dket2@hDa<`GvOacNo#U2pN zuxMnJEW5Q^8p}`9aGjNj;B5IKbno{CBk%YO`>nK0N%-w;r14;~?9cFk`JU^1q=k4N zfzofqhV+CAY}uHNR?C`n9OM)lE()OO5D;009YKK9jH72p7SEYP;!6BV^th(b-ri&O z=gktu2yJH(ZHg5@dd>91fO7wjTZRL^{g6O~(73Hs;x$&rI@yNuzeb^x@IRdRqb zI02P&)FAJMjZq8NJrh=21I<1GT$eHm1gXjP??() ziU9_=iKF4P`a^5&v{hZrH^`ihBGx#fXPcMy-YQXrC4t0oBWIA=EoPv1z%^kgzu!0M zD#%#b_@GlxHFwV1KML9!UThxq>TjO(S?|D0(R!0i$-jhOyl*=Hd>I)^gi(AmAZe%t ziRwlHG$&?kT4f7YfbRNijEIiF@Y8CXbBfgN z0ooG~?Qy92POxo^Uu-6xf=BoM^r9@8d$-lOClYk8a(L@?ySJPm0-ZlMaYf_ndpd0dRCYed(bFpC+Mdg}FU_VD?*I*N6 zL*Z2-VV7cK0~9LZ=_5!V51Zv&j`GTtae%e;_o+fyfJlm_s4x=5TK6_iLerBX}>#Y-J7BSA(QJ1zEM6#C-;XDNB=1IFQiy)NmZ%{At9}3Z_&NlgXK{epT|oC7aJtf3Uzl zM0Xw1_t4*3+|HFviAv(HC)X>oe_+Madt_e5etStd=G)|76rd(Rl1L!ZVckkObom7I zD_kIScISuTj0DeQ7p0y6tl@z`Owlfn&f%{QhQHH2$*NV%^?%)`EglHl$D^c3WS+>3 zQ!a?(+(&jQUW`T^7o3|ro@RqW`z%^%xBls7-C7vQBrtbiF@~g~FV-tRQa^$bIXVd4 zs@RH$Vk%R|^`)g0k{49oG1e}0-!X!cm1zk@4+EKQUlfr2yJ2Q*OgK?oSE6r7b^a|S zzc5{HQgUiXki0;8XY*_(6p+h_^XoQskS@yrkNMs)6|Nj%%1|v3%BvthS5$lud0=Nv}Z?WMFmK9<~&sd z!;ACC%iH3@Pw#nJOoJ@_o}d4Ud)7Ai{wD1z1G~rYPhaeP1%FPk^vdl_Q1qNZGC6mZ zZ9R7+#87re#Q|~G{--`Qw&z&;w9oosJRshI=!W3+eULr2P>SEQRqJvs#4*WC&uL^))>SpIn zjPnPpJ*I9EZOEqO^7llU*Mp526njE%h3K+5vb#V0>GwLb^Tfgt(yaz`Gj zb}K#*Ka&fSa}@8yAK!^iMGvBp>o?UxE=wK*T3Ql$U%arT#>@I)>be5B#(gW5{o%SU z^B#@CxkY%VGp?rJbCZfjFb^gdf8`5KRJ@vK4$$&Ev^KgsZ)y5tpJhkEMfHKGx+H;^ z@!KwciZg>(o=IP~Q{Q?e4VA91qwddUPBD}K#SYX!?CXDS_7hhIb88L$=>pJI4#fJ) zEwsnrBGq?Pm4sP!zjMXx@#G3@Vu*WW|J^42n_rJCd#n6Rp=aUZYpGS~U!w)mK4*KD zv7f_ud!I(cL;d~X}K@qs)p zeWstUc1MEHL#pG87Gq@{F&SIIQ8K+*pvU#{{w%2R4}c-8ZGHE4pJdSkKeL%H=bhQ^ z&`C`F8*wa}WNTD7o_1*^vvwLIvwF>NkwPwGrhoz*1zZc}VjBuWw|lja{{2rI&DtR! zJDv&?y2xMb2Le8gR5GQR#U)5rWUVp8rRf>Ll2km8RYQ^w$^-um85EV`pCzq8Yp0-#W}iaggq&d@9vC(VH#}bF!T> zb#wY{au}r>7DB1<s| z|L^?oOIJ_B0ZBtpsH38(rxa1zsM??~^?xL;%*DtR$|uWPZek-wLw*iPHIzfqv1Pf! zLS;D`03~)=vO;h^Y!*;zFg4Xlsx1OomqXqb&2pI%Shp&Cw%{4}3a8y$Znxk#pwxRw zPb4`+e?@-_4ZW$hb$$_bO|m*_HPk{f(5o6qD0=$-5oAyL1X%SK8?9xem@6qz^J6k6 zHN>?QK2p25uDF&OM~mV}#eeWpv&?vE*Y4j5yOV5++WdqRwG z3WOe}fP@BC?90t^x@Bb<*b`7#*!6Yw8#nZ8{40J|k)|H6#2{ax%Ze}H{<&M<&KZ_y z^VrV`bz=8;F71;+%-Wc7`iTGl=#WD-BTt*ZVd zVpHcNLIkBkSF3)s&)$=7X4y_RqqJyoz3%k=H#;VdX;BhmnwT`WxY=$Wg)ygN>0p6F zK-eVaIkvkRnE%S#984H($$PCpNe?}$QGE7+xFW$A|A&{MYH;czu^R6_qboA|D z9^wgzGmz`pZp(gXA>jNxYfAd=0jdzSZg~eIEETc7UC@-OmvJoqS|}6q(m6W9hjmKv z?Ph$lLKUO#^6ALWH?nMf1mgH!D@#YVFRaa}xQG{Og%D@|Aj{fGkn?IHU6m@-ExV;o%*~Z>sOr4OFQ0u~XNj6h)Kx*W%dS&%H z_fE_7CHH5_8(3hl*v0uB_r;&2@?Tor-;|egcwQz@4#8rZ1}RQ$K0`#H{OMPi`_Q2`*ZspgqGuZsj(6A*?qB%oJ$QHYxL zkkYL{PIdgL`Q^XnzG@jPqMJNoM`U+uA`gB7aPfOUaTYl_qtIJo_vebZ zT;EpWq@m|&QJv&POP(nMCC`Uod;YyG{M(tS6nhD43b&Tpq8}5=3ETwy0yWzD(WC`Z zx<}M(?M4y@@M8royLC3L%mk4Vn(Y13YF(d)q!jXLQit8`JzU4r(>$HYBh44klOwHx_qDi zAPSy!XRxI5iZ6jw^eYBhHa5+~McM1vt_gX)6UKDCd(JS|Q{GGG59ehYIV`rnNmh36WLCVnBvhbBoO)4J zy5)l8kEo}MKSOQuE!ByY?k4>v1r#GO$@aRJXNR(0{?4rHs6{Yaq9h$Y1an-xrPda{ zQ?5$~xlSxiT_OR2g_Fy# z-($i?hwiq^bq8`9RX>Le_#&1Er`owbwTRY}T00^s52n+PzqZWR57&{_NJ>ZFf@pcJVN?6?|c+7>D%_Y&SqOYX#oeNh@%Xqh zf}5Y`fQBb|!6e!Z*Ac^b@4x`Fckg0i^F$d~+dCn2Jxti2$Ro498MRxWVR3z)(-l(` zcvGIy^C0%SBtowWj{Mk(RKGKMeYx7Q9VfR3BdjSZ`v<`W#17OViMJrM)06d_IxIBePlsA7A3ukF>7Ed9=g}0)|6CfHY07CNQrL%&TwY zZtT(Q8~?F{3^T=zr>84oYpZ(Gl-$^t6)Fe9j-1qVJmryR~N|mOsJd5&PyzA;clFjM785<@OcJcqv=Tj0{bo4uAqL%`vq=oL) z>THob8SUQ(iTyvQ;fOE^Bn=T^FI<{8! zi~&v6PJ0$ayF?TPst|t)#ifYfG@}92nm~P6^{KAvlm*Y#HI>t#Pl7b)*pm75AxK+q z;`JaFgYO4wz8m#`^FxgZVg$E`oHjrI+qKhoiIQS{Iis052Gs~BnX5zLahIQ}r{N&V z9@-x>H`dMf;>rd>_s7a>Pk;~&tJ(9^FqaIq{%#b>>tb%mG0zKrB$>k7AtoM0IC}gE z>G}olz2^Vc3)G3dimCljJ#k<5+Fx1l4ia-9k`k4Qibn4mw>jvL%`XgL=fVHOVnVfU zvM&3hwu4#fnEW>totvZBw~62c@c0&_vFly(=-0mAeWaguT;F*ae6Av z6!73M?91C6C9|2Tjqc89XD3#i>5wCA$8yoY;fqphfxq>w%*?5pFW%uFD*6hWD6*!a z@TGnTR}k$s7@Ylh&J0~PW4bpNX+hgsy&7}1dlQ4-pVHW>uSK@~_e+fXIKThi9^}Ts zL~~m)n5=V%0m`>2IH0O_N@wR$Jii(fm@)kDnmMwx(t%0Ymf3^O!w(@^pZ(Z60T&c2 zb1pF+v2M*21F;3PCW#FK#OKAL?j?H>r0(c{T8yY8L7_Q!R~ILWOJdb%c0wQHm>i(g zpaU_6!6Io+maM59;qAiBRLoHtlP5s0{o}Jo1H!0(RT!&&zdOVIlzW6DjbF0FW+9e` zmeP-wu?noHsfPx)25)pac5L0Oo1;$}4y~{x&Y60KC#q*eL#kD7$YILx zfap8$&QN-sd4Vn30;rzInQJhaT=2}b*Isa~qa2mZ8dOAmz%6$dmX-8V1sQ+QE8I(9 zh%WTPDd$%#9I{~gplQ;WoyC=4=cYC~-|;ZF+}UvL=)!HG(yDG2_2LMo zoOu>hTq4Hmsim9I@W*k@Tr;)hTVN;H>qFUY*_!L~U0J=73d_6c?Cu1(6zc>s=}Yn5b3`g4ya12MAG7}&r(N1S1n+o z@)^k=ie$2DfA@SA+;T*1r4@w1<|hMz7)b z0ir)x!IQ@_Gw??#%5q@+o~W-`=rM&$z-=dGuM)c9*FJystAp@wl~-1eE6nSBWv0`M z7B2H4yb#CjH$(vhYGwN4KNGBe@E}+{`~-+Mec!It<4qHmywHzy5d~TN(_-Ay@)s_t z>!?qE5wkIrbt5|H(szlbz1+|_8di9zoSBL~Q19%6ov^e7BjW496zh5dRN3$P9&9fH zZIrDUzZRP|6Dbq<6w>aAMqO}*i5}z|%LtdGxP!}eVK{vEFge@v66Z`MGS*&n_hkl6kOW z+4NsF@Q8ir-@pg`m=dv0;@OapJQp9OyeJDI@V6?Bqj^`vbBR?6rwN*{q5O7Dh8O61 zOh72svK8|&b(k%2fqJOuQvW?()17%Nnol&56Onv-m+v-n z@7nD|L#&{Kq=OG=NeARKQDj_SQ&NgsKZ}lSoELM)r=9|s{TD_+_g1muakhT-F)prc zUg&)dIkUnNb-;JR`O-oYA%Q>7T!RB`7{8gT)h|C2>4@p2ym@35%Z^4U=@}GARIzvw zrRlA?1yS;a1R3!VPDdraI~i`RZio{W^9TM@y^tdi5irPPjE$@;g$_@gFkSE}uY0m9_PzY@Pid(aQxosR?RG-1@ z;Mo3&`d!WzWO&eXuWf-;XyRBXA7jZ1zl8GX?& zjGSm}iHoiJ%Eatn!8-%oO~rvMPg`G1H>$l-P8`@TY|`7a`Q{##pVSPTVhi=st<{Qz zPp}&vsQ+o}tSTw99rkKEDvYMsl$}@;E0v!RV!~ufH$iMa|LNB;$sVa0IXsI<`+5x% z>8B0Zs-1X?Io;})J|=L8@zVTS*&_A{fP!Esn3FL6pwFl23L<>)kEWNrs&?;GtMgY5 z-m#2z_QzEG6=5bKEYI{ZTl_TU^|G4#hHTch(2if4Y&%t_t_EL3vDK7$!*^s+HzpfP zh_jEAoiImr$!M74>#@0>ljcc&qSkrrh?>wpTFH%D(%ftvC4Lx%r8M|-U6Fl)oS<%jxkTk@$IqQ5BF5*AOB-kXr-yuuXkcu z{H&D%*FuPG30xEg&;HE`PECsCE%+HXxK&`bp(Xw!uJR`lqLb{oolDv} zZa2TSwRbv`bO_J3H9qJ18Q8lw6S|!6@U@M*EwUnauCU48o)gmQUCYbqJZtJ#A0B5kD;qPeWA$Q10Yd}9yh`qpiiQKV1EnDxr9BiygLc!iHM(=P^NQoSN&-jFYxRnRleGsisR-->twx`f7j z?H)W5XIBRovW&N7R=JJ>S2#Qd?|`!;ONd)_m00L~VoT zY1wS|13ht+mQ^##sc?nqdx(kjYK}(;DA%HypC=tW7Y{R*cKU1k$h5DzZN}I>m)9wI z_G8cI379?z@+8%?r+l?@ir{g4w=R0tUAq%pW3;xeO#MoXiBjI@uaHphgxQ#|ZMs!a zVM|vgd*Yi3*FFNQiEi;dU9ELW30})-hrJmVa)oKVRcWF0=`EX4^x)_%myWMFE+2<= zh3F-YJ2(yt=9HWTZVXkUuPfloZA#?vg`k$ZKn^m)FTC2PJfYP(uYN5d?)bXa4Og%m zDR=Z)48vl4VtuEJy{;u!(!Lhr6xs0;<0Y}j6qgI@$Opq&1L2iX zjC4&z5^mXU^4EMf37>d+UM7YcL#H`R&X#!B508`2j7fig5AK`yK0OpQ3Z>Ojd5aD& z&m>}AM@BQV!PHl*wDp^Miu~O21Ml7~I2b0~@}9ly;N4Sn{21)aRQmlpGPYEk)}Bz(A66@q*sANX52pj?_T^p1}7Dn})0nlcVO zzTaLknfG0u3s7t_HD+sYoCnc%u87pMtFJ)P$oNXvX3BTJjcwDdeeING6E)_9?_EZ3 zZ!*02F>4-q)*F3qnwbr~?tRhziUxk7JA1I72>jsg&!~d4Qrav+aE;=CA{y8#h7hA8gM~j0Nq5 z%UDhc(g9jd##$AZ($!C#X<9l=nC85Fi?wg7g)hgT&EbB@P48)UxmBl)6{XMO!6{5s zgbZk~m%eB`N-b#7*~K;U3h&gfrSFBkP7`+W z;PzITwTwC@#{NODIDzs>q5EF`dC_hEtqM#%-MxB!?fq1 z;zAB#m@)DHtogzs*MdmzkqhS`v$3maAHjHatx9^}_XI%lZEwkne#Ftku)eL^mr$|} zOUIK1opV~$R%q;B44;54c}mKZX%MNw8Xm?21?D2KjGD%58L3F$80xY;{FTk#ZEHMC z7dzgmM?6DM5qRaVgXdORh0hpir2fV>;}Bqnk!bo^60szQJA71!y&tWB2S`j+I$q*t zj5!;uGo5dC&Nwn?jAfQb84Y9iZ26eFp>X)piW~}KJ9tK5*g93RjOvZPP&KDU@4YL+ z{H{A0=b%|JHD`DXrL^k0V_M#+wFsIdU*nunP*;7?ND_*_n$A?Z;Yaj@OesBF^r%w!Q_oxc=x?~%?b=h3upj$IlbJ@72r7qgja9P0{m$K=x~#vgnxTMnbZM&l|aYH*|K$F~>Ue+R^z z&#J{gDDTU9~bUFaDgs68u=Ue45{5K&E?_Aa)lu*yWa_54**ulgb; z46x;QJqpp49S8-BStzwn^?epPruE5(WN+a_vhLc_ycZqRin4I2u+tX#9ru|7G5D79 z7ZcsYzJ~wMwtq!OnV9iUSB~cIE)>L}NBq9msCmlfUIQu?lF*gnqDGF~uY0jIe2MkE z&hd2DB`RvFO{l%bOe^TSM^{`O@qB+=2zS}*4d%eMw<)XB-?;M=FTXc?s=iuz0(P$d zEnu41Yw8PDjx1#{*S9PhhI1ohUNHrcR09_#Q6fLyXGip1MeNMFIfy;Q(hJC&5JCwg(y;CGyA}Nez?ndHHoVA z!9;Z%S$xZ?TBp>x??ySP?(8GZqE`(WDMOeN{YJ~ayNfRq zW<;IQ@2SN4Up-{sQi`!HzotN|HYc<&sru6sT>?*3nqK(?yU>_F} zzF&Z{w%D(6jPE2Aw?y!A*nD;8P4zlGQdkUX6+K*7$5oiQko_dEdpHsuP&VRtZf4d> zxL{A1w<^lvN5>Yivz?JGFZ#z;icL>z8`V-L>N@`d?mr0-OUcsMh9fgGvB7zH*STnm zNN(J3@>U(ys_x5lT^c=qZUICuKUS6v8Jv|gkW@mEK>IHnOO_)n1RI?rEF75@sVN~y zJsrgc)+l8o&VVS`J#$@KZr~|sg4B+@7KZLaWIw^!pR1TbGmOYgI_DNT-}rHPK1)c5 zUgDQimDm#Bq-poM%56MsN_=q4Uia0kOqhLB?<1@RjtG9h?1#^9GCT_puuJ+pwnp0cCj=69TpQT1D)wV)Fzda`)k>}d z#`R=IOWDL02ir`3sRiflj|G=Ad?t$y+1UGOL^o4&zyHl|{5#vU?QsEAyT_tE#$wUW zRdr2!GEl>DL3iqs;@v?1Cf?N>buNt%Iq$rz@DS7+}|IEJF!@0TpDzZx;g1hOYd1Wo%Fq8OUo}ktK3) z#kl`MFa33L0FOT?AtrSERnLHHdZeeCn#Z)>m|)RgN$P#O`uBMjiB^L+s@Uq?q+%~I zgcCV3Jr-KcWVgGyP15-s$4E!JX&a(MTDfLb{vtd+GPz8MB8BWf%duBk)#xaB9Bl)i zY{=GdrM7r0{z3Ip`LMZ7w5xEEr;Mg|8@-7_qPI5o#|G%vTcN6T*g}Ke$xq$u1#4Bu zQCA0J11Z_;2ztHe%oWA<;xps!@n1^Qj zVLS8Jw^v4 zr0&eoO%i_;_-Zaz5a$zcLmZAVZ3{1X$8Sqil5*%+1vKToD77~g7M-RihBRdbVQK~( zlhhD`SzvCqjV8OfMA_aWl7O$(KQ3p-MDp}Qv6aoc$RY$H7Cv7jzymdwaW|IQ?h760 zW9*0uCb)`gQt{V11!ddN=xpxFPl}0rSa1;Oj_gPZd9Tnez z4Qy;3{Wyx%5xTWl{$-m$!2%(F70OGhpZA32tJvn!IZpr%M+M#OC}BdOJ5+yIS9{v? zov+Y1ud|rAhV-4lMYsCm0XuwOeWkC;t|mhTn@3`aM8Rt;n3480A47{SHhK`pe#U#9 zz|PBR{RvPwL|ek0>|dS!^07Syjr7^i+LszAo?_k6N^dsHZ;y93&vun7+M}yjQ<@dIs*G3f5`ABE zD&H9x{(HBRdGe8LDTNmIhE|*xf%TcIQe*YAtH z-pxJQ7ZjF5S;nAEx+Bl|N8T|nb@M>jhsoC}<-7cI9{t5kbkJ;4=)buYGtmbA;pzO% z*+EPCu-z@6HIXt3MF|J$QkNb&Qx3AcKF7S{jQ8%Y<#h<<%N`$8 zu8Gh|{lNWYFnKL$sZ-*5Ykz=qF`GtzrTYE;@q8KnT%|J}lpKR~)w?7(Sgpv#Lq77> z>Qf=EmO}9dG0xBN#muX@#4WcQdvr|DOj(kGXX+@Tr-y1udf(YRtIB9B@9g!Ugk&8$ zhwKr=mAX77yh(Q3&TFvOcT;${WOs-J8ar-)^wcYQ=f>&inPd=(G6UdXGH~7vfJD)S>sc zNOt#DR@Ylpm|y&xeb;MoB7Mb=w|CiV?YeYT%X_WAJmx#m6W`8;TC^yaZ9oG>HxlQM z$3KKxM@V2MPmJT3S4{NBriFXiPsu%G8c ztW2|~2!kwMg90*54$q(*?G(Q$6&3fRg*zG5zCwXDmY|tg-KORlynlY4$#+vjPr%uD z2l}<%C?BF#c%ekx8vgvCuaM{6g-o4YbF;GTo(P4qGTT+q0Y%T2m+K(FJDNVz^9w)8 zc%itp)3PmWMBPi}Z)K~|_NQT^->!#A2$PO9H5Fm$nIn&Oq1=W>dWx+*C;#w?fJFK6 z=JvWKp?!HmH}*}#z5-i~30v?&)QvYwz!`=z9>(Q3&Yhnh^fV``e%a{P2RnrMNr@e~ zD#9*^B*}X7&Db)D^9s9H7|*G-8G z15_eg-3v*I9^tEPp7-STo%%*E8KCijF7ht&XnxShl@aY|?#ydvO-+KdjxXM+2Q;oR zWJhXEqW15876je>_O|=%Z$b9C)#tW;DI0adRxsVzb|n{TO6{^9Ct} ze7{)|$5$J^sEcSTP!AGQ)hNLL(Ho&X0b8eXM2Je+XG&ZbqJ7a#c9wa+N> z1}rRknd0TE?Mgdd?i)LoUnXY@>C?m1X*E9coV~={iFtUX7#`cl95PmRcs0L_pFhey z_JdEd)tA-#%c<57g`Tv#Iv4h$phUJEXHlRE0y3`4r{pXNQ{8xcH*?h4BwfQ_0%{t8dW!A~BgB zp_kq;6Q{Cia*c-!G)YDUrhqVHUI=1wA@iLdAtE~&!~SE74UvdL4~Ue6e~UVFmG}T( zP)wbBO4m~jVn-wCiTrAAj`j2u9Z@|a!a!1aKo1Z^FHG>g)zX&7Gf)l36<7YBNrW@AK1p@LwVnJIEr1|h1RC#TO2yOUfb7-AKt@n3NXA%1c z=Dw&P{b3QdlN+CqsQ2TX8%Xod&VN8mL%P_=z6GR1tHm;RSt zeg87c>751KQQApayRH+>1v8ER2>V11c_<5?RIpk}T!~N#6J=i-d2kRr7wv&f zg)&bFc_Q@x&R5t%(V>AUuNC+26%dQxd>QM4dS z+C}=ml8hafn}eU~ZT-ZTJjx1Y8=?tNKMc1WQ^giG)V3IbgExkj^)`syRgZrfSw8-9 zO^kDx-K_t;welFpY4h%?Id_c;POdjAuoAkgteh6zeUlyhuHHke=D;{e-*ddc8DT89 zHQdyQimuO-iJCVgOZ5xW>2VJvT>HM(LpwFC>>)?Tu&ii^ld2R(B($?gqOiS9VE6AK zxy`Rn;{dK%aj894{0ZkFJ{gzVt!YE)G(=lB?%VKgL3nxfH}MQ4k!a?qrrs|6!8!Hn zK_nrIw7Bhy?qn6JN8r#bcki9{SpSstwOTK=xCcG88}_j~Lf7XLL2lEb*_D;;60R~; z>ej5*QO?F(3(-!3^c-6?DAeHfHCu-K(+W8CY@!#HAwNTLXvH+i_Tn=|< z8)gCv-f1bp^EVWxgp^bnpL(eS8LPF$n9sg(PU)m4lc)skrKxp#Um!weFTdEoXbGEp zk)S>%>i_eWr zSH*oc(@|Ys@ES{cB^v{o(o)tJKm~%*b84BY3r(KLa0199buYaNP#c6HtQHbf#l|{| z%I*vwhdAWVN@_SGPQ`ao23^%xZWK#1KG(&)xha~QHVjXkfAu%saBm#*to+yeeO!dH zQS(fmRFaWu)4@++lEt+P%84D5nnOdO>{W7Ej<+s>u8^*TeMO!sv+j;Yq5=gxyGj)7 z>?7fr`_|o3v3-$Oevfd^IhHq>_c6F_XmoOEo* z%^HhavieYmH7oMXN~+`SpkKrdd_yd+y(mQX0xVoi8?LNO&dIJ6*00=?%z5z|x1m_sXahjM|cqy**=8|;-J?zN@ zW>2#O9#c6OAp3e(6q;6`mcecw$_tV(AwpcQ)Q&0{%M8`f*z4U8-!3Tc84-(+*dz>N zCawPf1|s`MS!sf1SmSx43{?RG80R0C^EF)0r^y}UX)*$G$%;!od@?Nj;s)c*j)iyLbhJnMHG9OQ4#NFCTH{6C=i z)n#=Kqj==2Hr2~-V~mexK7-^xI^%V<$z3kv(7JTs>5p$13K^tnc3fOql^Nh-?$frfi>9>N`7lyr*@JS>z-FvM53nFyL}Z`tyTK4`A|r?KFwzIX?E#$G3Lwhy5LZ(;3vm7@Aoa12qess+eioQ91M|BBH(4TLx*WqT79+Bd; z_MfNTp|H0O=X!u+BRi8PmK?XxeJWGdUN&_0kw5l*i5Kr4cg&514WGmB-2Rol4?w)u z=C``kEaMPeNi;WM8_WIJ-y60LNNy10IRp;kyw&0-LUmtGbsXYHj{g8lc?yvfcq+b< zdXhopoc8pt$vh)~n#(O4FBWuNA6C57S5H}Fx|Fu^Mq&rJTzARIuAa2lJUa0QrDk=R zG+lNf=EUVb8C`kW#Cn`#ccDJD<_nA4xh>7Tq8aYqUn1Hx_?es3s`fUns{!mi%Ir|S7zDp;P*WC(A!$DXWGOF(qImYmR-_TV(O`Wkcv^Dq4XEq1> z#oH!%B#dL^1z6q>>D?!&Ya+|48hcxLRI1)vV0kP@1;_B`)B}&FW@&WT88=Kf4gp~q zDxV|jDf^_fNJacn$8!>IakRUB?EOgPdw1vZs&Tt5WYV`z;}?V1o)q*w*Geez^;>J( zW?zCx;waL_d0p5gN&f&4?kdCs6%6o>-s;Oi7B^B!zAR!kZNXHG@P5AB9E0mqU&C9q z;YDEPEp5%@0y&}eqaugWq(oqQ(2<%idLK$w16$b(i*Dd_nULv8^LNcXupOK~r zxxUsOw*OHS^Zsy;YI$f z@~ZDr@vl_ZbvT|)iWSDqzYgC{{{ZaqUH<^IuZ4YAuWMGChLx&Bw&vT*ed=09Wn#>7 zJ<9|C0AW-oP-xnZN=FPA*D>5QZb#YxARap$_9OYzqq4x+Zt`5xe$aOsP27-PSlMb; zZGlDLi)O+C>9)4pF}^L-eTr zy?Hg;ECNC!gy(1{miPJmsxKsrvit;E-&?>ev3dC7Y9B1LLN_73Q zzM=NupnahsyB}Gt5I^eCgI-kqom|uC zN>{7=EpDgRjm!)YNdrCpSgze?Q|XS1=`rE%mw1;`nLg8~O&pj8QUDQ*a6rHu_NmuN z+(_EIa+}7AJ79j%0gG^zaok{yr`J68uRHju;ZCX3-D!QU*y?j!MRhw)Bb9g|jz9-K znd2gXG|SyQ|3BK8tBJ#-$+(@Mkcf;hDdO+nSa1Z$))SOsx8j zy?R~=6=IGSec_21;^@OP_|`+40VHx%hvn2kI-*oi}DII)1{*K>{0eUPe7V zs!c3@Y>d=BusJVkHK0z$jcV&Egg+G+6eI4@j8XxEy%4PdL_1QjHU$vvS{>+xBLiZ& zZt>=irFD+g!KyKiSl_rm56Ypl@D7J3HS`hB5Do(YZTeR{m&-+k*eqgz~%Oe#nzfsBU7B#!wU@!J(Ner?d)V`J8RXW~_r z-ivo<1b{4d(n#fdU@6LvPWT^FRUb+D*OD#G=n6^536YPXsIN!KcdT2@b#{WwObh{6 zea$?X&RMWHUibsK8P6DhlU=sAjM?emy_oU5c3-7IW5YCb(8L?}BrRn=1Ki4&|WOqi@iPVp@u!|1IsBK zg;SA}?eg{c44R=~K*8H1$x?TcxZlsI{vVgAu4AfK+iQO3YtgOlp)X-QF3RO07dMwu z-ALIHDlQR@dyi71_>X@;E563&dmH7N>Of>{;Z)_-bNs8HQr4riEffU|?Tzls#7pVN z^k2->dh=0e=Thk-%W;N8AeJEWnGestJ%IXFRBM+e&vI*W_|bZb?$Ykb+VLmT?=KQE z>@s0#Fsz`Dx;KzUat>G$IH;eDw=i334(+r!em8a>?=>FjOmx=svILgCy17_QUX^2pDpIPYG1 zu#)B-a%p2nc^N|p--7FsG8EvEnsVxnsJgM~o6BFd0dsIXq9Y@6ZvG-T9B1Jl%hyjP zG*fQ~hR>*_{OfOKSs?^&U_Ai93O<}vpGklM6|3l(XOOE zSjcYKPg={>w%O5i`qVikHj6Z1h`+qtL&V;r*B+z&_wQ4$4Z3t{7hh>u7Huv`HcXqc z@UiWmVc-7%6#<6n<5hCIqrgzu`-4Ap`XA4=TK))im7h)PEheM_>9;tJ;r3+5&I6I` z4}Z?9Y2z7Kl(;PGWaCqCJ!*Opa<~d@8z$S!1JZ-ikJ5zG(13bSdQygf&;S||IHRo^ z0ByVgPq>WZkw{;CzI1V>O>G^*C?ReTvNy<;Adjfcb>FXMt}iiRlg`Q8j4=6CJ6Ft2 z4bc3}sgv5NCMw%bU|n8pqZ0DWu7eP6Em+KWrLHYFQQ-VPXlzman z`cdSyyD`ZO;x*jA0Nf7@eF4Tdb_W^nUPS6UZAm=&Ww(|jUCdq17vH~Ze_BZH$Fjod zUZ=N$9k&0`xSEoDl9E48)!Ft0 zTV|A~B#at+Fo{S|h%vz9k@T%xzz*0)Z&OjR_v?H}Bw4xxAmQhKdrIb)8 zXeic&J!wT0(V?KCPUf@)5_dEX40ojmY6YbQ0Bm;@W3>nrY1q&K0a>W&>vv%BHb&*f zM{s=&Z9AHCC?d4CgUXaj@w?keW%EeB86H0f;bU|EEURY@G7_g`3?rBC_;JDY6bo36prCD6xzR2Nu9azn?;y4@>8soX~HMEn1w2`AAuFJS%Kk3a6dYstt8ba)Fj&mB4R!w+e(4zN#dVu?wm$!;*h*@ zIBDNJN=IS;07%Klxg`5kk`fCg`{8}G7n-i0pxVwZoZKT_Oq{6-e&{}_D!J{4NG1`!3wI>rBmV#u-!u`)i4=n4x5~Mvz4BLNiGnyBaw|RkONo^q zSom!l8RfI#WCzTIuH<3tSM=x86-o66K}V)eWNj&52OxRPv-HMs`PZ%QCrRUNj-?N` zKdGwkt!+}~85SgDAd+#;TmJwR9&y1mUU@vfsx8IK#~j5`AO&NFF|{Kez<)A(ns@0- zSdwLS#zFP;uI{PQ1-0AkKDp%bDZ0vbjBN+CHRYNWrHzTT?xKFqw(ziAjt{V_d-hs{ zORnDCC_5v!+ngQ=KBm0w70G7Xl5^|sDpS**B5K_SXFj8)Y6$~nOG6+GlY#hlpS*eO z3ZD2N)YQC5WRFu@^qvoOR;sP=yT(qL>i8kFy`DWES}Hf$&mhAdIYvFf$l&3*0~KkG zYWivLVM;ALGb=GzD>Lm%G#vi4l+woxZ6vZRXrnC5zyLmE*A}^~Y1$^8E}^Pj$7^jE z<>YK+`Y88f=m#devEyfq+K*7&wu7PD&HbG(?;wgU9`ObS>52a8Jd%3?Hsl=g87Zjp zSkvj}bHsn2_F8@Ax1F!e-|U303;fxB-Cy71d@A6 zL&X%08X#@EPB`y(o9M(pzAJo0MM{%U-S7(2dd=q-lylqcIi@mqf%LLqi5kIQQb9`nGgkY zDgDypoB{WU_Z|JRMK)^o(CT`3lOs*AFm3Tl!O^gAQHNunUtIIWW!%1e(%ncKY;pNk z>NyHW1oPjMNv*)K(b<%jnVgWPw_pZ+`(~noaB5L|i)xLLaFWU@t!t(<`+Z7gyVNhQ z3{COzxOu7t*{#RqSW9Wda-zo|@A$Kh-#!Wd020$0^<_3QCb!M#2-*W92)t*a6Z)tc9G<1KX)Rj_?7t}fESKHUtz5Tl-lhEV6(`Rg`BK$z^35hJ8j>v}>Lg8ICHv; z3!f~oW5LHBl`fG(DN;Bi6$t7=kjU}71Yy|w)qO(7PjwdmrWBfl8Oe0Hclr5w=6 z*mfY~eSURWO~-P6)Y0Wz9+RF)l(a0XjFQTG{JrXVqG`g$*;NtIRCXsBKBWBW#ntV^ ztjr()`|ZUr&1OM7as6psxdjA<>oUYM4l7qv1Vis#Ga!aM3TC-t&T@G5ret;r7lq97 zc!8our?nDmkSG;-C$V^kRf|qaCOLNCst5 z^QVHp4iD)~x^m`leTHew;P&*YdY6}i#VBRfEmeW;2|e@M{HhPs`eNKG`J8}0A69Rd z>sB@H3XsHL_XeXHy}si6HU>EZe_%Tk{{U}V&uen*UdNd{N$B`UmPp}<9$Vo6a(0n~6fSw240oa( zs6`O%Qjr9-s)|WzWTyaBN+@W@YHI^xh<2?C8;Rn!x0dE+*%U}*cE@D{B>sY$z-a3} zqtp6hL6=j$nkD=YM)Cynr9X86`TP0|nyb7n>Y26OcUu~T#4mjEiHvDGI>s@Lh!_G8 zV~#ydR{kmVEx%s%46@CAr5h*{Y!-&(O)5lJaA1lC9OEYqp2H(<4-Z92XVYC-r0a=x z%@lHNxQ;de$sC^m$UdsXa87tXTF<)d%L$jNqvg+&(RXHOvE{aVlZtA)yJDqew`n=# zrAcRm6d54k`gW%2pb8ELAb-AUHYy`iwCr&lMlq3rjA!2i)AOkQt(mR-tT6x(+fGLx zgnYey>S3s+_EkfceC_lD=kxmdQx2`BGR9UJLZ~NlY|eeYoS&(v@JN|W^;VSIo6Chs zIm>z<{QFgZb7Du?q4R zQhlmQqnD9&cBsWi+@JJn>)PGqT6LN)#CNKkZ9qQ}&mS_!IPN_`^#_LjB3<;fcAY(Q zeS6hT&xvs5-Vhg*haj|scRPq4pnz-5x-Oz1)6&;QogzjAKbQisJRB3A{fPEBBNb+N zIn%x&b#(1FPM_?ZAigHlq>su%Lu7@Xw>*==}}s zMbqD1Tuf%UxpgXJbLkQLMPn#x5!@ha!dwO=HI!8}88kAZ*P_xM(@}iL!_mMdOF$ccWjz_SmXGdLqkE_1R ztD%Qbv$)N|3}Q&Z_>_a+X~Tj!!S<~A;#m5grPDe*dS`s|+|tTeSxevb*zMQ!X0)-m zYk0tl(#K#U@dGTXqyGSjst2vQl&mmmY!-N}HyCZf_*zCeX(Yx8&TtpL0L}*o)g4bI zwZ~0$6!wycW3e0$;lS($F~@3n zi%_|=)nL7TdG+-(j|!NfYTyrX2ZyyVdEGsP6z4w(+$+7>h|XR zvlzom_IHev@h{@X1CR*z!LDiUWV+BTe8}I;x1CmEy|J+DVUkV>&VH3zT{=)vZE|f@ z)?s6ki{mTwI&H6^hG?0H2rwN$AbsA#oUiX~+zkCDOPynUD5??lk7g8r{z*tq|Am(sJ`9M^Qr1E#E&SqRUM&Fq?x(r<$!qn zsyD1&wD&HCUE5;`yBkS38PDbi=qP8pmg4cQE*ubpaVj>b#|oeh!~H3)V&$3e#FLV7 zn$)jL2tABDb(Y!72T`~av}mV@*BJu4bZeO{bvtW~#Q-~B1cQP;RN5DOoDYN?W8S+W zRw@Syp!N%#{{SpgW}=geWR=3%$x1QFVd$0?w|94NTirE64R6aHY7cpu^( zt>K?jt@fO;+-dhjMR#o>V3I}-NCb9}a#e^vqp>yb{+rfx?x4`D^{pc5F05}N62UTd zWj`-MIRqX-1Xs;SVn)*ZhRJ6|U954riOJ{lC-Sdfe#G}aUx&B4Q|l96yi<#nwvr8> zW=RVu!)M|QQ$20q) zy{4SYF2nx->Peq2Kg{RkYRWO2OOnR*sOMJo$4Ydu39svRmU5Amw=cS8`bN2L%z!J* zJ$2)!T=f)^L#N!wpy|z!BC=!Vy8PuhVt?rhKDAw0toK)KAbI3>KN|z_NZIF+$>-nt zRfuWlC+tFtrEUu!L}MPEWrhKla6X_rtrFF*QLzLFeAt)J9^Xod&#qbTWVu(3y*H2d{>5GTzMCD& zkW{fhJX8m%Iz}spV6hF&^7a1!O2U+yHzJcoe|M;9?1%S&oMxjMrQ`~y%fUF~=Tm)C z!|jS%7iiQjPDg5o>m4C=17)4Z9AxJoT1701;)sT$d>oVq*VdcVI~!uJw(m%qgq~@fC<5?o#J`a(}lLR zQm{gwaWnUuAH>{b`rvz3`Y)-qj*#kW9-+}LjlK1hzGO}4eHmDKs|Y01rwN-ifG901Ht2(GqBV=}?Fo z%Gs=?6yi}zAfZvsD1w0yY1)-hK$N0)G|*u!C_B(4D4w(m5{fkxDcsVD*w6u}qftV% zYA6F^y(l|Sj?_e|sb*lR21zFX4|0740;8(O0I@aHWKc7l`cxOC`sYs9dY4nvbkE+} z*eq(dSvx0kNPVN#SK%AIdB+$vFmf`db2fVrc=^>GQfhc*iX^2Rp)1yB8-s;K8?)|d zp6)2bZr*8Zj1p_M`rb(tl#r_*z4ICW08i49Xcjm(O(Bqx$HT@xpYOd?#^s#sKzg1e zP36j`pgVK%9{lMi0Mnj@;GBuEB3QI^cYKV5lp|>~m0!Ye9`A z^KI2*WMD_dyN=lJkT5;-O2-@)9w^%QcdBSQcC2C=PNOqLdnVtC-*yCZ4_&Hq7Rx3pdAI{4(Q31_p3fQSeguUrhq{n(yXaI-eqFNR0L_1Sh z20K$vRBPIIPUzC=8k@L{aH|?7=atU^6@5wl2Xlcz)ZI&^^z<+r`B|=R$s(w3sOvj@U&Um%)lsCowYHKQYimoU2?$^dC_F5#*a}Vua4~>QRn%Oo zgJhLg-z18=#7>F5S*|t3wQC(cq?Tyx6-C5$i0lAW__9zQ_#+#K0C7;c+fviwdxLN< zE#5Pj1ci%6yf!y-3S%WtY!6Dj^;xWM?eE2`@!dv?JP#c52wE9HDyL}}O`zZrgWL_= z0aEI^KBH;VR{FrSyta+4l%(=TNMP%?8;@{EU^wMY4sbD%UGkgxjaN#7$~|-8?@-zG z7ff5-THVEVV~rANBPthuz{7xWJ`xT+Gf#3ZK*=1Q5BgV{{{UpoLi1 z#$4gunRh$m&|t^U{p-?+FqSoVoHn1NNDt7_w zJ=f{lmEesc5v*vCZIG_=e(SrQ=eK`fudP6Jy(%V) zzZm@ezP*U1L7>5RF6fguLx)la>QDO=m?o-t-1Aq8cAwe$5s6D7xshN4J@d)q=nX=8 zdM~m0lABoO3hJfE`|R)w)sUzS1vE+*c`!TZm#%BR>vfxr~AGP*Wgs*x;PTygS$WucUg4Yu=o_ zYfGsiToBob$iWEcav4a-!TB76%|)fLMlH30wo@w_!Sr}>REO<(eY2)C4KFwsI;Wi{ z{Sd_Lf0-2_@Jq!je!tUAz4fx`n(ex7xV2Z@k`Be=KC6;A?lau?s3+}a?RC`sE2XLX z@myqq*gM3e{(y?JWBDEnT@Hypqg-UTdHpvj%@*cuJ4EUYWnXR75XEg94hppFcb~4% z!}G465UWN-%aY_sRxBeRk7x6`#V6sk_DtY7jF20PcK$BT-#5 zQ=3$=4R01KV&IKQJlNp?dh~`t|Ql>XF4|s@pj?wo5OXq+ovb z2mEL~bMY_Y?Yg((Jn1y6ByuE?+(a-(ZNnV>dsL3j*ky8zsr03(uE_JVYv|VLeHrI} z3*d(PPm&i>+fd5@X!nM6IV-ohp4sGPoqQ%9 zMb5owv9yNX`TV3+$?w4V3?JoBeMf2L&u^$(Aww!XvjxHcs@NR=01;Ol;QiB++Z`-x zR?`OfYai(Hb*PmZyCKFJYy6d*+Iu#n(MYi)x`vB(P?l~EeFhFe_NJX-Ev+v_t(vhA zg0e|2Gb8$(c2EZc*SR%tod7|m+iRlft>+Iafgx7{Rpm)vRVj`RJ|$8!lZX}Wn|E`l zop$aeaVXsigekp%+Y*fAV+ap(!ZuDwtawu>&PyB}Jhd>DQI>h%@*!6??2fj?C~@&f z3H)B)m2%95BRM!e-om95>5@lp`{9|s_U2KNNF4D^zu9|-#>5an;CHLenL;_iO07(8 zvM`I~rTXkOxl1S#KP_J6lslH?Q$2<~PxP#c<4~8Cypcz5-QjkY?@;nYb38~eAPuK05=&K^d0!f;;GcM{mM2qm%i-U)ao@A z_eQL!#p7?MxXC-*#)d<-!HPVJlm6@4ZA{Ao`yLTM(xcd56boQ*O#jq5s zdixK0Z!#FP%&5s}7X?2hUvplS z{fhN99c`{yOFrw1s}DOF;guaoAa>(>ex|&wXC}Y^Bp-UUKWDD9v*{la{{Xiycf1Lt z-A1v7Bh8Sg^3&?gfuBs~tHWo6YijX5G}}D``(XHoZ>DtM=(`)hFLeMPP)MAIOJkj= zNbTkk+rH3!#d%iFR%Ac|#~S^}m3E>Ja0eC3)>`JF)V*t|>Ux~!=IZX}&7HyE{3APk zly(Eq0qtEDOT6S~Xut;?X0~*&QAY`?pK~1)G{7*<^S~#Q^TGG`r|mw{cUN zvWrc&mPl>V&fH1mxVJ3ELC)VcLQ689I47L0agkn<@ST!Ah|?lnfUF=8r;g^cb+)LQVnlRvu=eKX2a+PM0wy^;BKTr3H!Di%6 zeL@IjiAKOZjw%z>wwCO;jYAR-aoVQ$0EmX;jO6`4-fA(AYq*mKmj0a8)UwLbcv&5^ z$jRDv6>vbxpYlISpZI6h{X^0kvAEFn#?ySm`{>zsTg%&tBRn7EeT_w};Mo*y+b`|% zu9+gt3d^_^AO;7KpYr)t^-d9(@5?=N(LOiZ&wKVxvWmk_f?}fIRJ3L`s>cIl;fXPG z+Dz>^!P-SzCzo z()~etw)Tet!LA!0W@6a#OCHgZM%?|FgM1p*3Gekwstj`VaRwHWO{1hlG9Af|4i>a8C_EgjU7%{qK?P# z^=oMj&ZiLg_9PL{Cf?a3fKfmL7|0}&2fa=GHJ(-q&s$DfQptPj{X?N>RhxpmA*d2o_D&XL-YE67W`vwTiTB;~SOob$z9qeY3QBRf;n zd*I^-f@9%5oyE1K{oSDul!!#Ku_Om#&JKHj2>Ir&T@vnVt4QuqgQUu=OB`c3J9FPZ zFHfykUKVJ>O6co5d06~I&tY;-)_WuB9 z=}P^m#}Aqqxfo)g^YpBYn>ZUa3wYHXgeP_e?%x-o;PLv@XI8V^7^dP z0AEU$KB)weuvZe1mB1Jwho{#R#jM`LZXeHvMUG5jXuv6tK)`=q57v-3JiYr(X%2Ob zIzWq9pEXR!LU!^ z%AUmIXHQ#|4okt2sI^ygV=cS0gVc4fIi#&;hUNsgx0*SjkYyx|!z&Mw92!|Eg@Ayi zM`}$4C_7LFz@{Fw>We;}vqdiyx|PWZXKK%bD;5irj&_21AaPGOQiv_3oTIpn3vEG` zY!1MCp2O!}d+MELPOj4KwZVHihBk_08W7EAwM#LDPBQUjRh6(LCA~t_7fq#z@;(tw z@4*b$y0=r-JYTdh-93czI5$^BiA-2t5-j84P{i%v6WkIBDr=fM=-qo^sr1XCnz>CH=e!W}~1 z+gXiRSzcM*`GaJKa-quuoa2z=EIZ^3WE#(wTrl~3vrA7(a+6>ER#F(XyIWa$qW4;d zW1x>W1jb|&2Kb&wRu~{06M#n?bInD3T+}r}hQ{XMj4|VEZ-FF{oDfJ+j02K8cjGkm z(mIclSsc7k+=5O9L?;Y+1A~Lxjw^@6>m|3*tzeP};CRy!uO>!}u5wRof)4|nHVGse zp|mfIUgnJGYMD=svyZtR@O`4w^{$`QX4Y-3%&gYmWOjFz!-AOLVHYfx%N;hoNd(+)8wz6i;tu9KLktVuoGJqa!~NDsWB! z+Iaw1qb;u^)wFwUP9wS-m{MftJJHk}AIzGcsV7{cWM3$`@al%55~kIZo_|sP!>jKv zV=ZeB+M~!;QN)nGFji$hb0qLaN7A`JO`O_S_WyrX<)b(rinluddzX?@X*Y8WPoaFMLoO+Nd zYVNXc4qFK>kEYWvQW48u{4c2)o;$;3HlcNH&@;R3Los8D|v3FjW9pIVXla}U_9 zlITXR_O_2c#`R;hRE+VET>eA2rcGjNiR_lnNpcb}i4Z4fX%1UpJT7ydJDzFdt~C_t zE`?as0eya=ayR|h@KP*e*OzR1hFa*O4%*aDH@@d(@|$_fWIpyub!~P+thW}rKrJFk z7Uqz<5=R~Iag6ruieBFCTW+F~E+LFYfbDWoi2x8d;Pc;*Hu~nH{u?B6TWWVxNXd3W zv<^dHHtetjak$||52jD0Jlb46*ICaCN+3f&o3)rR8yDfp^dpdQ-z0PEUT|(-Ocp4& zr!91bm2L0<@{PDO{cR*kMu(^LTYBWLH z)a)Za(yPWzMW%Kg2SXdRaj|h9^Wc_g_S?J)*2`yi+9GfdVh0%?tySGWc=5$7NgO_B zaPpjO$2@0@)y9kVPXn(bZQ#iyWU1|({OXHF7T2=|8BqI?zB&Y_`c+Iz7wTMsOQ?z|fEu8DOd zP{$l>RyWwkpTws>=6@qsM}eCBcHLw4M{F$97GF$~eX3nGow7)GQPM3&hEwIGmHH7+ z+FHh2Zxq&BDP)4~Hxk1%XKN!IatQ6(ob-OAyuRtniyclSj^;332wW>|;IjK?)Os9x zV+wxKpf`&l2k^#DeR!vv-8vum2T^k#ZefWY79V@h>;`>~SP|-^)1|zmzUFm0ctcGK z{K+i+c3-&z@E`2g9YxeqOE$$fJGXu;U=BWi>sE%N($ZYTFWBaQ2%RN%E1+y87=GrMjYm zP3n4ut;Ox7t4RfvUVY2t1y%VUQ|p1vN-k|RYpXrloar>LaK2iw18M`3Ngs#Tm*dJgkrmhiNPc_2hQ>4{A=Os`M+uS#NLqAMoamdaXoWeptzG`Rs}_ zq9QSMY_jYRBug=lGx(112R@wRnz*`UAOhn}NnUI4g7)RHjoHZDPo_vXUqw)TMOGaW zlYhm*L~1RPXFIbV;p#mZ$TgeOLax z*U;{?oBsf*+(lZNckamU-*lKJ{g8$MGq^St&49)}~IOu^ z1>Me_41R6YX=^c!qG@)-V`VO~u=T(Vj^A~A@m4SF>D6}==o@cczQ(e7c8UPb+~5zH zJfHkkch$luv~6zg)_$j|1dWmoKgQHqYz8GZlB_n`a953aPe>$wz6OKhX-CHwVOEj#k z#zGj6R`opl(q*{Td>iQf}Q1e52!iKE7l3+ zX;I){w1IKAw?FUot3#!{A=Y*7h=W>^^50L6ctNmrwcLBSz@;+{D{{U2X&8!KJjj(c9k}#ouEaU->YTNLN^P>75 zJN<91X4%xbb@N-ToXRGeHuCoSnBMMVk6LoFbh5q@S(4b?U3_M(FgbzfD* zttiB^WA5Fa)TG*#IXNdJ&>U5=xSeUc(KQFXGy$xwn#ynpprr?T1*H~|h=Nf{55+*8 z%_uw43KI6CS`aBj2Rs5#%78d}i&wPJGz-06_i={ZGqF#_Ba?(h`Lhr{T%Ss#HBDY! zYf-wnzet7D@&;iWd;w&M0^lFQtr+J&&ot}e?@*m{UOHW*gz4H_l-!3RYo5;EFub7o z?>@Bv)MhbU#fFYGxVCg4WIM9A!-Lz@cl!E)T2yKMIL61$e`6N?SF1VJxWoNNwBHGi zRrpLQHh{TOJAQxHBjsEN$xTKWrMM&%Nj4Gif{bvd)42p^jDzkgvfl7Egl?Du%OAp| z>kjmlly^xOJ=K%LVI_=eUyN;dHP3-ovUEBOoT`idMr6H3F z7j6LxbC5{;`2-P)i)p=e;d~d9F>P9VmGc;zh;pP6hv2aXjyV_@^{SH8_CK9rm+H{F z7s~R5Vet>Ex*o?(*0qbtZgh=S;$JUKorsVmK~-6E_W@D>QNaVBy;iSDb=IHNmiD%~ zebVOOD#0v6nC0xrBA#}G$xwR`bAety>aMxhdWT(#_DhYcZ8OPep)g`bIS770agSbo zO-npH>WiL})hs&8Ooj<;WRRuZq?p=|*|0gs2fjEB>}wj^VPvr-`JEP@O(;h(mmIIB zx$Ju=HF0>2bYfT&;hH53t;xXS7#@I*F`Dnay(~PGCD69e>Odkv$RCggkM$K8=|2)Q zp1AUtL1okRRR|-txR@f`4X6^G7;i_EfX>%M>=@Cgj;HfEJgfZlReMSNA@}zYw8ch$SH7=sm?%{&w z7@8>LcVq}j$r&D?6Orqj4Dm%vF9T-&T;z%9JwaupzOkxma9v9z%BgP@b9`40`P%?L zhXC*0&d?h*>mIXEP-HPNo?m$v-_ zd(^Pqt&CGML8J$K%ONg2$u{~s9oRmB2+jiLq7TU#Be(@5%Ti)iA2NVd|%v%x3?1GyWy#@;~(0=*~U-;Z|p zXtsTCV7hg{JAbrnk)*Vfj_kb256>Z&<}1dY9>`Owt<*N(GC2_E*CV&`t9(Kb#A6s< zLS$fYGsmyXKRi^LZ9FZjFJ?ok)lQpmmQ(P1X2Ry$_R=f6JBV&==1}oX3x`P*_A0}I zK9$mCC*`B#@f`9%=L8elvaOnSW7yQD z7=}w9N}3z)t+dsyo?{tRnna1QF{bb+C=uU%(0N$u+NsA0aC$R1co4m7{_(?s~g0PZqcEP zX%<&5C5a8T9eEHzJds5E5WlS>!zj7M*zzC8$+Lp4G=Get`bsMyDLXtwMDoDU#y>MA_25`(f9%mj1YdXE=t<-Sq zvd-623wyMA;PQj6-ziyep-SNp1_J~EPUPjHUHQ50tlKM1RyGOl<(HKjV=Om3J}xYsW2VzZVrx&YBYF2Cru#EqBlBQfw0{oTkRAlAOaA~8ZS@^O z{!7bCyid6jbQx4Y+yIS+#?kf3?OZoqO=Z-&W}~fj-l2JYugT_*I$3ZdLch9LB43B^ z_sAn|e(%CeA`+MkbI9a+*1nrT3b&T*=C!{O>MmWd*<%mSJpGG*qBQ6_hp6q|OO^i1 zvWQ<_T_-B3&OU#|^BC>Hs{a7Tje2c!ttLhV5g1}E;E2XP1Q_GtK%$*IGU{c~Bi3xL zBuK1o;9H3|Ha9#6DuV!@k9INgA3B0GcUrB@-m5Rh?$0Z)d}ovS59ljN&pWf4j5fk8 z9}F#}Uk$ZuumTvQOyDC3*cBU!{q92yZSFzsgHc@)N1v(hr-&Gs%)@$&n9l6v1N8i= zbKulXKSp&Qk=;D4l9pJU29R*M8?r$7w(-FC$cS`Dn*?`PHtQ2g?Va+0xkmosNyc;D zn5uJ=iX9u3NlISg8uw9MYyD?yYKa}arG%SehCURs{r4pD%vg6k@lu@{-K4yUa6xw5 zPp~y!bb__kwR3kP7A-DWvCdhEKhV{;;glb3>J?cOv~4P&$IXWypfyU=yOdGtKL|m` zO)ID6xB3$=6V^*V4;Il3Ox|p3@V*(>2a>+E=UrXQH)qU3gz$c1y$RyXR<;^khy=`E zgc3)L0qK+NUUAiK8sXgzLV|F#$*j!K>Q*ReRFmDd*2R&aMJ@jTL)lv@tL)!#Y(yJQ z{J51_$@qbeAVxR^6bx?Xv!e9|)B4Lvx-0_yq9RUlk&r2q!^^q9ur1TejiOnaHN%A^ z!v`4yEcrO#jASR$o^_2f<_kpEH!DVNR6(EJy~LpAQ5t$GfEeciLn`ukrFF@*D58f) z>hP+oDAY+MjHcce9?fs{CH^})P2(>ulvGAz>43SXU25i8dP3&WS#70Dc6WJ3#8Om} zT>1rEp2Xzf3WWI0uJ7>*Te}jUzqO730DOcn$_^BqVDQ;Km^}NXadUBh;Y=2jYH=IO z*E3v;!d0CanTR9;%t=_rIOE&a(x(`d-W0R?f7zX^_=Yv<-k(ngDMc;(W$pVM9t>N- zZ3MFYp=3jFcFLJh$haGTJb-!N;MMJ8EEjhq+PIR|B{8sG#(-H;mxp8P6FWxU18{yE*OpEiW#1;^u!e5y2`)A5Ox%9VDSDA=xb8 z`QxfnY&Et{*yFn+Y?=~7*HY@gXrIixwh*i!{n7>rBZdFq0e%Bucsdt=^) zZbJk2+usD|*XLXs$CGh&Z8OHi#@RyBU}W*0IQ?ol*IJ(VE{w7yMB1J4DwWzpZ3i8* zim@7%OEFkaE&a-mRXtRcWh}IkUuCflz3L^u)^0Gkw@=-v?|=x&`EkcJC9$-C=$99_ z7a!bPGKi#;YY8L`f&lJte>{q4@YhDPxVIN_K@^PbxJ4l&Xa&6o2Oas(rAc~`kP%@V z5yc!n$Bx+c&&!}(tXZA+%3sr~w&>d@$Ahe{-9@^WL}Hv| z`SwlkvDK``EiLrTJy<(MW+s#W0Om>YAI}>FM71J5(4(+a=Z}1!^QjK4a&C1y%Z!{B z*uOwKe=3V=54z=#Joe3L!5aNTwh>T|3E_Q^v}gBwPXzPYis?#2f-(E7^YiADwNo_f z9DI|2KDFJ|;b7G-AY286hzA|^;ZOen<5JfCbZ2VQ{+3evAbLLat*(md7}1&NushM<)n(TiCX5gM&qu zC@n2*r1MILX&Pr@L2z2TwX9HwOmL~(ga5L|WRF3PT zZFO6jt|Ps_)6f{ByVF#G_7YjSWB15>9jtNKh9HcTI)kMr=}UWdk59X~YuJv-@wzl~ z@hNs3kOGF=o^VESnz5$U)j0gWp1+^6O8W1u6=8QJzgdW zZ*v9ABm%1|+p_Lqk6zj2eznWiSAW`fP+HpFTtR4AF>w^GJmPpeaBv1YkTJ#vPDtSz zJ-zQj{f;AZJoC+Q9khmLX=FcWD-!R{;gG7QCxtouJj>k8uU1gu6U{bg?d+}^;$)5_ zjhx8G1ppaurUp$@o-&(@9)yZvZt$h;!lZlKWRfOXjDTYy2n)%2{}4d%5J2SGe*d4FklW zDl+nL4oNGLLC=0F8Ciz#RMzTy>XM4aQ9^L(HSZO=P#xaYnrh1b8`KDF6#o$hck zoGI>cNHUy&4=3E!Qw4A&;KyXr4(@O{{D=9}^P((78hjZ96C#s?+m3(Ap>Be0O5W6g zwmdt5$7;58J5MIw1d&NpQrofLj^FA2m7}1CGBUc2m`WorU)!+h@h8K*9S?k9mcZxp z6#oFxoqw#$)AMv}`o5uIp-RaNUt)=5xkq5D<(UuMg`B$(GDhMD1d2yq8qSreX}XM% z!)t9Vvs_1WdjV*QJ2XmgKM^YX0Qimp7zZLb)Owt*_gA*b4Xlslt;25p^JEdd!u%d% z9PT_|WkAN_)%DYi=iAN;Jvy0JUw6Ou8?_a-(t46{qTN5)?I&-t&%Ic(@&P+QQV7qW z$RmnuU}u&_UVbCGM(3ZIJ-WU0Gf(4>}W@NQ3W(aY*LFr2Q;dR1qnq)JX7js>3)!w;??%*S7h7S zW4gH+X)*o<$^QVTsP(Br*PeW5*T1&9{#$5Yb*7XJoGcH*WCt@p{`?Q5VytOt^=wJ- zw`EOj{*6tpUdWDU_tvbdVV^E&c-qJBf3Mf;No!742vh>tY&sx2omk;lBi(`%>Bl?@ z*0rQArDT^jk{EAb9}9uW`uvSErlT&QXFZ+tNScMPCTLiYuB2mlJ<5ZO{{TEz)Z>`A zob?apQvH81`$m)dS3{Fcwk7UXU|gu+jlUBNcflMVZ>>sn+j!$Zk*F&eBW~_S0rdI$ z{VE^RlUeBcjK=jE&hfTKEb7}cxih(yvl|TO8+pcg&jM4Y>ap4~-djm!sKU{<*rbK6 zN5#nPIbFvg=8DD`ra(422deg6h*AWdW zd#QJb?OFWAiZEG_kBLC;e)kydjGjAJoi(=We%{;`=+#*j9YS7l&T=($I^Oi>SjDHH znN?*00pzX-#AJ_C^7m8gUYO7`0Q0;p}_g7?COO#RYr&kHq4I=0I9PSjafSGJA_8CmkiL=24J zXFEvUY_O2j;aJJ!perEypayh{5 zz`z}jGJR^S@jF$w@iVTENa^?TTXd#?gY7ayhkMESAgYez;XM0ffyk%77<@vw)B0_? zTcbpQB_`f|Nt6S0-JMq-gmx|L0332@J!)6!UkbG)ws$wW#o}4Dz{;^*yXL|4;05E+ z&%IfzOyeI?NWr4`M#)hLx0d|p57~VY9`U;SP1O3z%RevPV@v-4np=YFwj`2Gpmqxzoc{nUew5ke;TPyC3tMu_@@`QT7-4uFIv*#h zEI>`fx#Kv`!fN=_ZKGxg6z~M>c9CCURU`Num!HJ6~GaN+!qHO{eEJ(JR^VGDb2VF) z9l=6W4mSYOHhcEigX(^jKB|53P+m&98oMU!mffiIZ|zyB+TTIbCeC(|Zf)A;+2R>2 z7_P@3mo-#$=SD?-tq!rM$8VJ`5(VB}fDFKL25@jk%;UE%ml~v+u8E;SvC8SALp%#E z+!Tz;j+y7MSpws>0VlO7)c#e5g{Vt2JhNI$7unj{LFF`yj5-o{P#c_j;BlJdwMuch z!5zkiF}F?Xcw3D+Mv>F2)Ymo|WV6j}X>l1wo&2XekSo6MJFmrn2cXU|R=%TUsq1=$ zjHcS&C{CL#vs|Ia+O}=mD<($R#~42r32bqS^X{G1n3C#ej@s(>>5;DG_|Szp4&}RK zhQI(E5Dz)UThG*RL(#fidTsfWSdljg;Y>$qW&tJ^&j+&-y}0f^5M3Il9H+Ti+X-XA z67e5Zj>^&WO)2-x>Qp!0O`wv`o<=!rlY_elo@z^V(~DuG>Gl@!8Kg@mm*dNs1~$sg zy8!qw|Z0{<|DyRZD!73wM_S(efC)JN(R*Y(TgH6#FMYO;E z(u&gVeMEr4S|weis33-7#9?~|_(|h5VZ~w~u-;h-u<5>#zUl6v>RzDM;jzBb=Cy{} zNoGd1k(F8YNSjN!d=()^R5&1armqw{f3xa4tuS=%mE}yf@<#LO69rh>ZH!moVoIja z2Ox3V(yCvz>w3ppjb7r)4MOHQ)>&Y?j4TSORlKJe_*VcfGn1SQlgyK^HGMZ+)EWN( zvuvimg-cC$GmCgacyyNXkoZMNVgCL4LZlWYQl1HA;CLO2fa4ydd8!pbmhNJ+3HU^!i1*yR2UKR z#Yro>1hHj}kHfgJ86XDvXBu>66`k&60gs)_=eo%#?YMEaFS?u-$;C0CLCrKVC<#kQY5xW%|kw>}QenFD5DX95l<4omTt;fo_{K|gj zMhoR5H<>Wls=eFa@~bn$T?ZOO(};Y_mpfqrfFZpfJ%$e;_Q)!GimiOh33|cW7?|nH zaCV@O^B6epdCA6pr_!xnn!K~J(k`^oKD#l(pQ<+TSWvN$ImpIWW1ptsa@X{;rVMUlfh&$JbZL}zdPAigrku>9$e`K-ea zh~0X95^!8>zcJ^aTFhvTWY;%w-i3RFMU8x=2m=ydcHmnJ*a7g2=DIbfdF|}A4}wVc z63|N^ZS2h7c<>cF9G`Xj#AG$u*lHRLm9C+v>34Hk>QM!XUoGQ`Ron^K#|8OJ5;pPg zQrn$NMb@;fF2Zh|nhj3yD$S|xAX^yRfMZn{0|O+nB!i6itFfP_Cndq=HqJNE7`(Zx z{+Y@{we_D%-)Q<@R%&;)^4v(R7+4Y76i;8I?!EP8m;cikbS}2$Iqn+S`1- zenFET2KmzFNVbf~qC0sN&-iyT zt6+S`>w;_4*V9EFooHh$6lc+72i0^HrQsP3d3#mKT z(;lBHs=DIhR-!6MTcC9{7IyHHAC|FWXvo5kiL1WU zz1@bXsuc;gGQ@4h3CZK~{{SkCbfjiIECIC1DhQNqJOlTRf0yQJV|54Ed_UHv+~YRC z7dcWu{{XnGt19&9oR_iEX?4-FsL@kfxv%g407VzlEWc*Bc4bY{G2F)ierM%Qm)EU# zszoHW_RyH%!22|X<81aLB;)B#{TXWo#+=dJ89bLQyUQ7V{g*zqiRC~iXj(%a>?v|L zet6?Ku2H`StMr;9EpMv%S^W+xhYRKs#DYmA4{qk8w=Ujuk}yg9vroE!Xsrydkma~W zQa3L-Vd;#Gr|VJ4n4Bja^HaocQPg)?+Pr^uNP0CCY8LZI3$L7|fbKvbpVt`t={~rP zpuD+OaPQ{)Ob$U~$-(vHn(gT*m|emnP|utyvX%!wA>Zjqc%YF*uWB~XoTL{%P(I~G z)BM4rxAT{Elqht)`N|vmAL721ah??O@+PhA5t`BqC5`aXO0Mm{V?(rt&PF!lj(rI~ z!YZEVcLP`Q$_L!Nk8TgGTzYdu_Q>VBw`+x26zG|<6>#ymj3`sbq1HK#Fz7^8oio2U0d6uL@%U~Kn!{12p7Ag5LEyL8xH0eA8|qk z$0xl<;<|$C?N_>o+g=-BWoaG1d2qm=F`VPOV*n3?000=O8XW|5w#siRJ~QLl7d1M$ zYHN*YrzE$>k;lVLHs0s1?(el-9{&JPw&~p|nbtUxRF36{b0aGA$|XmE@BgfXu~@^CVRpws=NPy+$b|rw z5X=~kdv*sMhkRAYmj)PQvbcvzxk=VfvKv`jEwyr4pEy-q9ofb^5)Mk$-WzFhBCe%0 zGdeVeJ7{Bll3dA-$bABmH$UMA+={Wa=&h`pGYqM5YjX@TdBQBl<%!yxGlCJ7!a@lI zp_W2H8J+4KRrbY|zoU)zcS#ca+g9t31+I;~!H-tyiA?6>`r0;!8he$1;SrMXcV$nyZ)Mn*_el~PBk+TL;Ukl0N*2=XDpMq%9G9G`!cP>%Z- z`+GKEb#3*Aw>^}4J*c<0wX_k)@*<&Jv0_-SsNqLBI3xfKG_Iei-d$Q~+8wpc#ogqK z7>YRaG`-IN@Du<}Gr;7U4HE7f+vK-_NVSOWB(4mt{6v3G%A_4N)K?ud)v2q;Y?oyt zlHH>xZs_EUtKqrY+?*U9-o~#(N!?SM-`Nx=tWdQ=JA1L;tvUmyG(8SGx7DwcQ(Fj4 z)uJLv6n7#q=3-KE%3HPv89b4SnEu3?qQlhvNud_NyRnM#7d&o?O|Aa`#!>oJN!5Do z&bcgsq5Cbq@XHWw%uirg;Ai9j#XWw{dfYuP;|_y=b0CV^=4P;)nuKQpKfO+B=LqLoA6j05!fk5A=IePO5DYF3MNY~tSLJozp- zD8L+p?5EXzd8hdHU`FREN5zwj{XgFn$FEz;CICXl5=e}w&d^6@_WpHhSZJeLah6cw zr%53ek8*9{nIKn@Bf~}KGT`J6Kvmy>ay>_SF>PUZuP}gG%`Qq$ZhW_3Mlta1!OngC z2N|wAiH@spCFf5{T0j;ns|lh8T;y`2(2<;D;cnpf0$1TrO5Sy5pkTH_BDDfpqqvl! zDD2A1f>5y-<9DxWq@`40nQmUPyJccfZDfJ#;`d8P?k*9p?G*F-yiBJ69l-*hepGJw zPD2i9r|nI0qFDAylWY+c#rT9_3oRf$Zc-zgpgfYyy{L9k^_3+ z5J4Va>@S1Rv7+(vx3{ z&V2s>*h5YK0E(|<(k+lYkj3XqAe1ao9G**)!()YBNzZI{oAmXa$4+UnX;u)V6YoY3 zcM+6Gg-K!>E>1zfV0a+$OkERb&{=BNR$_D^g}iJonG4QWb4h?k<^_T0wmqrePQJNz zmOE?fc7RSIg5LT9Bp(RN7LRGkAy+sfpIp>aa+7m$=i8FhAqYiOr1sx_GDyz*ZP66k(N* zBPRW>o zMHt%h58g`IEZe<7Jol<&sq|e-sXS1;w$trn)FQC8K*=F&4WoM!6mB1T?_dGXJ+nTo zP7z+C<(a((RNHKwNw*cx-`N_v)3o~zgj>j2l5uBtigE~)81P6v@s|8pJdx>DlYlty zine;9&r;L;Jii*AhXsU&5My<@FDB=B!-B2x4jDlwJo@uhIVW{xYGkD}(bs6BD8Z?7 zlfR_iI699{nnnQ@a9ihrSNuAzpY}wQUPm5RAiIuv0Oy`gc>e&^R3}QuJL|TeXN6sZ z9G2`47&*bmaaTR6>px(HO_j8YtF}e+g8`gyGYJ{SGCivHhQ}J-*)#f;D%%!bHaWVt zQV{9WNjYamLOj^;$-fy2et75es{5z)ShURw2bwXwM}**H4DRFVPEY0SUA`#X3p)f? zwu05J%%m7d1xHr4%3&5|98rfX^RFnOo{uu(w1p(<*9e zPiMRP7Po}ybE;}_>DLzt4BC~I!oe;`b;96t*bk39@xbqnI68@v+d}@%N#wYR)TDsn zfjdvW004u6xb>*!x35cY;zfnNjLwOrLLlICxR6+J>%h(_((_k}dxW0gnGAW`wSW?~ zdEjy*Ah5|9=RDRoP;1wJkBYHaoQY?Ht+~J_=>2<98~(ONN4@rY@uIb#=L=_x~Cj= z$3COt^{IzS;_JfgPSaApzlkEaU$EFLKf9HSW?31O5;4fmFs*}wyM&hMG)9&i$li4% zdxyEoH!wlN3taA%%Ngrrv)A2lr#4Kn? z$v=37_ccyNY$nsAeRs?I4Y*K}pmi`34&9@hk1$pXY)c4RV-ZsclvS**2HgH#TT%pPXQa#XfU1ITRhbCM6wn&|3r zqB2TSFzpb0v?K zijs$DnVo@goctM)vb2(*~WeQGI{PUc8VAM~;$9A7;!)0*J9wmzEHFU$3 z3~<8$;9z>!WY;s@*!2|f+1uP&J+G7KK+3S(Sn$@k{jFF_Xv_M9-$!sVN50f0fL z%}>Lwz*JQ&OR1(A8-?$tCH>9kTOJrSBU|&E~qUcPu;?}Bac9%804IK4Cc7Bp$()F ztgscHY>^wwla>P|q&?VxH#zQp72}Fi328OF-&Z%X0A- zvi-*BsZdL~ZvOyeEpe&ErD;-M>I6jrVq-)(F2EhC3Ga@6UbS9nx~99;T~}k%dP*20 zy0}&Hm$}|~B%WCJEKi^~_NQB)wDk?9r?1~3OB)MW4U_;9rG^57`H)Y!!0Zi1d^Ehd zgRbrGEhm;SVIz5ac+bLAll2+>YJQiYQogWh)?&p_pbTJYW^i z-Q$tRG~2Ap9S{lC8tg`qo;Gog>P`vz@yD%4wcTcUBaxtjSmTRwhMe#7M{q|U zT<5iS9Wsg0TUN)nK0KZ?@ea1GqkMm+lar4v@y^>bkY1bZag|U;OB1yh;ar};dw-QZ zb+1MBFIHRG+Gth^){#K2^IS@*4p@Rzk-3IJzyRY1n&i+J+ zK+yjYB}SdSk0>V zadW2xSnd3^aJ-X{P7Zqv=Qyf0^&9n?c}fX(<9GaLQK$HKL8IzbJ0|<{l)dqn?0NG; zhSGf@Vr^m@DNuq^KzSMD{{Yy3D(Sp4++QucvqwCyCOpW|(3|#&O%8J*zpU<@7sE9Wv8G+hX$ZvB%Yo{{YktrWAQNj5g*Wk8=F>CnGe+ z{qD?;H}D291vqQ15=ODAhLza4lR3in0DieYI^ebVrBzT|FKU_N232u#u|{0^`@PEp zr)|HnbOY3PGAd0bQ%r;LyY@2?=!xi0-fRKDCZPR7&9POED5#U)6sSJ0ODX zf<**OOf0TQk01bfEzbb|0GY);bY_=tebiR=Hv{b!amZ$s2*7lU1P{&3f6P}2nA?eA zg@YG(&SPvTM&uu20MF}EzY8rISLzK^u|>6lEjktcCoHo}e^QI-_f0=jW5;XZl5|}% zw^1M8^)5|OJe%DT#?s-u;X5;281he-8Nq*i78J)kmML$lLmMktwb@vd58adz#=n=Y zJx6+Y)>yw+>7GL|5!=pD^3w0PR zk`_e_t)AYXdUMCW6v?R@JIzhMv>{_|vqG{z#(|W|frD)zk+iV<>DE0#EP79;ZR9cS zw72_Aa^Sefmc>+#>_IpH_orp7rZ#QH-rRDC50Cv#8)?T&vvhR7oGWcs;ux?`>g138 zA3CJ=oaAG1xcz<=rfpTC06aTky65G%Qyuk;y&z80WvPf8T0K!$%sXsrH+(FZO&cu%vK*dv`Vg z5XbI$UCqc*&m^cXI!2uLajnGSXzfG!WFY)Cy=9)l0az%sN>1xk}`06p2(LB3vd@$=4JIZFk2$GM8?=(=UJTMT9^m=-0L-9e5uJAkx+ zWrhPRfsRh{3o$BxcySN3^z=sJRE@Ouc8sjJNT86YkgpkH7%p7$#z#3IS1^q(`g!5G z5XEz2A(0R`SmJ_0s>r+?C}oYAvzGYzR^Cj(tW1y3p=!t{LEgArn~JjEsdN_l_~@Gqrs)lg=`l+da*d z&79N2923CaOUUEPMD3jVl6e4j_BE^fsKt%lyUXTHa3oaU9`NHC$Zj_BPb1Q${v&ji zpGf#GX{+@VGsmpz@_BkajAX<1a6ox3Cm)Hn5CV2x<2gCS{{T%?lwxHoXKdW3$AYlw zKlxk95IRY*$PR?-_6RxhTVPF6g;f@FB05VViVB9E=)Q1t7$^6zY;y^VAK0NjQ4vHa3c{x#~>s?Ubqio^R^z;7!)vpFb#X}{WXCO$!)!PIhJaM@3 zfNV&VyQ&Tg5WlY#<~@FUZnW05`@&py2SR@d0)qm!QP))Xzc{FgW@^ZdV; z)}nn=f2pgWvzW&T0J5nWetplU`cuBGIf)A>z&@VX{{VlNe`2Zxt zTdB0UkzFLcvuTqgf-p(~>yyVh$phjf0Zm%u%CcJ6d11WRgDks<}BU(@Axmff`20XLH$=Gw8%)03FnOAhNl%iY9{P zApPm`0|mfvcmx8Z;Cv(kIT-Dp{;{-%;?jK)z<6#Wj@{Bya;7kLvXjFys2>5qZ1>}u z{{W#3@m=5l0BB8eP)H$VmMwsU;I`jG*dG0=NZBabp^dOa>Wr$C+utOEt^`W6!EDf& zu62chJgkARlQG6gWv#sv$*?Byl*wq$+vD)?2S0| z#vZIz_-x}Hfp7Dx`d!qY*ZUk-#M1fehqBM=z+`F7sTd3-RX;9ys2pA^^nR3@m4MUr z?N0XY{%5#IS#4rd=3pJ_Op=fkV|N396myyM^T%UheW=R0j^5T;+dhR5pYj#wPJ*-- zdfl02Su;3!6o98Chpv4&&mOf$szcVCVr~2w{R$^;Ie7gMG3zd*(7KPNFD~>OYpY0M zg3`>}T+8M>d7b1fyc{uR$!znU{+O+|WljmJFT_0-*GK5*k|>pKt}kGYW(ol%L$Lg? zct5XdtWO{1Sa#Rx%eUC?DYaV;_2iNCJPmQJ*`fkmmgfWvWOKM5m!>iASG3Syj*|G4^#0rBT?J7{@KSStZC$<9(&W6kGG{h^bVUAm!o~AM+q$GL}wTvk%RhY`BW3C`keJHzioMWYvjiSXUd2TIxGgmexC>fFB_lnYVAi_*I*>))>JZ`1y)!yVNf3 ztst|SBOI2KiIN%PEQcd&0zG)g)Gw_(^!)M33RzB!w)t5%fJxnq7F9jT7|ACCXs&l( zOsTgm=R5vKQPZ!5)g+cSOCM9RWOnHr4L07!Ew?t}d7WY_wX$2mR1h+C1gjkiABXAtCASZUsJ4In^^Z)6^9wZ0 zP6U};i1%aTImU5}3XE#^7ZY9IJ^MG>)kU@6oEalgwXlQ_iYW7(Mh}@UJGQAghfg-* z%Np25Q;Uzc=nc)y)K?C%%3{2?GDf~jVQoGi-op_*JeeQ7xjZfo(~v8#NscfnjHe!5xC(Z!8;D%-fG2Ca zWK~#SGd0qk7piMI74EPn@)F-N#L*q!Xku{X3v+@A%*T`XaJ!nJeM@y~Vbq#Vsc&a( zaTID4MCrO03PS`V)lVbzHG6cS*EalV77bJV7lkhi?RE zz`ot>w}t0{-{1Z7R5w>aZoN~hK?)TMbuumvcI^@N`S5D;>N$+^X_mPyB(lbF?x2JI zRYmn%dHqYQ$$_-$Hw=Ms%L0UOIKc0Lj>i>uq%{>JVtiX%lCM8fPvNbE<41tnDhDy6 zBTF6>FvDbmFu{~CIri=jY0Ih0>K$)=6P>n}7K;b&&RI9%nbl4`l=}U7z=ZUz=A?A> zu+!$8MqOO6ASn2VLQinHBr!Y=4slK$O)b`&pryOBA-j|8;wRc!B8+m3!1o2f9A_E% zRf;HUQQa;MUfUm|i1J3ic3i5dG!6zhR=4w5qd~s{Q$&IkHb{`JVSGn!S9CsVOL^X7suBWL<{fnd9 z%XfWre7N9@Wu)~#EC4b|;9!%*U7GL6%{09y7tTMyJkj+_)2Kx@gc{m;rc#mGL~|qd z04U|N+aEsF=&pd#HCe4K?zNAxrP0Z8YXoW{82l^-8HeJN?kV7IBBP!Vcy0dx57?vm z(cEA8A>_R|Fae%X4}UKi1%?_{{Td~Nz+Qdg{|hA2P-e#AT&gugJ3Mg zpS%keI3PEkF1lWBmDg^pwF7R~Hl&MffxL(<-HBF0-+RnlF>K`L;v==fu9Gz3U6j@LGs0Eg9AsWL(cEEtW@@I4T?B2nH0M zJ!#e3+Xn{lznd9)4wHVi%+nm>miYUP))!G6!bgs;$!=RUX{u-4h@ z4x@)kf<}i#wUBx8#>za#InE1yeEJNZQBjLAcOInHx`p-AhT$EYk+gAw2eJ4HeowJDj#}D;&K2eTfF8 z(OS~Ww3Zjk7nqUqbWmszov+2xZ8z=sTT z+ar%&MP7H>-PV_*rH#$ZvNfwa8Jy-kvC9wYIQGqSbmM6KNpzBHdb%0ojc%c`Nynbi zjleT{s<0|R#^aSg2sNdk)I&>mbK@sH@g0`+tun4DMX02`w*LTU2AQB}fzO8X;;^Cp5J}}4P3cTcjoNnZi=~84VN0T*l$kOIb-0}2TIStPONwN|}X)ZTG$vuyJQTQUr&&j zR#6V^gphdw$jJw507nL#JS(uf(Yie#)nzi?O(~vKR?49&iVz;hganb?ll7?Qgc^mM zwl~lzf+8*%-I>)@nTamr-*NRP&~_LVGSzyQr!;FxE^l?4DI|>=Pbp_v9zq8S6>@p) z`PUPwx>D|{D{TH}q3Ja7hi#JQ7~6uj@#9(k*S@jny*(CN!6(}8Po=u-AwfQbhTKQ6 zQQd_?y81Y6Iu7>S+nvB&$&ZKZp-CSy2s!+!HmWDou%jO@YPtM1u+;3dEA2-| z)V|L=QSNvllVp=9?g8(C-}0i#VI(u6Xg{jHb~Tj}5wGc5j-NM_W}9O!;FH7UNrUn{ zRC@74Ldzip`;m+vn5llgZr=3AMixO``Hu3fS9K9vu>ZGq%&K^Q+OTA!3p)~StZ zP|JUJoDJH3>wQWI{Ueky{c8UJ(4Hvt!*L@phBaB8lmb8;;15zxN9SI960js5`0rP~ zkt`APot#&DL>D%yk`uu#wNEFw7$={Qt^EYHl1G>INbd&D`&nb26x1&v5v<7fZ^<59 zLvUq0^Mb9$J&&zho(+g2@S{!DXSbG7bnkt44sc|7WG7(B$K7T2&IM3<)vUUAN*y{I zi6^|)?$*&HkOV$kG3)`rTyE#xiQpQ)H7$0>!_N(DBh}(}xPekDTR$@<=0w53CpbVh zlHP!T21uq`W81rww>Lf?)5Fx0hFW}klN~}yB-d=@kQI(ADO49=e6hJeLI*kb_x!3m zZzL&eEHX00TJGSYlFS?Thp{}7-xWY=J|gvYwW-TJ=DTri70Z=-dtcqa5x8XtJAw{5 zB%f1E9VM;xC7!Tg>9)U^msg(NIE;cY8ykGFl_SuT-;a8$R+XD;fe8!dOB(VyILROboSr!O)Ty;gEiw+A)$ZYr6tvTCZL675Ii$CKH*0^@EN;Cvh!Tr%FdRt$Q8^>x!w38G#`nGBc93#C%6qJ zjw59(cdrUdJK&D^`+= zD;WIPEtVLh^7gJbG;D-4Y6v?a;|$%&z{#!)Sfjbc%!78FZou244Eg7fuH)5mI3I}Q z_QqQZ-LmaO(y|(OxwE4}(<9VR+ch~Qj?W;&3h^Yq(lEwAA9ydf=uzscjY`}1P--)} zhx0tySP3_0JbqrI2LMy7O&}?EG4mPmm*1a<2LNZ9W9pl&?Za5D+YD|Bvbg*#4=P8` z6)e+jZY_Ft<*Z{DSNB5_vasxbKEu#_>ev0RdWS>S{6L4Ntn~dpN%Vn@Pf!r4npv0+ z+!q{`jc`aL{6`;#s&&)sQU|qon-FT(wv0Iu+<_{7OfYF{El*n0*o%l7c*aMYZ2QNj zb?-~*SLY%t?D2*@{>?W{)1EVE^WE)@B#LNTamL{3jz8bMdyR$|gXnQy61~?N>dGs- zc^BG>0!!^W3EU4yC^vud2CLm)V=QUxb}i8D zL+MJ9%>fkwWKW#dQF>NT0zE$r#`ODvpz`ao1S z(N}Yy`fJTPdmR!f63$bxxQ)qtvXb8I9T-vN^K_`4a1H*06i;PMwN8{xj;t0 z<@qr6&D}oL;2n3YHTx^nwCWdfSNo|D;x}J)+vl!h+w+oo&w7H6Ahl9=)v+3J3Qjao?0h z3YCkuiGP2wH0mv45~T!yGk4wV_5Qi_s9cw>x`mwZwq%NU**g=R!2Y$(tKJKnXzo)P z2M$N4x8+UxY{Y7IXLoCu@&5q4ReLSY#jQ0RxMjeCS=3##36WT?*3Qrw!QoFafsi=L zl79_vRF>Bf+edV}BLLh-Ap>i1l?&e|wmXBGTGfPlloG`$b&Ui}Kp;pTAh92G4j0f- z-9>2{MhWeY zdTOq1cTMfi!=06>cELRHmMyH<8@p#ooV%k8L(Wz;7-lRv03U@<$o(om3^H5CYokYN zYjYzid5X)u_Y;%x6S(A-$nDKM>dA0qHd8a3hPYUz8@Z2hA%WY=AqSFj2?OPhTj^rB z>KSgGkVcHcCM2*>6p}JWYz{c7V48B2-`I}M*(kyZr{_NZ075gTM0Fn%U140sZ4@vn zu6`a2gnqA%eX7KblH$hFNS}I0p_PZI_o%N!+gRG$-`-waN^fr3M|Fz<G~2$Jw3^ykQo;mf^Jf4)#9}f)ZtJY6CLfWnQ%iR6)%M4ha?|deSVcF@k>xT zg{G%)<-*-uTg4I)!a}8)S3Ubh&;9A2qyw}^$%Q8<0|i+M9DI*ENc1Du8LBou>m?`P z?0Q8?og6D(cl}2n7-D}S5cOo!c^Vo4$J)ToaL@^KmB$3I+Gy43$UrML?6~0L z!k6&FPqozbqxQCqG^>WjpqDdYhCm{Dz~z8or)ce;schWslxjsqa``o<^!A`SSamJL z{{UdLX(M|mr$;|%F61AZz0$WP)$lhDDW>;kdkl8p;KJH{}A77Vh zvnW?C+iUW$%GJuqIafaqC{x1?Qpx)E%4mkxn$mQgoCFTT50IeY01JVR_$Q79Th|wj zdwcemk(+h~=Eg_^kLg}x==mfX=AE`i3}ypAEFM0XuSu+5@+5*3tXal;4_y0KHPvB@ zQ};)v{3SIlGtY+q0DLg?*~j+n)%(~Pp|`dm+{-q^i24K`t-#6tVyOWVYC7Jb<~U5= zPa_@UfN{>@k%8}%{OZfLzP)RYCJC6wGi_ui?jSiW^7@LedTcP%Z{W-n3&p?vi>Klm3y}82}Eeh})@sJ543(GOc8;&YXCw!#mC(rmf z%ll%mhK~z$Tg`W=ZftCznZ)qCWz~YJ0td#;juaEg{JW41_2Gwxx<5(iHxXO7ZEj3M zd2w+U#T=o5SmZ&RMA*oP9nt~|oL~puiRlYoh_f19h?PadsB7r}dxpCTaT0(C^9cBn zJ4Q2|&&YhW%u$_8+pJeM{{Xw1#R!wg3`xr${`JmDInQjaaKM`u-rI4D*kmhMP53*Y zNw4lL?%8lqmX}R@Z;D{qS*CTyb{v4CE6VN9ahkO7=qmc%#nArGLO$N)49$W9x~T88 zwl<99e-Q&At6jmNTR&1=PL}#|>^ia|1`e13Bq#4XjNmH$@EUezN}nj5YPLX&To~ zg_)Ke!?5mHj_!MB(>NZau5^nyEeTb&RAt!_xy}jpRqUhnHBjhUg4@h8elY4r`nc1t7dwhhe=&DMilIq^DIwX>AS`Tdv&+IfpAt^ z3B-*T9>XW)%}x9mveT{B@zteU`QkA-E4B6?@Z*7yzP-oJxq3IIQ>Zmtduv!)&NVr? zxX;9>KJyQm-~fHcdbaJlPfFA7BD=PVXy%DbaR-xoA_2)~Je>FE*V}+4qCYlQ2QTqv zvY1uqN&N49zt7xcJvSBlnC^iwmsRq1jIhQJemNND{?zHy{V&q}9TbY+r-~hE<}+EDwtOmC17y1}%8YTwIl=biRTI`4*PSax87weK;83HUL%SRa_<9yeUGJ@1DXlg`3)nm3UiE}Ljj%#)7R%%tP&Sud+;f?Qu9r<12m0<0D9-rxwNv+ zRMgzzT8bX}6584IHM_lpPY`^Vix3~=anC<0yzI4(l)2K4ww2SC8eX8wwmC(_p@H-v znTYn_(+7sV5oc@C&wZ#}g}*8a+nD|oQ}~&fZ3o-G`ZjCVki|TZ?{`JO22cUcIl&&B zaaFCG?$3j@=si9Z>7wP8E?+C1_xB@RO$^q0iME~=id8=vk%B5Qe-qqK9kgl-%Gd+k z)SF$tH*?1<()pnw^72nA2VwbDhv9#Q?<7N8*Q}9I<5OxkZgmIo5!rGG@5s-kS<|$Z zSmCa5L_5 zc>e%m!@SVCV@uNKH)1(r36mQP1I|Z(e%R)quul$dwN1!$(I1&37b-t@y9>1bA%p(_ zx93|r8)5UddA`1>w+wwmx>id)OUF$f&iXU`m!ZJZU6I1Ylq9+NC<&jTre72~zJ9an z`;Av?MP!4?u(v&-fltCQ?tbz4pOrN9=A$jYM{9btj7Fu@w}uS*rK%9}`QVPMsPldWrGE`z~3MJZvA$C+yIG(__1z!h_~ba6_y#6B~eM zf#fabm}M!P@J>fRJaRMoS1zG-z9{YEIYs!K{{R!7=kn}7I!i{=?zN3x+f22=mU#j% z8Nom3Iii^32^BiE>s4cN{D(eoeK(|Z^@M7vA!uX^G-e~_PRze?GlI@{ zcHj(xeKV_zi>+MhQr%B8-r6!Nc{y_mDGF_%Q=HsvVdh9Pj1W#c*`LV{$H|dn^y~E3 zFOk0IPnO=|SWuvz+7I1%5B=ObauPCgnF@{4NKeDa8EQ)A;^tV8nGVR{RI~%1E)sH8 zGsyPba(Dwc7_F|Oa5TFc%QrUHHl?hV2r^bUq}*e{?e}+~Vo4=<^=fO=ty@!(ElQQK zjI?3n&+jA%NRQud9Fgom=OjTGO37hzRFvafd!Gj#-2VV*0I6lN53~YF^Lh0L@aI2M z`O?`oz&6q47X^?EgMv;<_WdzT(%u`H_AwhH{n%gO1A$$OaPlY%><8DM z;CA`{05e}j{6gwGzJu@*>s_^Q+J>E@TSaWr;QsxRIFX3;Ahzs!?XR6W5ol;a{0GS}D!p z7bMkj%PXxYPdv|YaU;zvu3hDkz9)7hmM7ejNi`r4lmz@;y|MoOlu&7=+SojYYi9%j z_lK|g)@pZZkq8qeQ;o!W5BJ)(BM!*m!kqi%lySYu53$rq4{RR7xvmvsZ#v3WUO52v z_x`o3i5SLFeRw0-dsAGzWX8l}uqVAoCQAf6$9o*p5j<)o&fUY*W7fT!`#wx(@K2>2 z{u>h?{>h|Y@~;aEun6-MmK+5GB>acHd-K9A67Q#cE$OWi{vj3Rrk@(zC-6wbyD9mC z_&+deozE#3sE#s8{b(AFgo>_O0nu+ zE|xJ`+9}V&7!fNU&vUWGMs$x4@8tomwb5#Q2Gh^dte+6Y zbaH^vFh4Pc`E4SxZ1t%Q%^ud?!X)@b&CGse#P-e)3OVctt!z_7ToY}-voLi!xhc4$ zm)#6nA4FbR%8v(92>`=q%OK07-s zT6`nIVdU&INiS$GvXHN${yG zY@Q{LS!k?nzRVOTwhAPXkapT4PzE^o#bs(Ci7YH5drNAR8*MhZ{{Uy5F2k;X8A}No z1uMo5a=7;SaZ1}c)MmA}l5L6@d6{2@dy+@UfKNPD+1f<4iP>6N7x#uWvA!jj4SbeDIX1wmcYmI6%FWZZZkaTJSHnhV@TsA zGUtQ4aX#QEC!PSO9Dlf;KtVoMEy{YxL}UJ`)MT~KbioE}N^J<0x*`?VG6AuB9KA;{-RB8?k*#8GpS{;kp7%P1%m|Xbmy`r;Jf<*INQbinxlTAI#KfU-o!o``5OOIsySuv9A-IXLGDsZ@90dw<-_z&ysW(yTu-)m=TF0kZ z;M|Av)({<6ZwsD2@7*J@0Znb`{aK|e9_-<|)mmyb4mxFsed6MtP+zi23 z5->1EdwNq9?9$21p9=AXAIhYE6UidFjlNTD$N*p;N@&rv%gt+5w$p4Odx@rX8&lDJ zSf0QE+@1w$*|N?~V!hu3=ZAei)!O9joiC(Z-x!$^SrRhI5pKngr>XpnTN2rI7f3;F zHG39&%a+^c$e-N$5Thp~k3+#6k~>w;;KrlUI!}k`rS#h(h#7>Ej!cpU0f;B>VYnx@ zc=fJd5kAv*YSOee*EX&)>CWJP!|XxloF8wUJ2g|3I}_Ns9V`_4@W|)02l=1)s{L-rA?5EeJT>Jao&47^&2x0FWM)LmV;cQ&z3W4F6TiZymBu8eV#MtyOS?_AvwG`TKysP#E!+a0uX z#;Y#Yd6ka*mGtl1xX2vyO8A=ga$m=DZ7i)M(#G*fa#0k9E?1qQc9iex)W_2Ds>!HC z6f#b+pM}DxM01?_{vZJu!5=(Rp#K2kcpW%6+LI(2-M)#S$!K7b;&}#Tm@vlZNOHb_ z0#(9}Jw1&_{7AjBz33Ty*Am}FaS;-;h6M~^SS~%86OwrcwMZ=Ei&$u_FKrfEkF!0i zA2EYORYuT2;NvF;(;#Egr9CmzT6Tdjy1I@%Pf%732)T`3J6sGaZN}D@9=(^)XPSzO zyIYK;+Y;*gl4)TTH$wD>Qs4DWhNOCPQ;S25!?f|)Fo{=gPTc+PQR|cIR(-Ce4g23& z+ZTnFU7=)CzXRWoPhnGm)>^imy82yCLaII?F%E=#6+C0*RhNi8b)z>_#j2ePWRoB_ z5ZlZ{t7i`)=kD%a%Gu<8bz40?2Bo<<#_yst#0D9U$I(G8{R$^ z;!fKWIoJ*kJ92T)v8w5(mrNSjw-G#zaAN_R8-6(kalzvpa6LH30%g&}p zJAi&Kx`V&5Zk7tSI_C4h{=a(E)m8PTlWi=l8tDBSQ0qR9mPqGxFfuQj4n|0k4%pOk z@~{VLX9t2#4r=J>?E_BfO)X*2_W73b8-Y5oGEeZ3QHU6OmQnz0ZI93Z2|b-Z4Zlw` zc2ESgvfR>G&9s#GD|4HP%E-EhT7@{xR1)Tkdvp|95Y9AjF}yM zosYt&;oMjsya06OqnOvnCKYg##HY^(R1jL;X)sRFNEYfg@?eFEyd3%xJ{Al7vq13PK`d%BoZ&8{%Jv!w7 z0QE`aV#XIywN87I?Ze{%PSOV_K3VZQ#4FDc_4kG;O`eNq8~v6UGNwShsbD^Tb$bl^ zl1f!qslfToL1mKevg*sesnjD|Smv|PEuEx-+aURB%u6#9$#PD4&d}HgIW)YFkwK{`Jm;N8==76u0tXT=nk-07AX9XDGZCvdKIpIzL&T0WJ zhc2CMdn9-Ek|II2NX|o?x{>l2Imri*J&j(CY6}B2`iC21H1UQVyGCN)7zYFp$2s|a zl=q_D9q9~(eld@s&u^tLD#>ZA!)(EPo2b*d`?)6!KTLfqwjI#6!Hk2E&UY#K`+Vx0 z;e*mTZ%}er^0_@};Qg#Re~0kujUSgK;U(QbEL9k)08gO7`SNRhL#~k_7Q+EaS7lSR z2*Jw}k52ypQBiLQI=T&8!_Vx-iVJCUx#hInW#!gBck*~Ac*xIl?TXf~4Xx7R`a>cg ziX~%$4tV+~1G&jLt1(!LNb}C6e6<>@Coe7k0CAgB>j|zI^xG(y-XLiNUS`x&*n;F? zl<|ymbB+&aQ|MP#a?Pk}>UDUFG?w;IF2#ZUU14L1Jv)%!T6fhapHS88W^-u;rJy2X z6l~8U?E?UD!N=>~vrxLayp3adWV9U%N}&>N`HU)O|T_o3(i_?U^>lvH&;$cHra>ah}-Xp}k$JSZemIEVgmMc)Xb- zTyOWt9DNTRspG_{ZSA!?h_tDvipAVYw$iR1+Du`TgWI1~;E$z1Zo^9%kQ@Px>OEGg zW#|_XoZ)R^Yx=d6!n7 zb?tj>PBo1;M866yP03x@L=G?*A5)GpeSSu&KNux#FGAGqU|%LVu7#3C=I?OB1@rX) zkbZsX+h5nUi0o#(jh+Tk@gpuU!@B!>)pM^s)!dLvs7~_T5^wHRy9p!!M8EqNdHE4m zDz(=cKNvC=m->q#1@P3nu+c0nrMtQyTfqcNmUkqRf%7Gfe>1?Py1PTvv<*)0 zMb&M@7uKbGriuRm?@Ow5{p3*E==WC9X!uctwh~#SaX9t}{#p5AsIA^x-0djpP*Rqb zsQj^u-|Ts@VrG56JXDLJbXhNw>sGjF;kC78ZSf{#mHfoSQIA(`$_Jp_Gr<73>3u7E z9<6k`oxI5nh3U1O5fss)*BrLu0X>r-o=T~i#v!-7)5n_~rKFaxWcN1Vwp*)6 zRbhj^WDZwzABoCjki50%%`MJ%TZKGD{r>>Lt$JDSH3q!7ns{WA+C8$yeAKa0%8dj} zN#Al(;umB+eY|e)2;6FBSN{7XJo;* zjIZ$p$$Vg7R z{d=|R*(HUig7PRLk@sRjfw-TDA7Sh2D_cuitv1X(Y~7Pl@QA*=L|Vk|q4Bj@v_ zG}(7x2KgEdQ4PXxY;GKP&(gSTBq=5s42*&)*~1x*=#p#aS?#UkU<$OHoMf&}KgfLf zs_ym~#q9CQGbC(7{IGFTxm?R>fXsG){CD~P07~G#wuI{u-K1cN;suYObvs)h%j;2F z2RS55Ow=OAzMIh{nt0`!NbREB14#Vr1oy}K)r+Fp$86Dt03hH3IKems4^%&|Z(I`1 zqs41ycN~%^nld(~c~w4uk5WCpl;UTZWZvwoT%F2OfuCP*`+iksh3W2!*UQy-Vw#-q z7sI)?eb1IhY#(aiw}E5$<>a3H;8vxVEYr&)IZ=V;pTao7_s6OA@9HVG>iMPMG1^W+ z^(55tu|3MMt*wL9=7>tmA1J5cKBRw^X{p$=9Fj@>N61mWN%z6^t9SNp@O`@HQabLv z42jbgQtfrgQ-3J$5})87jfedsl5t*h#^tKvosjeUDELhJ#;X^I9Ybr4F49fo%o4|ve_^kyE)Yt`-QPD%A9ia4T%7~p~^;ff>)9C4^sMFRm;4gdpy1z{CVPdk^R z9@T*|Ijv%k6fN4CNG3&P5fzjH$~#ua;-LnKC=nif{iFW?<$OZ3m=pqg2pzVZIVkcd z8TS7Gy)tzD^G?g>OvfK}hBpq}fIFXl*%bZzS?ISnTGvfq+q%Xjx0=bPO32H&FbvPealt0Osp4a+ANV!FqLb!c8@QuonG4h`tndfQr-P_y_^tHcKYZ|tf(_JI0 zr0H>>dG_ceT#~ZNj0evOr9tdLA6l_)v|AY@^O`k|Mar1}01FSNr_=PPM!7wS>LG6> zoRM8X6_B#!8Xo7LuWa%;s(P(FQtpfVjU67YZ{(K$0AKcGld~8uRFXu5H~^3VB$7Rk z%B0aqiPNhZ0uAip0Kv&zf%tojEBE9dtwc=D_Rzm4EUAnRdmc}=M{0VtoJVn{2H6}6 z%`Q-F8-D0L{ytdpa7GC2icZ}2q0yE*q&}&V;U$PdC6QBTjThpPvGCvy=lv^!eLjC9 z{%cP}EU|@mw*^}R2XV$#f%|Sdxo2^Ge*mW103Lv2*4-us`sI!I)=M>sZK!i z=6PL@rb#FCsAW7&C&JuI;U}k(j_u(ue6qvp3+QzH1|@Ds;ZNgie?jZ|delomu(;PX zPc~i53uK>qM_9d^T-E}|yUAaIB=%qRJ?hlxEeR}i<(1HGBR#9mektp{&Gx;P%l)3& z@GrwP{5{i?n*RWnd&&EICH_X;dRTNl7^>k&Ue!l+CZ#3a<8ob@hr|3$=stsw%+;0R z?fGkmiBH0VkyO%1B)e&3Ambzg2?U>~=lrSQPnRY()8%i@?bnN|1TVpvcr}eMh$++}6|SeHE9yj?R|THN9^ACehYW31E_3zyd^L z8O$*H0J-i)4&r$yb9w^frEX*CqL*!!F%ZoBd^pD^xfrV3!j87I=+rYsF3TIP7Q`Si zTQC`D*z#UbNz3|!lj;ScopYdd9+xG+*P)0IH=$q|PI0#$;PLJ%6(mVGwD&y8;hIQdm&qV(RUc`%L`r7;%TUmS7m{{VXRcU8YlM`bJAYL=@K zFU2FU!1pAdQ~rXgZlLPx8wC3oN$KlxCJ6gPpoQm+%Zw7;{#<&~)U&Z0So#r7YQ*Tm z8)>f|d0ITl512?npPyiO!ur*X;I@*MeL1M#T0sL&v6FE;*gIooU~s>SJaP5pW~(&P zZ#5gwI_7bEJ;4tw!eFe$xEqxB_3hXL+Z8hKr5{OcPR>i%uONoXOQRQ>N+T#_E1vmK z;k$9cBBPUD3sH_D`(FqR;FDB!Aj>N?uHo@}G^BO}_88;2^%Wp8gi`F=Km?M1t#YfV0aOK50aZs} z2LOIm$F947WX@BtLR;u6t6s#3`;fl1>c^Vk!EQ(hejo6A9D1?;0OCG%06LUY-8HuC zjmOQ9oR&VLKh)PQwRY=wG_L;u6J(x!OCxSM)K~otzs~4qqhknM<&HK+xJB?5HT7C1U5c5kP>(RQZQSj=yZjaOPx{@ zA5WRa#&iSWW;<~lD%eK@;&F~hVO^UpZ|x$zw2ljXR@L^nm3IF1zVdB7qAD`2jQz$P zt2yJj$t+~Oex}V%BPa65I9|*BnJ<~B!fY>E2i2|O+cc{YSS+C+G;^YZ@b20ec4Abg zU~4rER>H@puU}i5M2EvjlKcICoqzP)`If9r*Jnh@K;BU0u{x`kEPS?Vyou?O=vH0k_$J!x5es_Ra?f znJbI2e5F>@IS<9ov!AT=n~CJSxQZ)D!u^P$K`Q;v-fDb&c83*N3 zUkP-Gk4oyg)Qvxww^sJhq;jlb$UCG26Y-s-V0vJ%tMw%aG>B!1&MEo9VrrK1jRrZM z&c}7KrpTkYllMsK0r*D?yPioo$?s8ZJ5F@ft#!M7GS2@1QTs=c4V2jtTucw#Il&wd zdH6}&+zeMnt9Tkiq(>~a@kt1H-B^&(=i*#}lZ=u8&T~z6T6D5?4xedxG>97JGPCX@ z%8i+D>PINRdW?Wa%AiHU3ni~--+{grnVJq<|BRK9r$E`{F zXT@(aNi8+xXSQ;DnS%Y_U*2QC<@BZ-%`$CAQY#eJ!b#!s-3;o9kiQA%k3cc$icUV4 zh_tG+#_&ZB`6h|C$C24lhjH$G_^H&RjGX=5D(iI{r zPhti=Kc;FA*1bn{sJ`1nEwoBJ$HrBNAmjOSQD}7Y8#IZTk(nGYP^5`_T}) zwh_jwpWZ+|C-v=|4mO1)<=zz!|X>tchqEaOmjmdVM2%A55iRUH1pFsF14%N zl(V*$X#pN&hlPw0j!EsFMPo)Otk4+7@^nNXA=saWv$lI@1b6vWZQ_IJ z3Dmcj7WP9=XR`C|VL|Y*9IS{td_)}l&(5Kg6nHuv2U?x~042QRpL~4^i(1qpve)5g zqYCiCBbk))v1DY_JbhYFe)GcWjX>w)5sc!o!S%>yAAt&HbOe zF}d*@t0dNSsND4RrOB4|RvUiXKL^QNb{)54?yeWPq3}b%ZEvgvwzJeJHjSVhI+h!5 zZkO6MKXFh00Fl4+gWkQ7)7nm(q3IU7MvrL?m4&JZ;DRBAMgF9d$tRL|HBQSOFkZRU zq{^xHN$sC+`y63Vv4llbRTx!J08j=11JD8dtHeGrSq)>xZl~8FD}SZWcAsUI&Oso& z9#R6aus9oW@jqz&c`6Ot`%?8LyJGrhOj*6%j+3ZBX1utxC+!zwe$#G*0#O3|ay~dx z2RYhkiv(CmHz^ZMY7AEq*&ry$tQKo1n2-@vOB{~*$;+`MfGI1dD=ep{)z3{gIP%sG zgK<2XgX*$MZ?!vT7V}FU@A-sv^W%m9#PH!x`5)qK+rJ)JpH+lKZ*ivS?H~UDPW}Ca z$sXi}7Yp#lLHpZwIed~xkdDBY1(EG0vw}vCYP;z&TR|~c72}XRATh};C+<~`Cu*@R ziiGRBpt_t}Tfdf;@w`$^6O|rWb}JGL@q}eYKA`}~$2JsRa>%0G-Vznob*V01TYWlK ziq>FE&Zrl5{jBi}akW?=BkEf?nna%^5p)Mg4{}T5}5abNd3@q-}*@&pVFH(jl%vpS-~uTGj|@K zo;@gr_LOe&n{WpY$T{PWo@+ugZa6vX>+P|f@gSPPjy~_@*!y}@W%aW|D$ao5o`1;G z=8+3BOA{iDw+g(d1Y}_QWd8tKR`Sd}u`k9~?w&hS@sw8X?UO@xohUd5g*<*#j-(7$ zEX&BZkUxIo0Z;NMe_Bf4Fw5nzJRE&#+E~_A+T<0Lc1SbE4;lR%Kb1!F$s~4roH|ye zAC=`xpD2#QxcP(fr?r+pA(iA#+mVN2zKlPo%eS>PY1(F*s-^8BQ-Zsnk)K}m-*5e& z5|~j03f+n^DjTDs1?mn?<2X9z<5#EHHhR#1C#Hvxtpn*RVnO$(mD=AC4Q zPu@VFi34v?e*>@e`uc;KnD`UW7Cn2`X{X=nQP@}{M&?+=fo?949%6)hj(4}=9P&*^ z-|k4G>>q)C6Y5?eYd&nIYd)T|lWq0tj29!HCLeb`r?L)DI0xH0Hk)Ci=~lW{muVH9 zjif>=cp?~B&~^j*l6fSOc{Q3onWyNwRkoF+*+XSvYZ#6TcvoshzQg*Gc_)+2bee*5 za%Gz9QAQdPIj#ConqejZGz8IlP;gB0OS;7y$J0@JJ2F4Z`wxN%<%FH#f2n$ zO-dpQ4DIt3$~1xP767 z0yFF8dGVG0hxD&HT-ZT*46w+Ess|~2kM@4K^fgy$nm)e;>cweoCB%8!vL_3) zbICjdfzEISJu1o3;JmxhQRLJXdv)dGh^ixyNI1{m_q{ATIOpLNFPa840dHp+M z8K`WxzBrS@C#IiHS&GXlUZ1v_<_OYNc$^hAU`KKWamN|1OXzgzNi$qtTFTGH)dDBl0Qg88_CKJmRkM^`#|$pA<##Lk4E+01 zEJ?kG)S8aZ!0oPHK^~ENJZ#IGW(rx)#>_BD9*3UYy=otzmb1{duMc!`5BNR*05klD z&Yty{ZQ-?RYtkGx%tVI9#O7g}IqH&lweO$r9UX=V-K?16)m3S^Bh4-Oea;F53&aED! zXH9-WtCPQvu4}hg=-mTSxw6#bCC%(!eYTyvqe&nHye{~{rx{kxSROrCdWS=^(S&2+ zq+|SDqFRj{EKd$Uy7v2!o{G>ewX1)%%#uf89HuvM9{FW_e{~xEDG#9RJ$(VP%#i6q zD=SC@x^Pxw9Bvg1V1mpX@%FSTH%fR+d_91ylI}ZrgzzH1o+9#YBaMtoU@XCQZT{^6 zCoTt?>^>2Ulfc&zM$uRaVir<`lE9CI@ZSXTD!0QBA(hi4pP3mOw_Pi6yf5x~#-51R zpLQy<+S=~3n(13jyonE&hLl14dmlrZ=rsg~OB3F@wcU-Q` zIh7l0s*jVm3VSoCJDYVOb*Qzf`As1g5z6i!S0OE>0er|#(nus=pd2Z{P<|pA`cm>O zLi19yxx+ygo>69);#2Wq8c*UQJ3;&)`uC-(nOg9M-*hR9l)Maj#<4s%(pia}Bb5H} zNEwr1_+0-052*?^!sfjalSE3P@!cQawP%1xq8p(WVeEy}xI+z_w@F zUKPnDRk59?I3VN^-m+xuO|J&8bQ(jpm()66nW$UX7_Md)kln!Z5WA&C1y29~M$8P4 z#p~#Tc}sbDa)HC%T)oV@pLQ9};sN7;N&QLnrjs_81>Bc0s;$KCOGv{m?8npn1zG+a z_*2wfNb%k3@lT;?C*s{QV(M}K0O`hl;6KLW=%$L}W-*+MvQT@Wzr+5O)jGddHadRT zV21|o?nNaaQ``QgK8yLB4r%kPx)R$@wX-(IRf-$)acE=IpjhQuBRh8hycBP|qz#HTTKO&_fJB;OHrX&;+TLYf z`H0kd<#0Yhxv8yCrXMri@IwbHCGzlj{{X`O00{=2eKxJsP^HGb1ElsA1w^rh4}*X* zB$$DKr{RpU7UzL)#v`B1wv|-q+LN`wUo#k(oVH!KUBquPHh*{sQTJ&nAfDdwqPK_w ztkc>lll$nLk2)e(miUMvR|+#3$;EW-vXsVOciA-o=RrPE3^NXKkGh28a;Z2+R_;g# zHg>yO$)Os(HG>&1Zevz~cHh~Rj#(s>#Rm&qMR|5EBHtr9gOrI8@wrknow@E$a(O<~kc8&a@K050S(AGupZ@>^ zI{yHxdXuZQi<#zutu11g&5qFt3m)L1ij3eBfKEI2bZKU&s=J600G zmu;?XJpNwCY`$pHjp)6N?bXWfP6mn#Kf-MoFWW`+-PD(a&ZJw1 zQt^aU$KJvryd-UuK;oWwPfhlvM+|3UEJ1MlFe&x z8rfbn%l1K>uw3_0OEKKQ9qt7+$u!ApPPxWy9CBS5?rj><-d`@>7G+}0Re>b-B!Dtc z1NmfA)`(}0>dMulH&+ncg9KP+XxT`@s^<(b*kpS0Im~mVR%`83$&GF$jeNq&k@@Zy z2RK8^<2-;nXOBv-CDJXSj?Vd?sx2k7Rahq`AdnrT=gv&10nc;WBoYTAowHDBHf&k0 zS5oUx=gvki7j(Ea9e(QO!DF^q$PXJSj1Z_;kPjmObDrdkcfhNy-r8(6wZwBF>=g1n zvOAB`tPd1r)bDi7M^arzS(HFpStLlL4~4)PjNqOhkT^a4YLxpr000|GfrbYJRGNC7 zvCCxIUx*%xF|KjSUy~(ofFE|~*tX!y!2})%^#pgPS#PGg)2?21M7z0l6Kvoy1LACU z1-=q$3NQ6rU_6_rkw@W^3_qPMA6482#m$sW+i_f}IH$`^V&NOwPl)<_>{dy*ykm9m z^+c}X&&tyfzm|(BRourM7hLDqV>HLE>GDT3*D-*CMBVUkjtS!gas25GYpYW8Lt7ho z<$`6E9VAu*XdU?g9QOSwqSL80OJKodjj}LuyGi=v8T{(rny%RHa(@wPR;N&%Z8VpM zw=W-_4*M%RwsKw>Tp%p`auskhpQ!`$sb+^Hmev<;l1I8(Ruy?7EU&bE+>cz7`c!jF z>ib2Peg2fBmD#x%cFubd^7p6vexvED%Xlq}=H3)?;x~zu1|Wh#T#DP%xUM95?z%Qk zU9yA6zA}XxTU~Fd?KG>`7BKGt_JQqzx7eKbscy5<7gif>q|=z&DGL?EK(Q;HC!lSQzs#EqlZ4V?P=x>+6Q6}u?MzuR_>|NIzFmid7_RRmy~XjBe7h8#^vqUWAmoHC#S45E0>z?c`oh@ zgu@!)65IiTO97mb*a2Bnl;?#f!O+tvRHfLlo^kXbIwyy7-cGZvx7qD?WoNijxETW~ zy~+MvvsR6Uo2BS_RB%QX81Mv)09hQ2WRd)2_wSEPR|Wp9Wf4ZVxN&s(BMe!9BpjUa zoB@(C^Q!*0v`})E z&y4<_?QA%#DDmqmq9ryZi>!H13ViE;SpuR^Tl12$muj?F>8ZEx=F)OP&>Z zK7h@1++nd(x3!+hv3>D_l1@GOHAeNHiv3Ncc9E)k8+219MmM<7zw4{guyP8r7 zNW;2p+vu!Meq~+<_1)iC^xehlwXMwS`{j}gcZvwmk_s^-Zr9@5-mxh8; z2QBducopf5LdpC_@LFXE(`x#LkQo(*!v(^S-ca^qECC!b9I1>SUMH>mxb_`fC1jK$ve@6&`t`&zOQy{$oAc$z zsE)$k;t?b)vGIYqa9EBx} z{HnxLZ(~N7Mws@?l0gU$8S20Pg%RE-=H<>DjgJh%>-MD;S{DG0_o()}sar=?4 zmxOb^nH}m!aU^nrlRQNgca`B;K_kS^!XZp=_ico7s{$3h)o-9 z2bJ8iD;)dfart(xshO4`9H6UtTU+75Y&sG|v`ZILxu&09POe2hgrCc^{oO z-o|bu`${ju6byeMQb%aurzi7dkLoay`@xfr8w0qdBapSaqwqfI^gQQa(WZb#N zc8t*y z(e%$VOt9VNO7p4~WdO3MCoDi9p1^vVhH7_Fw5bd{#;izxRopuR{&ne}*k4>*`wph) zOTdq8_`Z%OP$< z2P1Ohf=3&0dvjL*0ED6SUl=-e_ z0Z;uA;ru;4;bq>p4BC8d&Lq?2fm%z)y8A(jFw$@n5AbY3@3^-jy$p~+YZ$e*h8Q4? zh)AJ8h`jTK00B^UIqpvgC^Jotah!T)fYF>#8hG0=*n>W$l0EiB#Aowhe=4S|WrhsQ>lDxCDus~zBYtoW z2I9ny5k4cI!Z2#-{i3Z!$4TjOB#q|9sGX6h#{9%`Oa292pq9bow;N3(^N!^D4`_wO-G@y* z#;dDLZ}l=t`Ad?h{-IyiPW<*>=RUQUP<0lou3s(Iqbc&C;wZO#C@=>e?SL}Jll-b1 zbqcla%M@uHuYHkg^j}}S*ICbjQ|uEQH4; z%=tph6=ESUI4)RZFXi*^UNN5bbtPjAsf_IdAm`WTnh)(=NWg9W*>tc-K{1INX&?0G z2C=S{mdm-ba+<2DIEp>Xq4ge*si*E5DG~+>jW8L>&p!Q!bMH%Q+D3=3$0e=9taogH zdFVz)PCz90UffrW`a7+4oeKRKFJws-b}Wq945yupJ&(Ou*4`^0HB9gw$O5qq$s}^# zz#I(V^H4_HD3)UI$-N5hSenWf>w7lojaBsBQ%<@rtB9DsAeAxQ2po=np!epYudO&F zS=+~K2*yxlzf)Jflc>Yumxb4wZRBI^(MRT64Y(p?w}BDPQtf@c#fK(K-&FraHT!L{#I>+5FZ>W{aafL}TB zZ*4aWi*b%l^*;iP$m{5)TMwj zsfeam?7o8`m-m?i?l-Ap>RL@I)7GH3!OUy=o!Z&=N$j{@5vgXlS*|6P-Z3f@7Nw`7 zW+mR(X@Y7}G>{bZ&t#;F*kF=!htf8ZR3Mc?=+@mU^?y_UB&2#Fy zoOY1HMa*CF(SZwuCM&o|+5iXp#|lETJ^&krGJsT0?(y$c0_81YW@sWTTVW^S%N|(# z&vHrQHKk2X$Ao0xYN{R2_{B9#xo#tl7Gf41sG*1_U>H)wcWhv1+W>GYG%Z22on{?C zt|4g5cNrNBq-{AEIlgv$^3efDH#qV!59eBOUCDJ6xV-!%3+oTu_?m-GR?it6~-=#W@Zw`yE+TWW=Hc#d52RS1vxDDz^ zATs0<2nLery<l=S&_+3ZXz4UDak=8xE}L z7HvCGoN4!QETdMAGP6j;Wm+-Z9G>0BBZHc*E-c>Ob7OUCG}bo@9j$~rq+46a@D^!+ z&fzcv3&>zbdFKEzEEHj~w69q8OtIa~btS}i8l~eG^eZ3u$Q*I_SMxtQ z(Swu13us`w9A095uM|X zeQK@gi-|n{05zm%W^K&Od1LBH^)+mCzNG6KkU<=9dkVQ+Jw1xHS={nPFZ?rfm(5}3 z0{;LD1~?>oW86{ObR2f^8CozRVYDzG56`#t_p30-w}gv@UoDRTSnm3o;J&kzqms`W z;~xv)0sdb~m%Z>o*oyN|&@{Mh-bs06_)8I$?dhI-Vv*4;G^<|^yylV=nnNSR@4qJu zz#^49S%%zvV~qOeH0RWi6y6+3zb9)E$ERWM^7W<*t#3oqx=}QzxSem~wN{o*p9*F| ze&_{o3mg;9aqKAxVHYg@5PcwRzqARuv9q{X*Un|QxOn4L1OQ8}J@8I(%~)Dph>}R< zvgs`=GfWcRNc9P)L)_zJXLAw(;AIT1f@O4;r25B$RMHhpT#ecJ3<0 z({Fla=c+C49@5&~<&gQQE276Eu_K?#y0puy%`)yg+bg;4WsQReib8~7=hA^! zYa01 zxjs)%&uee^ZY8)3S$#4vFnjjmsTlg>Q`0OO$4!dH*s;sa4Z4SK^8CQ39Yd_TqUTDm zEvno`vcW4KEbZJ?$jKuIwm2TuPE|bYTB)}d{lEMzi@hsa)zn)-wi`k*B&;^59f%#c z`qvEC{U_5O43|Rkc!6w1hAn~4Fme3FMDBH6e_6JSUSHl^DO{o!!i<6FFgd8+w#lb7 zY9Jy>321>O8wV(GFot*MvI*aro%jOW+sOD(n1>OpQ}nkf)qEQnQeC* z+7Koe3cvDJ=VVcw0I|s3n294o--PcS`&7U7v(Y@?4p*q-1MONx;>$hI&O{R884*8( zVUY399CK9v0D~HZ#jVAx_NvY({2{Yu`@X%KY;X>ekUhcU{7&N6xue69%R1DOEi{KW-W zgbKU3$W_P(G&=W&xBVTVSamN{#|cK%7TRqvOZ)X3e8Q!MKzsrS8QMyd?N;GpM?o5f{n&merxHFao>(l%B%?VCkN>r>zPoikhWA#szH|MX%#eAj8EmLxyc2-A`HG)-d#XL>OHRVw zOi|8by=5g@>u?1@e#!<;K<|J^?^o!#So=|(Zklw-Wu}bj{VXnntVsk=PqN#i)cKJ|8;ql1>}mHKxtvCA+jvCiTpksWeO#Tavp zY&;CwqFT9y_^R?3vqc&xWalrV3gWXRZ{j*42M5AoWj7T?Q z<)7)rJ}-7>%NRm=UEdFEf%NyL%|bsmBN_N{_edP${rXp9NMm*_kP8ux{rz~P9!TUM zu33j=13tdA*={;JJ%&fM!XLazqBV>_DC+DC8kx|yE!buOBGgTU;#Wx&uZB;B-1g1a9h0xr5C*q zr3?V{q4c6-L+MBWP_-AO2c-aNva+&(Ifpbr(2mqYwK!=E@o%NIZmH4q^wF=<*6&Zd zH%O8n-OAfQ0f%BZJQ6$i;;7D?&^qT`>Z`jLG-zxtZOn1pvplI4x-Y}au*hs=h3}5s zV!cg@%ZrF@ZDhE+kgct}Z7igo7By1LfA)h-kIOFaAmiU2)_9Bdk#v~5mua2k0XoCQq$KLJiV@-vf~tbmLV$1DNlgZ%5Ktabaor0VTcsV@%ejY{rI zmj}=}$Y05DKPpNVM~FBbkJ711UD9mQR6632jJS6>P7iW@&!qvY!CPPnh_btDoLEbBzyW(#*-v7wj@)KMow{0z8dru=U3gd ziyWfH5ASC^iS7r;5$jggyWy^Z4v9O#WfhBC{AS`IwTS~68R3`Qewe5)-Z;+XjXe8g zvbF=lT|imeqC+buz8k$-5Hio1Ny{%%My zo@>iKA+Fz5=1rr~I%}`fP74Hu6$L7Wq1{v0{Xd{B#8-FmtnlG+wV6zW{lZr`AOcCj z+w4yi?VxLMYI;q+ps`60+EH089Ohgyh8wZQ(ij!n_mqan2Nj6=PN`I`lv<17vURMc zRyyM>$#@*~Z?Vg!+(UYy?Besj^5_`nz>kSjUospVC)r8XB$;c3&=CVkDp*E-$k|aEk4#$oS7voK0^wF*+-}QJ?pOC zP0=dw02~3@n>Bl9)Ah@nS2$FY;68;9Do4nIE2`X;QR$x9tvw=1Wf;2mH!j(2j{}4( z3uI#;j>p$Exfn$$m3wCx?Z$ck06Iq5wuA8V!36pZ)G|KwF(Y}7#9(6|32sR~l|FHj zgi-6i*h*JD2h^6QnBmoQ&rH{?dfwJOSjMLi%%&`3%%I_P2E>yQX^DGA5 zc7H*WSC>I)(?h4(M<9%{#DIBVfsAqA*Wc$=HSMpMtfTP)E7E=)SVw)*`aYt75piP- zpPs-E^8|iX1YSv=!1V`N=$#9puA!{T%`AY6OE`%0u9$i*e(H}`JP(<`suy4Co}uf0 zwzJiGzSc&wUxv#}w`_fuHW^tWLUG3oWPTge_QdPj#i3yY;1MzbhlmOv0MN)}bz3E3UO0_eFXa(Bz%%YSHYhLfuL8MO%Q87{PN zZtW~Bu&r|%sVy4=$0Onl*(!HnWvkA84XaCWVl+FLrnb4Vh%JKK2Ya&!!>^RA+g4T}hdGG~@VO`c|tsSg!A9 zwKqO>xXCf>+@pd)9@xjNd6UP_5B1*;yhKfJq4cYzu+{Gp=E8e<2{2!?gXOpIrbbnJ zHcsL~;=PS?Y5k?3&tZDa8{JzNt_`%U8!2SNY<1;G$-x|s4J_dsBZ(}6n}f3!Bn)GoYux@P^hJa{M?JoxT0IkA z+9JJ^EG=(>h5h00i7}k|lHRzlHh72OKDX1_oZ>qv8RI#4mzF!tqz~h{Hf2}#i8EA^5aonF%^vyH~e^f5m zot+77xZ1-xUi^HR8uJNA~w z+DRvY*)V{*ka9Wn?dj=N{{Z|w99Pccm5<^HyYFL?KTbb7usUy8k{t;ht)5Gpr1BCd zfy+KH=i~tHeqxw4hBmimFr{1!0(%U9yi{?Vl$vFyTzhhh^QHV5>D@cP{-o*X)tvC7JUvVQqS-8coWgNCRzA z_&_;0_hM_wq<~FmRT#x=W?J8KOxW#OZ)(aCed=N{DFDTB$Ohw@?2y+a-RQ^w2RN}Sh(PDOtxLlatWtrw*o7nmmLMFQ3TsGoCn%z* zga8Ksab3P0#3$?io{5)meNF=2 zP3(%{63EwHtAd#s~iZ z8k2Ocp3Sw2nyadK9Y+KQs%<{S)l)!zHB#FV##KXM)L?RQ8?fZ#8LOki^(&%j9ZM`R zO!m{l*9PL*r9i9%5)zB?G00Zi+be^{NP~FGqe*S5*REwjrqrs(!4{t{$4?h1p!JheXR)P^+UHaAGaL-7kz z)3se>{iSYhY+LR$qB6VCk}(`At=KOhmhFLFyYQz}ms#-lNYo~@dq;SR`D}79tW*_0 ziAWqbsKKfW!oLIVdW)zg>Mp#~4ThMHwml?*t#s%}W{{tY%Pa{b;K)hrytk*|NaBei ziZqTiP>6`3Ra6WB0N?;P0;1H)J4G?DPmx+G5>-^QvZ{fSSl|FTB%eWEVDWeMjnr;# z?sT6HCAPEE;BySx{lt6x*Yy_2MhNzD2|d6AnypoCZwQ_cl-?uB=&rBouAb=q8TD?Z z)ot|4MFfK@d{Up`MDFM00k0SMz1K4Mo7F$tcNUMQ^yQ}I)2!PICB{hIE93D#7bn?5 zgTXZd)-}VZ>Mg12*IJKI>TTZZQ<%6PMakpk+vQDn+KtuhrLJY%-NKFlVfql+BE182^&f?x?03I0$T>gf&H1)dPak)oP)GM5^meTTEdNW@=?DlP7 z>-MObQ4*nkvX9TZ)InV1`@Y`F}6RdQr zN4}Tr^4=p#k^tL+oyX9g%Z%4{qZHQBMZL(9Wj6|{gcMRw83Qay2fsX3h-J$h8NEj8 z?NOTF_9J%&@=Hf)gixcYbdjVdXzlRr2mSF~a^7475yoCdBW_YXfZ%(-{{ZvpUACH^ z*-@?K0IMh}J`$`B?}X%i!dT=E{PCV?sI-!7hCen@7{MWOPjAyAom!J`F9iwq;b9!o zO*0aAM5OWw1<&*2`BHL;9V8Dc%Y_^k!hw!>9AcI7u;fb7uOARCxsN&QIi#eS1acUl zmQvq%kP%xQhCBA8agW?|T>Jp6kUTSccnv8Z3!B)(>(R#dl>v#LdafUAZX zAf7w(S8E;s&tZY`9@P_6I&*PnrkEljRB%+7Nh_@C`qlg4FG0nTCmoM zpbb`5y(=gKoOYsG5ILew=9P^Kp#K204x-v$f_+PI4tCb!e= zuq|1x6dkgb_;!)lk5ivoeOidvw&cdUODf)91>51>?XH2LNn>K3eas{mi4VuHgM)$h zhu#OjBeyj9_UnCSV{bEwt*1k}J8YL&$iW~E*z8xI&b;5Ibq)QFqHlc0cOL_S*e@CM z2j3uI`-+_C{a*#V>2+XU1a=;F1^^`T%sC+aGmk+|lvx~F@J`dhD~%=I?$1&Cb%T6R zx+uzmcnm!aNHiN+Y{8CEu%Q8%_8#9~m2>oOi1yOlAGSLTNBDVT$R8L38h zGDS735a*Jq@r4JUg-03ZxyA>2+rI-ZNJ#FbJgUlX)A^lq_xi+g!58vg*z zxU`CL4ZW(7nS8)bKJjMSeiaN07CcvHhJ8&HhO=^#Y8Oug%wGENoU_KvKxo;A?$u8l zkH*T+fC7P2-n`JQb^evOv1j`y|&zshH;;f(wW_Ujlx zHd_z781yauetqktppY=m?caUKQkx zV+G@Eg2p~L-H%LTpF>@`{{TVT==ww!x=ckQfqRbZeM$M6wwgpy$pW}9Wr5VIvYOFf z6KgJvE1UUa`~Ah~x*9ZhyI~#Q?e-|0Powmnrqj{^cA9>-0+d+iArfJn{_V~e>l=~J zD%|3t?cr}+!zs2+9%_aRI=O%{6vpT`u6%%lU>y^Aow#CRR+q0n^;^)v32aT5ovbPk=O(*FR)-+zq1*JAVihAica!>w-h z;V73Hph%?jkzY7O{0) zWXOIvahweKiWF!60LfJAmKkPiDD+;v(^_v-bjG98da`EAP=LcB112-wWMAcnvQ25GNV{gc#tOWnx<}aU`^-DwcWhH1 zCXZ!J409=tc#=iWD#{pSe2p#-YD{GPO&IM`ksSK2pJS@&)9Lz+tXFoI2vS(28CD-Z z>z?Dds`FLoeyY=L#ioaQr|bGse{o~2DKguS;Yo)ieMmnuR&eb@ZE*#~vfW$H6wo^F zSz~a>%kCI)kDW(4<00{zj|C?eNA--CUT^#h(@`)u-ElaPPwL0|RWIW&TEMc|rkkbM zY1c}x9m-n|+_6^SlzM-k2CdH;JWjBOs2b0rbt|1iTl*ZgR}w)!Z0+NAR%au#N;cpy zI8aY>#d*70(?+3oH2YF+?)PR{M*$8yXXWkrnyDx#yk{kt$-T)ZhT7b^)}M0sSQ6XM zvwkob6$hV~OYk_676{o-tB}fqE*x4 zmNv>m0>v8r>?q{t*DP>P&vVjg+>ZvEPWA zjLRaW{5S6o;lnCG6L3DcIUkjIP@jm3`Xl2unPZ@S&D~R@>DK9MWo6UzK?F(JR3-rb z0KciteC&|oM*7yIV}*ku%el4|Xj{EtG%RVr1epT4WhdH~mAI~H8iWF6s3MV{Ax1EH zW7vPKW3bI`++}g@X+Tv}w*xzUN$zW^HNX^{LYGlP18Yk|pCaW(?0I2`P(2xWC(sP% z`qE>H+zbm$rLa7QC5q$>VDf!3Mn6BDHG#o63hV|u|G!j_qE)VZ5L>^Az>LZgR?)KacY7eK+eIu^3v^2LjTL@Lp z?#aR0bK78Q1oSktcL-bg=-wpM!QhT49k6~tC? z!*cO6-C;yxSexK~W9CN~<2}I{$f&RE@1pCsy;7F_I+3l6UQCy_xBmd$q9G2=6l9E@ z%Gf^r_JLPjj)$T3KDX386?)=JDP@B3mKIWAkw|^N;UnTwJK&F%SN{ODPe{q3*>v86 z(^_rK#7Py+?Dq967oIqp4OV{>hL0^Qlq4a5=> zQb=T35cBBDNI30XiQ2q@`#SZ$snWe8C9SojO=%#KNp7xp2rX5NX#|hA-Tc)9jleHF zRj1(>ixcV}@ob$@t1R|9R+zepnPOO;6#%4&Hi6uMx4sXrpoUs`Cejf-hR@pHujaS4 zhr?|a>&d#*;qg=`osy(7<|lLtTHAL6er+&mMyM-)MAVjCcj7}~!708`x3+WxtD zt=?)n{-G?7f2>>=b0d~zh;WIM$-;#nl{CvMDiE!PJZ|>)HF}D>QcIIk6{;)Dy|T^? zUgyh{#c8o{aK(>v>?kZQHoA!v#`(&UG6^2u^cNazn=>4Nf$R@)TrtHhxjW=B4^iA2 z){TCd&+0WU_TF~zi2;FGNr;XK!Cn}T;XeF!q~(JHapaNw>z>n4iv!I&nNBya!-4OU zPB2Rp@$Zd;1MG2+txp^>gqw(tk0`O#EiY9`FiMhPP6+_=2ew8 z89SQVI}LK$BvJnE6bd*PP(R<+nDqd=jwAiy^$m_c&ZQTz417!qfxsQV@$;zesE@b4 z(egjj{b|-LErhMM_dQ+v4{Hy5;bx_zNX3qot2mE7k<5ZXKAVI7wR^FzEB?Xv#Wk+6 z(zEcrt)h(8a1yADuz94zbnb$n!S+4P|9r>6%I$6LCjZ zR3q#&&9CZn7Ts{eKGnx%CC;aN_IJ2s&*H{SWo2}_T|G(d6f;*jS6l6P9=d`%E|s>j zXl%vh&GcLR*_`1y_8gzo50!d9_BFn^)Vw~`n(NClF0CSmSk(z%DHsE^41jZt_Q|ZQ zu7^aDTOL8yH8$B}?v_ivYPtQU?BTfZJE$d)e0FVbbrJS>?o%JAW*?npWnP~Q%a13J zc7dO*55}N0QZ#Y~_#Jzr4nq%^ARo%Iva;U=)7!lnsY|W&mW!?2BBc73oo#t4uVC(g zwmw+zzlE(Gq*g|wXAA4uwLk~T0AJ1TAXlP`H`0`8$~OS zrz2}+7oL51=CZPp3ds_V0jO_U^)&q%9;c|xC)Hw+qm~`E)+MirYnJsj_l+&%4a2|Vyn;(!0DkeIYsyeqtdu>BnHaaz`VRWr71+=%@ z-)zc~SttrbKmLUj?kUB`urdUKY`(exZM=t*WRy^nDPWBcy)-aoRBMLYf92_4? z%F2O~$C1+5gYa!oBOGlQ&2(9^xZ3C#V0zY8Q%4HivqxF3~iqfHRLl$7LRXaambb=vx)-4mW(mX5?EUV_o*v46lNKQ%aNjc9Rme;haU1vemH2cls z86#+!qiwFa7~HHmR{4s`%7)#nSoRh_gpTN43DY`XNlVh!x_BCO72W{ zB&~nTAIP9}LQUjQ=L(R5KRV#LmDG$MZ|7NAT6%xPjUK800E$4Y(giN8K^#`#m+cu@ zP&cM)D=Spv*OQX+);Oe6y^9f#Y8yM5EgVS}JMs@=52a;gMGwSE_e7F-#_&PLK(5;u zS8fkD8SPnFR+lA5!1H4oA!Xa0y@{s=k8E5Xaz=4kSyIG-oL6cI$8DoI_o)7;fTQ>1 z6IoeJvw}*wEx*`mQobtda3}#M(3zMILX*Tt^)>7U?^#({^5oKo0)DhUm6epi!1_?P dYRbw01KNbvR!|1LD1QxQWdH%JtgNR2|JkJStOfu8 literal 0 HcmV?d00001 diff --git a/ggml/examples/sam/sam.cpp b/ggml/examples/sam/sam.cpp new file mode 100644 index 0000000..e5a58dc --- /dev/null +++ b/ggml/examples/sam/sam.cpp @@ -0,0 +1,2370 @@ +#define _USE_MATH_DEFINES // for M_PI +#define _CRT_SECURE_NO_DEPRECATE // Disables ridiculous "unsafe" warnigns on Windows + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +// default hparams (ViT-B SAM) +struct sam_hparams { + int32_t n_enc_state = 768; + int32_t n_enc_layer = 12; + int32_t n_enc_head = 12; + int32_t n_enc_out_chans = 256; + int32_t n_pt_embd = 4; + int32_t n_dec_heads = 8; + int32_t ftype = 1; + float mask_threshold = 0.f; + float iou_threshold = 0.88f; + float stability_score_threshold = 0.95f; + float stability_score_offset = 1.0f; + float eps = 1e-6f; + float eps_decoder_transformer = 1e-5f; + + int32_t n_enc_head_dim() const { return n_enc_state / n_enc_head; } + int32_t n_img_size() const { return 1024; } + int32_t n_window_size() const { return 14; } + int32_t n_patch_size() const { return 16; } + int32_t n_img_embd() const { return n_img_size() / n_patch_size(); } + + std::vector global_attn_indices() const { + switch (n_enc_state) { + case 768: return { 2, 5, 8, 11 }; + case 1024: return { 5, 11, 17, 23 }; + case 1280: return { 7, 15, 23, 31 }; + default: + { + fprintf(stderr, "%s: unsupported n_enc_state = %d\n", __func__, n_enc_state); + } break; + }; + + return {}; + } + + bool is_global_attn(int32_t layer) const { + const auto indices = global_attn_indices(); + + for (const auto & idx : indices) { + if (layer == idx) { + return true; + } + } + + return false; + } +}; + +struct sam_layer_enc { + struct ggml_tensor * norm1_w; + struct ggml_tensor * norm1_b; + + struct ggml_tensor * rel_pos_w; + struct ggml_tensor * rel_pos_h; + + struct ggml_tensor * qkv_w; + struct ggml_tensor * qkv_b; + + struct ggml_tensor * proj_w; + struct ggml_tensor * proj_b; + + struct ggml_tensor * norm2_w; + struct ggml_tensor * norm2_b; + + struct ggml_tensor * mlp_lin1_w; + struct ggml_tensor * mlp_lin1_b; + + struct ggml_tensor * mlp_lin2_w; + struct ggml_tensor * mlp_lin2_b; +}; + +struct sam_encoder_image { + struct ggml_tensor * pe; + + struct ggml_tensor * proj_w; + struct ggml_tensor * proj_b; + + struct ggml_tensor * neck_conv_0; + struct ggml_tensor * neck_norm_0_w; + struct ggml_tensor * neck_norm_0_b; + struct ggml_tensor * neck_conv_1; + struct ggml_tensor * neck_norm_1_w; + struct ggml_tensor * neck_norm_1_b; + + std::vector layers; +}; + +struct sam_encoder_prompt { + struct ggml_tensor * pe; + + struct ggml_tensor * not_a_pt_embd_w; + std::vector pt_embd; + + struct ggml_tensor * no_mask_embd_w; + //std::vector mask_down_w; + //std::vector mask_down_b; +}; + +struct sam_layer_dec_transformer_attn { + // q_proj + struct ggml_tensor * q_w; + struct ggml_tensor * q_b; + + // k_proj + struct ggml_tensor * k_w; + struct ggml_tensor * k_b; + + // v_proj + struct ggml_tensor * v_w; + struct ggml_tensor * v_b; + + // out_proj + struct ggml_tensor * out_w; + struct ggml_tensor * out_b; +}; + +struct sam_layer_dec_transformer { + sam_layer_dec_transformer_attn self_attn; + + // norm1 + struct ggml_tensor * norm1_w; + struct ggml_tensor * norm1_b; + + sam_layer_dec_transformer_attn cross_attn_token_to_img; + + // norm2 + struct ggml_tensor * norm2_w; + struct ggml_tensor * norm2_b; + + // mlp.lin1 + struct ggml_tensor * mlp_lin1_w; + struct ggml_tensor * mlp_lin1_b; + + // mlp.lin2 + struct ggml_tensor * mlp_lin2_w; + struct ggml_tensor * mlp_lin2_b; + + // norm3 + struct ggml_tensor * norm3_w; + struct ggml_tensor * norm3_b; + + // norm4 + struct ggml_tensor * norm4_w; + struct ggml_tensor * norm4_b; + + sam_layer_dec_transformer_attn cross_attn_img_to_token; +}; + +struct sam_layer_dec_output_hypernet_mlps { + // mlps_*.layers.0 + struct ggml_tensor * w_0; + struct ggml_tensor * b_0; + + // mlps_*.layers.1 + struct ggml_tensor * w_1; + struct ggml_tensor * b_1; + + // mlps_*.layers.2 + struct ggml_tensor * w_2; + struct ggml_tensor * b_2; +}; + +struct sam_decoder_mask { + std::vector transformer_layers; + + // trasnformer.final_attn_token_to_image + sam_layer_dec_transformer_attn transformer_final_attn_token_to_img; + + // transformer.norm_final + struct ggml_tensor * transformer_norm_final_w; + struct ggml_tensor * transformer_norm_final_b; + + // output_upscaling.0 + struct ggml_tensor * output_upscaling_0_w; + struct ggml_tensor * output_upscaling_0_b; + + // output_upscaling.1 + struct ggml_tensor * output_upscaling_1_w; + struct ggml_tensor * output_upscaling_1_b; + + // output_upscaling.3 + struct ggml_tensor * output_upscaling_3_w; + struct ggml_tensor * output_upscaling_3_b; + + // output_hypernetworks_mlps + std::vector output_hypernet_mlps; + + // iou_prediction_head.0 + struct ggml_tensor * iou_prediction_head_0_w; + struct ggml_tensor * iou_prediction_head_0_b; + + // iou_prediction_head.1 + struct ggml_tensor * iou_prediction_head_1_w; + struct ggml_tensor * iou_prediction_head_1_b; + + // iou_prediction_head.2 + struct ggml_tensor * iou_prediction_head_2_w; + struct ggml_tensor * iou_prediction_head_2_b; + + // iou_token.weight + struct ggml_tensor * iou_token_w; + + // mask_tokens.weight + struct ggml_tensor * mask_tokens_w; +}; + + +struct sam_state { + struct ggml_tensor * embd_img; + + struct ggml_tensor * low_res_masks; + struct ggml_tensor * iou_predictions; + + //struct ggml_tensor * tmp_save = {}; + + struct ggml_context * ctx; + + // buffer for `ggml_graph_plan.work_data` + std::vector work_buffer; + // buffers to evaluate the model + std::vector buf_compute_img_enc; + + std::vector buf_compute_fast; + + ggml_gallocr_t allocr = {}; +}; + +// void save_tensor(sam_state& state, struct ggml_tensor * t, struct ggml_cgraph * gf) { +// if (!state.tmp_save) { +// state.tmp_save = ggml_new_tensor(state.ctx, t->type, t->n_dims, t->ne); +// } +// struct ggml_tensor * tmp0 = ggml_cpy(state.ctx, t, state.tmp_save); +// ggml_build_forward_expand(gf, tmp0); +// } + +struct sam_model { + sam_hparams hparams; + + sam_encoder_image enc_img; + sam_encoder_prompt enc_prompt; + sam_decoder_mask dec; + + // + struct ggml_context * ctx; + std::map tensors; +}; + +struct sam_point { + float x; + float y; +}; + +struct sam_box { + float x1; + float y1; + float x2; + float y2; +}; + +// RGB uint8 image +struct sam_image_u8 { + int nx; + int ny; + + std::vector data; +}; + +// RGB float32 image +// Memory layout: RGBRGBRGB... +struct sam_image_f32 { + int nx; + int ny; + + std::vector data; +}; + +enum sam_prompt_type { + SAM_PROMPT_TYPE_POINT = 0, + SAM_PROMPT_TYPE_BOX = 1, +}; + +struct sam_prompt { + sam_prompt_type prompt_type = SAM_PROMPT_TYPE_POINT; + sam_point pt = { 414.375f, 162.796875f, }; + sam_box box = { 368.0f, 144.0f, 441.0f, 173.0f }; +}; + +struct sam_params { + int32_t seed = -1; // RNG seed + int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); + + std::string model = "models/sam-vit-b/ggml-model-f16.bin"; // model path + std::string fname_inp = "img.jpg"; + std::string fname_out = "img.out"; + float mask_threshold = 0.f; + float iou_threshold = 0.88f; + float stability_score_threshold = 0.95f; + float stability_score_offset = 1.0f; + float eps = 1e-6f; + float eps_decoder_transformer = 1e-5f; + + sam_prompt prompt; + bool multimask_output = true; +}; + +void print_t_f32(const char* title, struct ggml_tensor * t, int n = 10) { + printf("%s\n", title); + float * data = (float *)t->data; + printf("dims: % " PRId64 " % " PRId64 " % " PRId64 " % " PRId64 " f32\n", t->ne[0], t->ne[1], t->ne[2], t->ne[3]); + printf("First & Last %d elements:\n", n); + for (int i = 0; i < std::min((int) (t->ne[0]*t->ne[1]), n); i++) { + printf("%.5f ", data[i]); + if (i != 0 && i % t->ne[0] == 0) { + printf("\n"); + } + } + printf("\n"); + for (int i = 0; i < std::min((int) (t->ne[0]*t->ne[1]), n); i++) { + printf("%.5f ", data[ggml_nelements(t) - n + i]); + if ((ggml_nelements(t) - n + i) % t->ne[0] == 0) { + printf("\n"); + } + } + printf("\n"); + double sum = 0.0; + for (int i = 0; i < ggml_nelements(t); i++) { + sum += data[i]; + } + printf("sum: %f\n\n", sum); +} + +static void ggml_disconnect_node_from_graph(ggml_tensor * t) { + t->op = GGML_OP_NONE; + for (int i = 0; i < GGML_MAX_SRC; i++) { + t->src[i] = NULL; + } +} + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +static void ggml_sam_sin(struct ggml_tensor * dst , const struct ggml_tensor * src, int ith, int nth, void * userdata) { + GGML_ASSERT(userdata == NULL); + GGML_ASSERT(ggml_are_same_shape(dst, src)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous(src)); + + const float * src_data = ggml_get_data_f32(src); + float * dst_data = ggml_get_data_f32(dst); + + const int ne = (int)ggml_nelements(dst); + const int dr = (ne + nth - 1) / nth; + const int ie0 = dr * ith; + const int ie1 = std::min(ie0 + dr, ne); + + for (int i = ie0; i < ie1; ++i) { + dst_data[i] = sinf(src_data[i]); + } +} + +static void ggml_sam_cos(struct ggml_tensor * dst , const struct ggml_tensor * src, int ith, int nth, void * userdata) { + GGML_ASSERT(userdata == NULL); + GGML_ASSERT(ggml_are_same_shape(dst, src)); + GGML_ASSERT(ggml_is_contiguous(dst)); + GGML_ASSERT(ggml_is_contiguous(src)); + + const float * src_data = ggml_get_data_f32(src); + float * dst_data = ggml_get_data_f32(dst); + + const int ne = (int)ggml_nelements(dst); + const int dr = (ne + nth - 1) / nth; + const int ie0 = dr * ith; + const int ie1 = std::min(ie0 + dr, ne); + + for (int i = ie0; i < ie1; ++i) { + dst_data[i] = cosf(src_data[i]); + } +} + +bool sam_image_load_from_file(const std::string & fname, sam_image_u8 & img) { + int nx, ny, nc; + auto data = stbi_load(fname.c_str(), &nx, &ny, &nc, 3); + if (!data) { + fprintf(stderr, "%s: failed to load '%s'\n", __func__, fname.c_str()); + return false; + } + + img.nx = nx; + img.ny = ny; + img.data.resize(nx * ny * 3); + memcpy(img.data.data(), data, nx * ny * 3); + + stbi_image_free(data); + + return true; +} + +// ref: https://github.com/facebookresearch/segment-anything/blob/efeab7296ab579d4a261e554eca80faf6b33924a/segment_anything/modeling/sam.py#L164 +// resize largest dimension to 1024 +// normalize: x = (x - mean) / std +// mean = [123.675, 116.28, 103.53] +// std = [58.395, 57.12, 57.375] +// TODO: why are these hardcoded !? +// pad to 1024x1024 +// TODO: for some reason, this is not numerically identical to pytorch's interpolation +bool sam_image_preprocess(const sam_image_u8 & img, sam_image_f32 & res) { + const int nx = img.nx; + const int ny = img.ny; + + const int nx2 = 1024; + const int ny2 = 1024; + + res.nx = nx2; + res.ny = ny2; + res.data.resize(3*nx2*ny2); + + const float scale = std::max(nx, ny) / 1024.0f; + + fprintf(stderr, "%s: scale = %f\n", __func__, scale); + + const int nx3 = int(nx/scale + 0.5f); + const int ny3 = int(ny/scale + 0.5f); + + const float m3[3] = { 123.675f, 116.280f, 103.530f }; + const float s3[3] = { 58.395f, 57.120f, 57.375f }; + + for (int y = 0; y < ny3; y++) { + for (int x = 0; x < nx3; x++) { + for (int c = 0; c < 3; c++) { + // linear interpolation + const float sx = (x + 0.5f)*scale - 0.5f; + const float sy = (y + 0.5f)*scale - 0.5f; + + const int x0 = std::max(0, (int) std::floor(sx)); + const int y0 = std::max(0, (int) std::floor(sy)); + + const int x1 = std::min(x0 + 1, nx - 1); + const int y1 = std::min(y0 + 1, ny - 1); + + const float dx = sx - x0; + const float dy = sy - y0; + + const int j00 = 3*(y0*nx + x0) + c; + const int j01 = 3*(y0*nx + x1) + c; + const int j10 = 3*(y1*nx + x0) + c; + const int j11 = 3*(y1*nx + x1) + c; + + const float v00 = img.data[j00]; + const float v01 = img.data[j01]; + const float v10 = img.data[j10]; + const float v11 = img.data[j11]; + + const float v0 = v00*(1.0f - dx) + v01*dx; + const float v1 = v10*(1.0f - dx) + v11*dx; + + const float v = v0*(1.0f - dy) + v1*dy; + + const uint8_t v2 = std::min(std::max(std::round(v), 0.0f), 255.0f); + + const int i = 3*(y*nx3 + x) + c; + + res.data[i] = (float(v2) - m3[c]) / s3[c]; + } + } + } + + return true; +} + +// load the model's weights from a file +bool sam_model_load(const sam_params & params, sam_model & model) { + fprintf(stderr, "%s: loading model from '%s' - please wait ...\n", __func__, params.model.c_str()); + + auto fin = std::ifstream(params.model, std::ios::binary); + if (!fin) { + fprintf(stderr, "%s: failed to open '%s'\n", __func__, params.model.c_str()); + return false; + } + + // verify magic + { + uint32_t magic; + fin.read((char *) &magic, sizeof(magic)); + if (magic != 0x67676d6c) { + fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, params.model.c_str()); + return false; + } + } + + // load hparams + { + // Override defaults with user choices + model.hparams.mask_threshold = params.mask_threshold; + model.hparams.iou_threshold = params.iou_threshold; + model.hparams.stability_score_threshold = params.stability_score_threshold; + model.hparams.stability_score_offset = params.stability_score_offset; + model.hparams.eps = params.eps; + model.hparams.eps_decoder_transformer = params.eps_decoder_transformer; + + auto & hparams = model.hparams; + + fin.read((char *) &hparams.n_enc_state, sizeof(hparams.n_enc_state)); + fin.read((char *) &hparams.n_enc_layer, sizeof(hparams.n_enc_layer)); + fin.read((char *) &hparams.n_enc_head, sizeof(hparams.n_enc_head)); + fin.read((char *) &hparams.n_enc_out_chans, sizeof(hparams.n_enc_out_chans)); + fin.read((char *) &hparams.n_pt_embd, sizeof(hparams.n_pt_embd)); + fin.read((char *) &hparams.ftype, sizeof(hparams.ftype)); + + const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR; + + printf("%s: n_enc_state = %d\n", __func__, hparams.n_enc_state); + printf("%s: n_enc_layer = %d\n", __func__, hparams.n_enc_layer); + printf("%s: n_enc_head = %d\n", __func__, hparams.n_enc_head); + printf("%s: n_enc_out_chans = %d\n", __func__, hparams.n_enc_out_chans); + printf("%s: n_pt_embd = %d\n", __func__, hparams.n_pt_embd); + printf("%s: ftype = %d\n", __func__, hparams.ftype); + printf("%s: qntvr = %d\n", __func__, qntvr); + + hparams.ftype %= GGML_QNT_VERSION_FACTOR; + + } + + // for the big tensors, we have the option to store the data in 16-bit floats or quantized + // in order to save memory and also to speed up the computation + ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype)); + if (wtype == GGML_TYPE_COUNT) { + fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n", + __func__, params.model.c_str(), model.hparams.ftype); + return false; + } + + auto & ctx = model.ctx; + + const size_t ctx_size = [&]() { + size_t ctx_size = 0; + + const auto & hparams = model.hparams; + + const int32_t n_enc_state = hparams.n_enc_state; + const int32_t n_enc_layer = hparams.n_enc_layer; + const int32_t n_enc_head_dim = hparams.n_enc_head_dim(); + const int32_t n_enc_out_chans = hparams.n_enc_out_chans; + const int32_t n_pt_embd = hparams.n_pt_embd; + + const int32_t n_enc_layer_local = hparams.global_attn_indices().size(); + const int32_t n_enc_layer_global = n_enc_layer - n_enc_layer_local; + + const int32_t n_img_embd = hparams.n_img_embd(); + const int32_t n_window_size = hparams.n_window_size(); + const int32_t n_patch_size = hparams.n_patch_size(); + + // image encoder + { + ctx_size += n_enc_state*n_img_embd*n_img_embd*ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_state*3*n_patch_size*n_patch_size*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_state*ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_state*n_enc_out_chans*1*1*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_out_chans*n_enc_out_chans*3*3*ggml_type_size(GGML_TYPE_F16); + + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + } + + // image encoder layers + { + ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_layer_global*n_enc_head_dim*(2*n_img_embd - 1)*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer_global*n_enc_head_dim*(2*n_img_embd - 1)*ggml_type_size(GGML_TYPE_F16); + + ctx_size += n_enc_layer_local*n_enc_head_dim*(2*n_window_size - 1)*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer_local*n_enc_head_dim*(2*n_window_size - 1)*ggml_type_size(GGML_TYPE_F16); + + ctx_size += n_enc_layer*3*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer*3*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_layer*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_layer*n_enc_state*ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_layer*4*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer*4*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + ctx_size += n_enc_layer*4*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_enc_layer*4*n_enc_state* ggml_type_size(GGML_TYPE_F32); + } + + ctx_size += (8 + 14*n_enc_layer)*ggml_tensor_overhead(); + + // prompt encoder + { + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); // 2*(n_enc_out_chans/2) + + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + } + + ctx_size += (2 + n_pt_embd)*ggml_tensor_overhead(); + + // mask decoder + { + //transformer + { + const int tfm_layers_count = 2; + const int qkv_count = 3; + const int norm_count = 4; + const int n_hypernet_mpls_count = 4; + + // self_attn + ctx_size += tfm_layers_count*qkv_count*n_enc_state*n_enc_state*ggml_type_size(GGML_TYPE_F16); + ctx_size += tfm_layers_count*qkv_count*n_enc_state* ggml_type_size(GGML_TYPE_F32); + ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + // all norms + ctx_size += tfm_layers_count*norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32); + ctx_size += tfm_layers_count*norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32); + + // cross_attn_token_to_img + ctx_size += tfm_layers_count*qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16); + ctx_size += tfm_layers_count*qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32); + ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + // mlp + ctx_size += tfm_layers_count*8*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); + ctx_size += tfm_layers_count*8*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32); + ctx_size += tfm_layers_count*n_enc_out_chans*8*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); + ctx_size += tfm_layers_count*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32); + + // cross_attn_img_to_token + ctx_size += tfm_layers_count*qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16); + ctx_size += tfm_layers_count*qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32); + ctx_size += tfm_layers_count*n_enc_state* ggml_type_size(GGML_TYPE_F32); + + // transformer_final_attn_token_to_img + ctx_size += qkv_count*n_enc_state*(n_enc_state/2)*ggml_type_size(GGML_TYPE_F16); + ctx_size += qkv_count*(n_enc_state/2)* ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_state* ggml_type_size(GGML_TYPE_F32); + + // transformer_norm_final + ctx_size += norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32); + ctx_size += norm_count*n_enc_state*ggml_type_size(GGML_TYPE_F32); + + // output_upscaling + ctx_size += n_enc_out_chans*n_img_embd*2*2*ggml_type_size(GGML_TYPE_F16); + ctx_size += 3*n_img_embd* ggml_type_size(GGML_TYPE_F32); + ctx_size += n_enc_out_chans*n_img_embd*(n_img_embd/2)*2*2*ggml_type_size(GGML_TYPE_F16); + ctx_size += (n_img_embd/2)* ggml_type_size(GGML_TYPE_F32); + + // output_hypernetworks_mlps + ctx_size += n_hypernet_mpls_count*2*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_hypernet_mpls_count*2*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32); + ctx_size += n_hypernet_mpls_count*n_enc_out_chans*(n_img_embd/2)*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_hypernet_mpls_count*(n_img_embd/2)* ggml_type_size(GGML_TYPE_F32); + + // iou_prediction_head + ctx_size += 2*n_enc_out_chans*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); + ctx_size += 2*n_enc_out_chans* ggml_type_size(GGML_TYPE_F32); + ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F16); + ctx_size += n_pt_embd* ggml_type_size(GGML_TYPE_F32); + + // iou_token_w + ctx_size += n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + + // mask_tokens_w + ctx_size += n_pt_embd*n_enc_out_chans*ggml_type_size(GGML_TYPE_F32); + } + } + fprintf(stderr, "%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0)); + + return ctx_size; + }(); + + // create the ggml context + { + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return false; + } + } + + // prepare memory for the weights + { + const auto & hparams = model.hparams; + + const int32_t n_enc_state = hparams.n_enc_state; + const int32_t n_enc_layer = hparams.n_enc_layer; + const int32_t n_enc_head_dim = hparams.n_enc_head_dim(); + const int32_t n_enc_out_chans = hparams.n_enc_out_chans; + const int32_t n_pt_embd = hparams.n_pt_embd; + + const int32_t n_img_embd = hparams.n_img_embd(); + const int32_t n_window_size = hparams.n_window_size(); + const int32_t n_patch_size = hparams.n_patch_size(); + + model.enc_img.layers.resize(n_enc_layer); + + // image encoder + { + auto & enc = model.enc_img; + + enc.pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_enc_state, n_img_embd, n_img_embd, 1); + + enc.proj_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, n_patch_size, n_patch_size, 3, n_enc_state); + enc.proj_b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, 1, n_enc_state); + + enc.neck_conv_0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 1, 1, n_enc_state, n_enc_out_chans); + enc.neck_conv_1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 3, 3, n_enc_out_chans, n_enc_out_chans); + + enc.neck_norm_0_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + enc.neck_norm_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + enc.neck_norm_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + enc.neck_norm_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + model.tensors["image_encoder.pos_embed"] = enc.pe; + + model.tensors["image_encoder.patch_embed.proj.weight"] = enc.proj_w; + model.tensors["image_encoder.patch_embed.proj.bias"] = enc.proj_b; + + model.tensors["image_encoder.neck.0.weight"] = enc.neck_conv_0; + model.tensors["image_encoder.neck.2.weight"] = enc.neck_conv_1; + + model.tensors["image_encoder.neck.1.weight"] = enc.neck_norm_0_w; + model.tensors["image_encoder.neck.1.bias"] = enc.neck_norm_0_b; + + model.tensors["image_encoder.neck.3.weight"] = enc.neck_norm_1_w; + model.tensors["image_encoder.neck.3.bias"] = enc.neck_norm_1_b; + + for (int i = 0; i < n_enc_layer; ++i) { + auto & layer = enc.layers[i]; + + layer.norm1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + layer.norm1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + + if (hparams.is_global_attn(i)) { + layer.rel_pos_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_img_embd - 1); + layer.rel_pos_h = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_img_embd - 1); + } else { + layer.rel_pos_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_window_size - 1); + layer.rel_pos_h = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_head_dim, 2*n_window_size - 1); + } + + layer.qkv_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, 3*n_enc_state); + layer.qkv_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_enc_state); + + layer.proj_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, n_enc_state); + layer.proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + + layer.norm2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + layer.norm2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + + layer.mlp_lin1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_state, 4*n_enc_state); + layer.mlp_lin1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_enc_state); + + layer.mlp_lin2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 4*n_enc_state, n_enc_state); + layer.mlp_lin2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_state); + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".norm1.weight"] = layer.norm1_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".norm1.bias"] = layer.norm1_b; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.rel_pos_w"] = layer.rel_pos_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.rel_pos_h"] = layer.rel_pos_h; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.qkv.weight"] = layer.qkv_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.qkv.bias"] = layer.qkv_b; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.proj.weight"] = layer.proj_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".attn.proj.bias"] = layer.proj_b; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".norm2.weight"] = layer.norm2_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".norm2.bias"] = layer.norm2_b; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".mlp.lin1.weight"] = layer.mlp_lin1_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".mlp.lin1.bias"] = layer.mlp_lin1_b; + + model.tensors["image_encoder.blocks." + std::to_string(i) + ".mlp.lin2.weight"] = layer.mlp_lin2_w; + model.tensors["image_encoder.blocks." + std::to_string(i) + ".mlp.lin2.bias"] = layer.mlp_lin2_b; + } + } + + // prompt encoder + { + auto & enc = model.enc_prompt; + + enc.pe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans/2, 2); + + enc.not_a_pt_embd_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + enc.no_mask_embd_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + model.tensors["prompt_encoder.pe_layer.positional_encoding_gaussian_matrix"] = enc.pe; + model.tensors["prompt_encoder.not_a_point_embed.weight"] = enc.not_a_pt_embd_w; + model.tensors["prompt_encoder.no_mask_embed.weight"] = enc.no_mask_embd_w; + + enc.pt_embd.resize(n_pt_embd); + for (int i = 0; i < n_pt_embd; i++) { + enc.pt_embd[i] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + model.tensors["prompt_encoder.point_embeddings." + std::to_string(i) + ".weight"] = enc.pt_embd[i]; + } + } + + // mask decoder + { + auto & dec = model.dec; + auto & tfm_layers = dec.transformer_layers; + + const int tfm_layers_count = 2; + tfm_layers.resize(tfm_layers_count); + for (int i = 0; i < tfm_layers_count; ++i) { + auto& l = tfm_layers[i]; + l.self_attn.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + l.self_attn.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.self_attn.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + l.self_attn.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.self_attn.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + l.self_attn.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.self_attn.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + l.self_attn.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.norm1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.norm1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.cross_attn_token_to_img.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_token_to_img.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_token_to_img.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_token_to_img.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_token_to_img.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_token_to_img.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_token_to_img.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans); + l.cross_attn_token_to_img.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.norm2_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.norm2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.mlp_lin1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, 8*n_enc_out_chans); + l.mlp_lin1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8*n_enc_out_chans); + l.mlp_lin2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, 8*n_enc_out_chans, n_enc_out_chans); + l.mlp_lin2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.norm3_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.norm3_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.norm4_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + l.norm4_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + l.cross_attn_img_to_token.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_img_to_token.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_img_to_token.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_img_to_token.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_img_to_token.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + l.cross_attn_img_to_token.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + l.cross_attn_img_to_token.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans); + l.cross_attn_img_to_token.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + const auto prefix = "mask_decoder.transformer.layers." + std::to_string(i) + "."; + model.tensors[prefix + "self_attn.q_proj.weight"] = l.self_attn.q_w; + model.tensors[prefix + "self_attn.q_proj.bias"] = l.self_attn.q_b; + model.tensors[prefix + "self_attn.k_proj.weight"] = l.self_attn.k_w; + model.tensors[prefix + "self_attn.k_proj.bias"] = l.self_attn.k_b; + model.tensors[prefix + "self_attn.v_proj.weight"] = l.self_attn.v_w; + model.tensors[prefix + "self_attn.v_proj.bias"] = l.self_attn.v_b; + model.tensors[prefix + "self_attn.out_proj.weight"] = l.self_attn.out_w; + model.tensors[prefix + "self_attn.out_proj.bias"] = l.self_attn.out_b; + + model.tensors[prefix + "norm1.weight"] = l.norm1_w; + model.tensors[prefix + "norm1.bias"] = l.norm1_b; + + model.tensors[prefix + "cross_attn_token_to_image.q_proj.weight"] = l.cross_attn_token_to_img.q_w; + model.tensors[prefix + "cross_attn_token_to_image.q_proj.bias"] = l.cross_attn_token_to_img.q_b; + model.tensors[prefix + "cross_attn_token_to_image.k_proj.weight"] = l.cross_attn_token_to_img.k_w; + model.tensors[prefix + "cross_attn_token_to_image.k_proj.bias"] = l.cross_attn_token_to_img.k_b; + model.tensors[prefix + "cross_attn_token_to_image.v_proj.weight"] = l.cross_attn_token_to_img.v_w; + model.tensors[prefix + "cross_attn_token_to_image.v_proj.bias"] = l.cross_attn_token_to_img.v_b; + model.tensors[prefix + "cross_attn_token_to_image.out_proj.weight"] = l.cross_attn_token_to_img.out_w; + model.tensors[prefix + "cross_attn_token_to_image.out_proj.bias"] = l.cross_attn_token_to_img.out_b; + + model.tensors[prefix + "norm2.weight"] = l.norm2_w; + model.tensors[prefix + "norm2.bias"] = l.norm2_b; + + model.tensors[prefix + "mlp.lin1.weight"] = l.mlp_lin1_w; + model.tensors[prefix + "mlp.lin1.bias"] = l.mlp_lin1_b; + model.tensors[prefix + "mlp.lin2.weight"] = l.mlp_lin2_w; + model.tensors[prefix + "mlp.lin2.bias"] = l.mlp_lin2_b; + + model.tensors[prefix + "norm3.weight"] = l.norm3_w; + model.tensors[prefix + "norm3.bias"] = l.norm3_b; + model.tensors[prefix + "norm4.weight"] = l.norm4_w; + model.tensors[prefix + "norm4.bias"] = l.norm4_b; + + model.tensors[prefix + "cross_attn_image_to_token.q_proj.weight"] = l.cross_attn_img_to_token.q_w; + model.tensors[prefix + "cross_attn_image_to_token.q_proj.bias"] = l.cross_attn_img_to_token.q_b; + model.tensors[prefix + "cross_attn_image_to_token.k_proj.weight"] = l.cross_attn_img_to_token.k_w; + model.tensors[prefix + "cross_attn_image_to_token.k_proj.bias"] = l.cross_attn_img_to_token.k_b; + model.tensors[prefix + "cross_attn_image_to_token.v_proj.weight"] = l.cross_attn_img_to_token.v_w; + model.tensors[prefix + "cross_attn_image_to_token.v_proj.bias"] = l.cross_attn_img_to_token.v_b; + model.tensors[prefix + "cross_attn_image_to_token.out_proj.weight"] = l.cross_attn_img_to_token.out_w; + model.tensors[prefix + "cross_attn_image_to_token.out_proj.bias"] = l.cross_attn_img_to_token.out_b; + } + + dec.transformer_final_attn_token_to_img.q_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.q_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.k_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.k_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.v_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.v_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans/2); + dec.transformer_final_attn_token_to_img.out_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans/2, n_enc_out_chans); + dec.transformer_final_attn_token_to_img.out_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + model.tensors["mask_decoder.transformer.final_attn_token_to_image.q_proj.weight"] = dec.transformer_final_attn_token_to_img.q_w; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.q_proj.bias"] = dec.transformer_final_attn_token_to_img.q_b; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.k_proj.weight"] = dec.transformer_final_attn_token_to_img.k_w; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.k_proj.bias"] = dec.transformer_final_attn_token_to_img.k_b; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.v_proj.weight"] = dec.transformer_final_attn_token_to_img.v_w; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.v_proj.bias"] = dec.transformer_final_attn_token_to_img.v_b; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.out_proj.weight"] = dec.transformer_final_attn_token_to_img.out_w; + model.tensors["mask_decoder.transformer.final_attn_token_to_image.out_proj.bias"] = dec.transformer_final_attn_token_to_img.out_b; + + dec.transformer_norm_final_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + dec.transformer_norm_final_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + + model.tensors["mask_decoder.transformer.norm_final_attn.weight"] = dec.transformer_norm_final_w; + model.tensors["mask_decoder.transformer.norm_final_attn.bias"] = dec.transformer_norm_final_b; + + dec.output_upscaling_0_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 2, 2, n_img_embd, n_enc_out_chans); + dec.output_upscaling_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd); + dec.output_upscaling_1_w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd); + dec.output_upscaling_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd); + dec.output_upscaling_3_w = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, 2, 2, n_img_embd/2, n_img_embd); + dec.output_upscaling_3_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd/2); + + model.tensors["mask_decoder.output_upscaling.0.weight"] = dec.output_upscaling_0_w; + model.tensors["mask_decoder.output_upscaling.0.bias"] = dec.output_upscaling_0_b; + model.tensors["mask_decoder.output_upscaling.1.weight"] = dec.output_upscaling_1_w; + model.tensors["mask_decoder.output_upscaling.1.bias"] = dec.output_upscaling_1_b; + model.tensors["mask_decoder.output_upscaling.3.weight"] = dec.output_upscaling_3_w; + model.tensors["mask_decoder.output_upscaling.3.bias"] = dec.output_upscaling_3_b; + + const int n_hypernet_mpls_count = 4; + dec.output_hypernet_mlps.resize(n_hypernet_mpls_count); + for (int i = 0; i < n_hypernet_mpls_count; ++i) { + auto& mlp = dec.output_hypernet_mlps[i]; + + mlp.w_0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + mlp.b_0 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + mlp.w_1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + mlp.b_1 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + mlp.w_2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_img_embd/2); + mlp.b_2 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_img_embd/2); + + const auto prefix = "mask_decoder.output_hypernetworks_mlps." + std::to_string(i) + "."; + model.tensors[prefix + "layers.0.weight"] = mlp.w_0; + model.tensors[prefix + "layers.0.bias"] = mlp.b_0; + model.tensors[prefix + "layers.1.weight"] = mlp.w_1; + model.tensors[prefix + "layers.1.bias"] = mlp.b_1; + model.tensors[prefix + "layers.2.weight"] = mlp.w_2; + model.tensors[prefix + "layers.2.bias"] = mlp.b_2; + } + + dec.iou_prediction_head_0_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + dec.iou_prediction_head_0_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + dec.iou_prediction_head_1_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_enc_out_chans); + dec.iou_prediction_head_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_enc_out_chans); + dec.iou_prediction_head_2_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F16, n_enc_out_chans, n_pt_embd); + dec.iou_prediction_head_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_pt_embd); + + dec.iou_token_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans, 1); + dec.mask_tokens_w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_enc_out_chans, n_pt_embd); + + model.tensors["mask_decoder.iou_prediction_head.layers.0.weight"] = dec.iou_prediction_head_0_w; + model.tensors["mask_decoder.iou_prediction_head.layers.0.bias"] = dec.iou_prediction_head_0_b; + model.tensors["mask_decoder.iou_prediction_head.layers.1.weight"] = dec.iou_prediction_head_1_w; + model.tensors["mask_decoder.iou_prediction_head.layers.1.bias"] = dec.iou_prediction_head_1_b; + model.tensors["mask_decoder.iou_prediction_head.layers.2.weight"] = dec.iou_prediction_head_2_w; + model.tensors["mask_decoder.iou_prediction_head.layers.2.bias"] = dec.iou_prediction_head_2_b; + + model.tensors["mask_decoder.iou_token.weight"] = dec.iou_token_w; + model.tensors["mask_decoder.mask_tokens.weight"] = dec.mask_tokens_w; + } + } + + // load weights + { + int n_tensors = 0; + size_t total_size = 0; + + fprintf(stderr, "%s: ", __func__); + + while (true) { + int32_t n_dims; + int32_t length; + int32_t ftype; + + fin.read(reinterpret_cast(&n_dims), sizeof(n_dims)); + fin.read(reinterpret_cast(&length), sizeof(length)); + fin.read(reinterpret_cast(&ftype), sizeof(ftype)); + + if (fin.eof()) { + break; + } + + int64_t nelements = 1; + int64_t ne[4] = { 1, 1, 1, 1 }; + for (int i = 0; i < n_dims; ++i) { + int32_t ne_cur; + fin.read(reinterpret_cast(&ne_cur), sizeof(ne_cur)); + ne[i] = ne_cur; + nelements *= ne[i]; + } + + std::string name(length, 0); + fin.read(&name[0], length); + + if (model.tensors.find(name.data()) == model.tensors.end()) { + fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.data()); + return false; + } + + auto tensor = model.tensors[name.data()]; + //printf("ne0 = %jd, ne1 = %jd, ne2 = %jd, ne3 = %jd\n", ne[0], ne[1], ne[2], ne[3]); + + if (ggml_nelements(tensor) != nelements) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %d, expected %d\n", + __func__, name.data(), (int) nelements, (int) ggml_nelements(tensor)); + return false; + } + + if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1] || tensor->ne[2] != ne[2] || tensor->ne[3] != ne[3]) { + fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d, %d, %d], expected [%d, %d, %d, %d]\n", + __func__, name.data(), + (int) ne[0], (int) ne[1], (int) ne[2], (int) ne[3], + (int) tensor->ne[0], (int) tensor->ne[1], (int) tensor->ne[2], (int) tensor->ne[3]); + return false; + } + + size_t bpe = 0; + + switch (ftype) { + case 0: bpe = ggml_type_size(GGML_TYPE_F32); break; + case 1: bpe = ggml_type_size(GGML_TYPE_F16); break; + case 2: bpe = ggml_type_size(GGML_TYPE_Q4_0); assert(ne[0] % 64 == 0); break; + case 3: bpe = ggml_type_size(GGML_TYPE_Q4_1); assert(ne[0] % 64 == 0); break; + default: + { + fprintf(stderr, "%s: unknown ftype %d in model file\n", __func__, ftype); + return false; + } + }; + + if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) { + fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n", + __func__, name.data(), ggml_nbytes(tensor), (size_t) nelements*bpe); + return false; + } + + fin.read(reinterpret_cast(tensor->data), ggml_nbytes(tensor)); + + total_size += ggml_nbytes(tensor); + if (++n_tensors % 8 == 0) { + fprintf(stderr, "."); + fflush(stdout); + } + } + + if (n_tensors != int(model.tensors.size())) { + fprintf(stderr, "%s: model file has %d tensors, but %d tensors were expected\n", __func__, n_tensors, (int) model.tensors.size()); + return false; + } + + fprintf(stderr, " done\n"); + + fprintf(stderr, "%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors); + } + + fin.close(); + + return true; +} + +struct ggml_tensor * sam_fill_dense_pe( + const sam_model & model, + struct ggml_context * ctx0, + struct ggml_cgraph * gf, + sam_state & state) { + const auto & hparams = model.hparams; + const auto & enc = model.enc_prompt; + + + const int32_t n_img_embd = hparams.n_img_embd(); + struct ggml_tensor * xy_embed_stacked = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, n_img_embd, n_img_embd); + ggml_set_name(xy_embed_stacked, "xy_embed_stacked"); + ggml_set_input(xy_embed_stacked); + + struct ggml_tensor * cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, enc.pe)), xy_embed_stacked); + + cur = ggml_scale(ctx0, cur, float(2.0*M_PI)); + + // concat + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L192 + { + struct ggml_tensor * t_sin = ggml_map_custom1(ctx0, cur, ggml_sam_sin, GGML_N_TASKS_MAX, NULL); + struct ggml_tensor * t_cos = ggml_map_custom1(ctx0, cur, ggml_sam_cos, GGML_N_TASKS_MAX, NULL); + + cur = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, t_sin->ne[0] + t_cos->ne[0], cur->ne[1], cur->ne[2]); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_sin, ggml_view_3d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], t_sin->ne[2], cur->nb[1], cur->nb[2], 0))); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_cos, ggml_view_3d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], t_sin->ne[2], cur->nb[1], cur->nb[2], t_sin->nb[1]))); + } + + struct ggml_tensor * pe_img_dense = ggml_cont(ctx0, ggml_permute(ctx0, cur, 2, 0, 1, 3)); + ggml_build_forward_expand(gf, pe_img_dense); + + return pe_img_dense; +} + +struct ggml_tensor* sam_layer_norm_2d( + struct ggml_context * ctx0, + struct ggml_tensor * layer, + int n_channels, + struct ggml_tensor * w, + struct ggml_tensor * b, + float eps) { + // LayerNorm2d + // normalize along channel dimmension + // TODO: better implementation + layer = ggml_permute(ctx0, + ggml_norm(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, layer, 1, 2, 0, 3)), eps), + 2, 0, 1, 3); + + layer = ggml_add(ctx0, + ggml_mul(ctx0, + ggml_repeat(ctx0, ggml_reshape_3d(ctx0, w, 1, 1, n_channels), layer), + layer), + ggml_repeat(ctx0, ggml_reshape_3d(ctx0, b, 1, 1, n_channels), layer)); + + return layer; +} + +struct ggml_cgraph * sam_encode_image( + const sam_model & model, + sam_state & state, + const sam_image_f32 & img) { + + const auto & hparams = model.hparams; + const auto & enc = model.enc_img; + + const int32_t n_enc_state = hparams.n_enc_state; + const int32_t n_enc_layer = hparams.n_enc_layer; + const int32_t n_enc_head = hparams.n_enc_head; + const int32_t n_enc_head_dim = hparams.n_enc_head_dim(); + const int32_t n_enc_out_chans = hparams.n_enc_out_chans; + const int32_t n_img_size = hparams.n_img_size(); + const int32_t n_window_size = hparams.n_window_size(); + + struct ggml_init_params ggml_params = { + /*.mem_size =*/ state.buf_compute_img_enc.size(), + /*.mem_buffer =*/ state.buf_compute_img_enc.data(), + /*.no_alloc =*/ true, // skip allocating as we use ggml_alloc to allocate exact memory requirements + }; + + struct ggml_context * ctx0 = ggml_init(ggml_params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + struct ggml_tensor * inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, n_img_size, n_img_size, 3, 1); + ggml_set_name(inp, "inp"); + ggml_set_input(inp); + + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L392 + struct ggml_tensor * cur = ggml_conv_2d_sk_p0(ctx0, enc.proj_w, inp); + cur = ggml_add_inplace(ctx0, + cur, + ggml_repeat(ctx0, enc.proj_b, cur)); + + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L394 + // keep in F32 + cur = ggml_cont(ctx0, + ggml_permute(ctx0, cur, 1, 2, 0, 3)); + + // convert to F16 + //cur = ggml_cpy(ctx0, + // ggml_permute(ctx0, cur, 1, 2, 0, 3), + // ggml_new_tensor_3d(ctx0, GGML_TYPE_F16, n_enc_state, n_img_embd, n_img_embd)); + + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L108-L109 + cur = ggml_add_inplace(ctx0, cur, enc.pe); + + struct ggml_tensor * inpL = cur; + + for (int il = 0; il < n_enc_layer; ++il) { + const auto & layer = enc.layers[il]; + + // norm + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L168 + { + cur = ggml_norm(ctx0, inpL, hparams.eps); + + // cur = ln_0_w*cur + ln_0_b + cur = ggml_mul(ctx0, cur, layer.norm1_w); + cur = ggml_add_inplace(ctx0, cur, layer.norm1_b); + } + + const int64_t w0 = cur->ne[1]; + const int64_t h0 = cur->ne[2]; + + if (hparams.is_global_attn(il) == false) { + // local attention layer - apply window partition + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L169-L172 + cur = ggml_win_part(ctx0, cur, n_window_size); + } + + const int64_t W = cur->ne[1]; + const int64_t H = cur->ne[2]; + + // self-attention + { + cur = ggml_mul_mat(ctx0, layer.qkv_w, cur); + cur = ggml_add_inplace(ctx0, cur, layer.qkv_b); + + // split qkv into separate tensors + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L225-L229 + const int B = cur->ne[3]; + + cur = ggml_reshape_4d(ctx0, cur, n_enc_state, 3, W*H, B); + cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 3, 1, 2)); + + struct ggml_tensor * Q; + struct ggml_tensor * K; + struct ggml_tensor * V; + + Q = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 0*cur->nb[3]); + Q = ggml_reshape_4d(ctx0, Q, n_enc_head_dim, n_enc_head, W*H, B); + Q = ggml_cont (ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); + Q = ggml_reshape_3d(ctx0, Q, n_enc_head_dim, W*H, B*n_enc_head); + + K = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 1*cur->nb[3]); + K = ggml_reshape_4d(ctx0, K, n_enc_head_dim, n_enc_head, W*H, B); + K = ggml_cont (ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); + K = ggml_reshape_3d(ctx0, K, n_enc_head_dim, W*H, B*n_enc_head); + + V = ggml_view_3d (ctx0, cur, n_enc_state, W*H, B, cur->nb[1], cur->nb[2], 2*cur->nb[3]); + V = ggml_reshape_4d(ctx0, V, n_enc_head_dim, n_enc_head, W*H, B); + V = ggml_cont (ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3)); // transposed + V = ggml_reshape_3d(ctx0, V, W*H, n_enc_head_dim, B*n_enc_head); + + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + + struct ggml_tensor * KQ_scaled = + ggml_scale_inplace(ctx0, + KQ, + 1.0f/sqrtf(n_enc_head_dim)); + + struct ggml_tensor * rw = ggml_get_rel_pos(ctx0, layer.rel_pos_w, W, W); + struct ggml_tensor * rh = ggml_get_rel_pos(ctx0, layer.rel_pos_h, H, H); + + struct ggml_tensor * q_r = ggml_reshape_4d(ctx0, Q, n_enc_head_dim, W, H, B*n_enc_head); + + struct ggml_tensor * rel_w = ggml_cont(ctx0, ggml_permute(ctx0, + ggml_mul_mat(ctx0, + rw, + ggml_cont(ctx0, ggml_permute(ctx0, q_r, 0, 2, 1, 3))), + 0, 2, 1, 3)); + struct ggml_tensor * rel_h = ggml_mul_mat(ctx0, rh, q_r); + + struct ggml_tensor * attn = ggml_add_rel_pos_inplace(ctx0, KQ_scaled, rel_w, rel_h); + + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, attn); + + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max); + + cur = + ggml_reshape_4d(ctx0, + ggml_cont(ctx0, + ggml_permute(ctx0, + ggml_reshape_4d(ctx0, KQV, n_enc_head_dim, W*H, n_enc_head, B), + 0, 2, 1, 3)), + n_enc_state, W, H, B); + + cur = ggml_mul_mat(ctx0, layer.proj_w, cur); + cur = ggml_add_inplace(ctx0, cur, layer.proj_b); + } + + if (hparams.is_global_attn(il) == false) { + // local attention layer - reverse window partition + cur = ggml_win_unpart(ctx0, cur, w0, h0, n_window_size); + } + + cur = ggml_add_inplace(ctx0, cur, inpL); + + struct ggml_tensor * inpFF = cur; + + // feed-forward network + { + // norm + { + cur = ggml_norm(ctx0, inpFF, hparams.eps); + + // cur = mlp_ln_w*cur + mlp_ln_b + cur = ggml_mul(ctx0, cur, layer.norm2_w); + cur = ggml_add_inplace(ctx0, cur, layer.norm2_b); + } + + // fully connected + cur = ggml_mul_mat(ctx0, layer.mlp_lin1_w, cur); + cur = ggml_add_inplace(ctx0, cur, layer.mlp_lin1_b); + + // GELU activation + cur = ggml_gelu(ctx0, cur); + + // projection + cur = ggml_mul_mat(ctx0, layer.mlp_lin2_w, cur); + cur = ggml_add_inplace(ctx0, cur, layer.mlp_lin2_b); + } + + inpL = ggml_add(ctx0, cur, inpFF); + } + + cur = ggml_cont(ctx0, ggml_permute(ctx0, inpL, 2, 0, 1, 3)); + + cur = ggml_conv_2d_sk_p0(ctx0, enc.neck_conv_0, cur); + + cur = sam_layer_norm_2d(ctx0, cur, n_enc_out_chans, enc.neck_norm_0_w, enc.neck_norm_0_b, hparams.eps); + + cur = ggml_conv_2d_s1_ph(ctx0, enc.neck_conv_1, cur); + + cur = sam_layer_norm_2d(ctx0, cur, n_enc_out_chans, enc.neck_norm_1_w, enc.neck_norm_1_b, hparams.eps); + + cur = ggml_cpy(ctx0, cur, state.embd_img); + + ggml_build_forward_expand(gf, cur); + ggml_disconnect_node_from_graph(state.embd_img); + + //ggml_graph_print(&gf); + + ggml_free(ctx0); + + ggml_gallocr_alloc_graph(state.allocr, gf); + + { + struct ggml_tensor * inp = ggml_graph_get_tensor(gf, "inp"); + float * data = (float *) ggml_get_data(inp); + + const int nx = img.nx; + const int ny = img.ny; + const int n = nx*ny; + + GGML_ASSERT(nx == n_img_size && ny == n_img_size); + + for (int k = 0; k < 3; k++) { + for (int y = 0; y < ny; y++) { + for (int x = 0; x < nx; x++) { + data[k*n + y*nx + x] = img.data[3*(y*nx + x) + k]; + } + } + } + } + + return gf; +} + + +struct prompt_encoder_result { + struct ggml_tensor * embd_prompt_sparse = {}; + struct ggml_tensor * embd_prompt_dense = {}; +}; + +struct ggml_tensor * sam_prompt_encode_pe_encoding( + const sam_encoder_prompt & enc, + struct ggml_context * ctx0, + struct ggml_cgraph * gf, + struct ggml_tensor * coords) { + + auto * cur = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, enc.pe)), coords); + cur = ggml_scale(ctx0, cur, float(2.0*M_PI)); + + // concat + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L192 + { + struct ggml_tensor * t_sin = ggml_map_custom1(ctx0, cur, ggml_sam_sin, GGML_N_TASKS_MAX, NULL); + struct ggml_tensor * t_cos = ggml_map_custom1(ctx0, cur, ggml_sam_cos, GGML_N_TASKS_MAX, NULL); + + cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, t_sin->ne[0] + t_cos->ne[0], cur->ne[1]); + + ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_sin, ggml_view_2d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], cur->nb[1], 0))); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, t_cos, ggml_view_2d(ctx0, cur, t_sin->ne[0], t_sin->ne[1], cur->nb[1], t_sin->nb[1]))); + + } + return cur; + +} +// encode a prompt +// +// - points +// - boxes +// - masks +// +// TODO: currently just encode a single point for simplicity +// +prompt_encoder_result sam_encode_prompt( + const sam_model & model, + struct ggml_context * ctx0, + struct ggml_cgraph * gf, + sam_state & state, + const sam_prompt & prompt) { + + const auto & hparams = model.hparams; + const auto & enc = model.enc_prompt; + + struct ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 2, 2); + ggml_set_name(inp, "prompt_input"); + ggml_set_input(inp); + + auto * embd_prompt_sparse = [&]() -> struct ggml_tensor * { + switch (prompt.prompt_type) { + case SAM_PROMPT_TYPE_POINT: { + // PromptEncoder._embed_points + auto * pt_embd = sam_prompt_encode_pe_encoding(enc, ctx0, gf, inp); + + // overwrite label == -1 with not_a_point_embed.weight + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L86 + // TODO: extend for multiple points + auto * pt_embd_not = ggml_view_2d(ctx0, pt_embd, pt_embd->ne[0], 1, pt_embd->nb[1], pt_embd->nb[1]); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, enc.not_a_pt_embd_w, pt_embd_not)); + + // add point_embeddings[1] to label == 1 + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L90 + auto * pt_embd1 = ggml_view_2d(ctx0, pt_embd, pt_embd->ne[0], 1, pt_embd->nb[1], 0); + ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, pt_embd1, enc.pt_embd[1])); + + return pt_embd; + } break; + case SAM_PROMPT_TYPE_BOX: { + // PromptEncoder._embed_boxes + auto * corner_embd = sam_prompt_encode_pe_encoding(enc, ctx0, gf, inp); + + // corner_embd[:, 0, :] += self.point_embeddings[2].weight + // corner_embd[:, 1, :] += self.point_embeddings[3].weight + auto * corner0 = ggml_view_2d( + ctx0, corner_embd, corner_embd->ne[0], 1, corner_embd->nb[1], 0); + auto * corner1 = ggml_view_2d( + ctx0, corner_embd, corner_embd->ne[0], 1, corner_embd->nb[1], corner_embd->nb[1]); + + ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, corner0, enc.pt_embd[2])); + ggml_build_forward_expand(gf, ggml_add_inplace(ctx0, corner1, enc.pt_embd[3])); + + return corner_embd; + } break; + default: { + fprintf(stderr, "%s: unsupported prompt type %d\n", __func__, prompt.prompt_type); + return nullptr; + } break; + } + }(); + + ggml_build_forward_expand(gf, embd_prompt_sparse); + + struct ggml_tensor * embd_prompt_dense = ggml_repeat(ctx0, + ggml_cont(ctx0, + ggml_view_3d(ctx0, enc.no_mask_embd_w, + 1, 1, enc.no_mask_embd_w->ne[0], enc.no_mask_embd_w->nb[0], enc.no_mask_embd_w->nb[0], 0)), + ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hparams.n_img_embd(), hparams.n_img_embd(), hparams.n_enc_out_chans)); + + ggml_build_forward_expand(gf, embd_prompt_dense); + + //printf("used_mem = %zu\n", ggml_used_mem(ctx0)); + + prompt_encoder_result res; + res.embd_prompt_sparse = embd_prompt_sparse; + res.embd_prompt_dense = embd_prompt_dense; + return res; +} + +struct ggml_tensor* sam_decode_mask_transformer_attn( + const sam_layer_dec_transformer_attn & attn, + struct ggml_tensor * queries, + struct ggml_tensor * keys, + struct ggml_tensor * values, + struct ggml_context * ctx0, + const sam_model & model) { + const auto & hparams = model.hparams; + const int n_head = hparams.n_dec_heads; + + struct ggml_tensor * Qcur = {}; + struct ggml_tensor * Kcur = {}; + struct ggml_tensor * Vcur = {}; + + Qcur = ggml_mul_mat(ctx0, attn.q_w, queries); + Qcur = ggml_add_inplace(ctx0, Qcur, attn.q_b); + + Kcur = ggml_mul_mat(ctx0, attn.k_w, keys); + Kcur = ggml_add_inplace(ctx0, Kcur, attn.k_b); + + Vcur = ggml_mul_mat(ctx0, attn.v_w, values); + Vcur = ggml_add_inplace(ctx0, Vcur, attn.v_b); + + struct ggml_tensor * Q = {}; + struct ggml_tensor * K = {}; + struct ggml_tensor * V = {}; + + Q = ggml_reshape_4d(ctx0, Qcur, Qcur->ne[0]/n_head, n_head, Qcur->ne[1], Qcur->ne[2]); + Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3)); + + K = ggml_reshape_4d(ctx0, Kcur, Kcur->ne[0]/n_head, n_head, Kcur->ne[1], Kcur->ne[2]); + K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3)); + + V = ggml_reshape_4d(ctx0, Vcur, Vcur->ne[0]/n_head, n_head, Vcur->ne[1], Vcur->ne[2]); + V = ggml_cont(ctx0, ggml_permute(ctx0, V, 0, 2, 1, 3)); + + // Q * K + struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q); + + struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(Q->ne[0]))); + + struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_scaled); + + struct ggml_tensor * KQV = ggml_mul_mat(ctx0, KQ_soft_max, ggml_cont(ctx0, ggml_transpose(ctx0, V))); + + struct ggml_tensor * KQV_merged = ggml_cont(ctx0, ggml_transpose(ctx0, KQV)); + KQV_merged = ggml_cont(ctx0, ggml_permute(ctx0, KQV_merged, 0, 2, 1, 3)); + KQV_merged = ggml_reshape_3d(ctx0, KQV_merged, KQV_merged->ne[0]*KQV_merged->ne[1], KQV_merged->ne[2], KQV_merged->ne[3]); + KQV_merged = ggml_mul_mat(ctx0, attn.out_w, KQV_merged); + KQV_merged = ggml_add_inplace(ctx0, KQV_merged, attn.out_b); + + return KQV_merged; +} + +struct ggml_tensor * sam_decode_mask_mlp_relu_3( + struct ggml_tensor * in, + struct ggml_tensor * w_0, + struct ggml_tensor * b_0, + struct ggml_tensor * w_1, + struct ggml_tensor * b_1, + struct ggml_tensor * w_2, + struct ggml_tensor * b_2, + struct ggml_context * ctx0) { + + struct ggml_tensor * cur = {}; + cur = ggml_mul_mat(ctx0, w_0, in); + cur = ggml_add_inplace(ctx0, cur, b_0); + + cur = ggml_relu_inplace(ctx0, cur); + + cur = ggml_mul_mat(ctx0, w_1, cur); + cur = ggml_add_inplace(ctx0, cur, b_1); + + cur = ggml_relu_inplace(ctx0, cur); + + cur = ggml_mul_mat(ctx0, w_2, cur); + cur = ggml_add_inplace(ctx0, cur, b_2); + + return cur; +} + +bool sam_decode_mask( + const sam_model & model, + const prompt_encoder_result & prompt, + struct ggml_tensor * pe_img, + struct ggml_context * ctx0, + struct ggml_cgraph * gf, + sam_state & state, + const bool multimask_output) { + + const auto & hparams = model.hparams; + const auto & dec = model.dec; + const int n_img_embd = hparams.n_img_embd(); + + struct ggml_tensor * tokens = {}; + { + // Concatenate output tokens + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L120 + const auto& sparse = prompt.embd_prompt_sparse; + + tokens = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, dec.iou_token_w->ne[0], dec.iou_token_w->ne[1] + dec.mask_tokens_w->ne[1] + sparse->ne[1], sparse->ne[2]); + + const size_t offsets[3] = { 0, dec.iou_token_w->ne[1]*tokens->nb[1], dec.iou_token_w->ne[1]*tokens->nb[1] + dec.mask_tokens_w->ne[1]*tokens->nb[1] }; + ggml_build_forward_expand(gf, ggml_cpy(ctx0, dec.iou_token_w, ggml_view_2d(ctx0, tokens, tokens->ne[0], dec.iou_token_w->ne[1], tokens->nb[1], offsets[0]))); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, dec.mask_tokens_w, ggml_view_2d(ctx0, tokens, tokens->ne[0], dec.mask_tokens_w->ne[1], tokens->nb[1], offsets[1]))); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, sparse, ggml_view_2d(ctx0, tokens, tokens->ne[0], sparse->ne[1], tokens->nb[1], offsets[2]))); + // TODO: Sparse prompt embeddings can have more than one point + } + + + struct ggml_tensor * src = {}; + struct ggml_tensor * pos_src = {}; + int srcNE[4] = { 0, 0, 0, 0 }; + { + // Expand per-image data in the batch direction to be per-mask + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L125 + src = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, state.embd_img->ne[0], state.embd_img->ne[1], state.embd_img->ne[2], tokens->ne[2]); + + src = ggml_add(ctx0, + ggml_repeat(ctx0, + state.embd_img, + src), + prompt.embd_prompt_dense); + + srcNE[0] = src->ne[0]; + srcNE[1] = src->ne[1]; + srcNE[2] = src->ne[2]; + srcNE[3] = src->ne[3]; + + // flatten & permute + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L83 + src = ggml_cont(ctx0, ggml_permute(ctx0, + ggml_view_3d(ctx0, + src, + src->ne[0]*src->ne[1], + src->ne[2], + src->ne[3], + src->nb[2], + src->nb[3], + 0), + 1, 0, 2, 3)); + + pos_src = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, pe_img->ne[0], pe_img->ne[1], pe_img->ne[2], tokens->ne[2]); + pos_src = ggml_repeat(ctx0, + pe_img, + pos_src); + + // flatten & permute + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L83 + pos_src = ggml_cont(ctx0, ggml_permute(ctx0, + ggml_view_3d(ctx0, + pos_src, + pos_src->ne[0]*pos_src->ne[1], + pos_src->ne[2], + pos_src->ne[3], + pos_src->nb[2], + pos_src->nb[3], + 0), + 1, 0, 2, 3)); + } + + struct ggml_tensor * queries = tokens; + struct ggml_tensor * keys = src; + { + // Run the transformer + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L62 + for (int i = 0; i < int(model.dec.transformer_layers.size()); ++i) { + const auto& tfm_layer = model.dec.transformer_layers[i]; + + // Self attention block + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L154 + const bool skip_first_layer_pe = i == 0; + if (skip_first_layer_pe) { + queries = sam_decode_mask_transformer_attn(tfm_layer.self_attn, queries, queries, queries, ctx0, model); + } + else { + struct ggml_tensor * q_0 = ggml_add(ctx0, queries, tokens); + + struct ggml_tensor * self_attn = sam_decode_mask_transformer_attn(tfm_layer.self_attn, q_0, q_0, queries, ctx0, model); + queries = ggml_add(ctx0, queries, self_attn); + } + + queries = ggml_norm(ctx0, queries, hparams.eps_decoder_transformer); + queries = ggml_add_inplace(ctx0, + ggml_mul(ctx0, queries, tfm_layer.norm1_w), + tfm_layer.norm1_b); + + // Cross attention block, tokens attending to image embedding + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L163 + struct ggml_tensor * q_1 = ggml_add(ctx0, queries, tokens); + struct ggml_tensor * k_1 = ggml_add(ctx0, keys, pos_src); + + struct ggml_tensor * cross_attn_token_to_img = sam_decode_mask_transformer_attn(tfm_layer.cross_attn_token_to_img, q_1, k_1, keys, ctx0, model); + + queries = ggml_add_inplace(ctx0, queries, cross_attn_token_to_img); + queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer); + queries = ggml_add_inplace(ctx0, + ggml_mul(ctx0, queries, tfm_layer.norm2_w), + tfm_layer.norm2_b); + + // MLP block + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L170 + struct ggml_tensor * mlp_out = ggml_mul_mat(ctx0, + tfm_layer.mlp_lin1_w, + queries); + + mlp_out = ggml_add_inplace(ctx0, mlp_out, tfm_layer.mlp_lin1_b); + + // RELU activation + mlp_out = ggml_relu_inplace(ctx0, mlp_out); + mlp_out = ggml_mul_mat(ctx0, tfm_layer.mlp_lin2_w, mlp_out); + + mlp_out = ggml_add_inplace(ctx0, mlp_out, tfm_layer.mlp_lin2_b); + + queries = ggml_add_inplace(ctx0, queries, mlp_out); + queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer); + queries = ggml_add_inplace(ctx0, + ggml_mul(ctx0, queries, tfm_layer.norm3_w), + tfm_layer.norm3_b); + + // Cross attention block, image embedding attending to tokens + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L175 + struct ggml_tensor * q_2 = ggml_add(ctx0, queries, tokens); + struct ggml_tensor * k_2 = ggml_add(ctx0, keys, pos_src); + + struct ggml_tensor * cross_attn_img_to_token = sam_decode_mask_transformer_attn(tfm_layer.cross_attn_img_to_token, k_2, q_2, queries, ctx0, model); + keys = ggml_add_inplace(ctx0, keys, cross_attn_img_to_token); + keys = ggml_norm_inplace(ctx0, keys, hparams.eps_decoder_transformer); + keys = ggml_add_inplace(ctx0, + ggml_mul(ctx0, keys, tfm_layer.norm4_w), + tfm_layer.norm4_b); + } + + // Apply the final attention layer from the points to the image + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/transformer.py#L99 + struct ggml_tensor * q = ggml_add(ctx0, queries, tokens); + struct ggml_tensor * k = ggml_add(ctx0, keys, pos_src); + + struct ggml_tensor * final_attn_token_to_img = sam_decode_mask_transformer_attn(dec.transformer_final_attn_token_to_img, q, k, keys, ctx0, model); + + queries = ggml_add_inplace(ctx0, queries, final_attn_token_to_img); + queries = ggml_norm_inplace(ctx0, queries, hparams.eps_decoder_transformer); + queries = ggml_add_inplace(ctx0, + ggml_mul(ctx0, queries, dec.transformer_norm_final_w), + dec.transformer_norm_final_b); + } + + + struct ggml_tensor * iou_pred = ggml_view_2d(ctx0, queries, queries->ne[0], queries->ne[2], queries->nb[2], 0); + const int num_mask_tokens = 4; // num_multimask_outputs + 1 + struct ggml_tensor * mask_tokens_out = ggml_view_3d(ctx0, queries, queries->ne[0], num_mask_tokens, queries->ne[2], queries->nb[1], num_mask_tokens*queries->nb[1], queries->nb[1]); + + // Upscale mask embeddings and predict masks using the mask tokens + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L136 + keys = ggml_cont(ctx0, ggml_transpose(ctx0, keys)); + keys = ggml_view_4d(ctx0, keys, srcNE[0], srcNE[1], srcNE[2], srcNE[3], srcNE[0]*keys->nb[0], keys->nb[1], keys->nb[2], 0); + // ggml_build_forward_expand(gf, keys); + struct ggml_tensor * upscaled_embedding = {}; + { + // ConvTranspose2d + keys = ggml_conv_transpose_2d_p0(ctx0, dec.output_upscaling_0_w, keys, 2); + keys = ggml_add_inplace(ctx0, keys, ggml_repeat(ctx0, + ggml_reshape_3d(ctx0, dec.output_upscaling_0_b, 1, 1, dec.output_upscaling_0_b->ne[0]), + keys)); + + keys = sam_layer_norm_2d(ctx0, keys, n_img_embd, dec.output_upscaling_1_w, dec.output_upscaling_1_b, hparams.eps); + + // GELU activation + keys = ggml_gelu_inplace(ctx0, keys); + + // ConvTranspose2d + keys = ggml_conv_transpose_2d_p0(ctx0, dec.output_upscaling_3_w, keys, 2); + keys = ggml_add_inplace(ctx0, ggml_repeat(ctx0, + ggml_reshape_3d(ctx0, dec.output_upscaling_3_b, 1, 1, dec.output_upscaling_3_b->ne[0]), + keys), keys); + // GELU activation + keys = ggml_gelu_inplace(ctx0, keys); + upscaled_embedding = ggml_reshape_3d(ctx0, keys, keys->ne[0]*keys->ne[1], keys->ne[2], keys->ne[3]); + upscaled_embedding = ggml_cont(ctx0, ggml_transpose(ctx0, upscaled_embedding)); // TODO: Shouldn't be needed + } + + struct ggml_tensor * hyper_in = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_img_embd/2, num_mask_tokens, mask_tokens_out->ne[2]); + + for (int i = 0; i < num_mask_tokens; ++i) { + const auto& mlp = dec.output_hypernet_mlps[i]; + struct ggml_tensor * in = ggml_view_2d(ctx0, mask_tokens_out, mask_tokens_out->ne[0], mask_tokens_out->ne[2], mask_tokens_out->nb[1], i*mask_tokens_out->nb[1]); + struct ggml_tensor * out = sam_decode_mask_mlp_relu_3(in, mlp.w_0, mlp.b_0, mlp.w_1, mlp.b_1, mlp.w_2, mlp.b_2, ctx0); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, out, ggml_view_2d(ctx0, hyper_in, hyper_in->ne[0], hyper_in->ne[2], hyper_in->nb[1], i*hyper_in->nb[1]))); + } + + struct ggml_tensor * masks = ggml_mul_mat(ctx0, hyper_in, upscaled_embedding); + masks = ggml_cont(ctx0, ggml_transpose(ctx0, masks)); // TODO: Shouldn't be needed + masks = ggml_reshape_4d(ctx0, masks, keys->ne[0], keys->ne[1], masks->ne[1], keys->ne[3]); + + // Generate mask quality predictions + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L146 + iou_pred = sam_decode_mask_mlp_relu_3(iou_pred, dec.iou_prediction_head_0_w, dec.iou_prediction_head_0_b, dec.iou_prediction_head_1_w, dec.iou_prediction_head_1_b, dec.iou_prediction_head_2_w, dec.iou_prediction_head_2_b, ctx0); + + // Select the correct mask or masks for output + // ref: https://github.com/facebookresearch/segment-anything/blob/6fdee8f2727f4506cfbbe553e23b895e27956588/segment_anything/modeling/mask_decoder.py#L101 + if (multimask_output) { + iou_pred = ggml_cpy(state.ctx, ggml_view_1d(ctx0, iou_pred, iou_pred->ne[0] - 1, iou_pred->nb[0]), state.iou_predictions); + masks = ggml_view_4d(ctx0, masks, masks->ne[0], masks->ne[1], masks->ne[2] - 1, masks->ne[3], + masks->nb[1], masks->nb[2], masks->nb[3], masks->nb[2] /* offset*/); + masks = ggml_cpy(state.ctx, masks, state.low_res_masks); + } else { + iou_pred = ggml_cpy(state.ctx, ggml_view_1d(ctx0, iou_pred, 1, 0), ggml_view_1d(ctx0, state.iou_predictions, 1, 0)); + masks = ggml_view_4d(ctx0, masks, masks->ne[0], masks->ne[1], 1, masks->ne[3], + masks->nb[1], masks->nb[2], masks->nb[3], 0); + auto * low_res_mask = ggml_view_4d(ctx0, state.low_res_masks, masks->ne[0], masks->ne[1], 1, masks->ne[3], + masks->nb[1], masks->nb[2], masks->nb[3], 0); + masks = ggml_cpy(state.ctx, masks, low_res_mask); + } + + ggml_build_forward_expand(gf, masks); + ggml_build_forward_expand(gf, iou_pred); + + ggml_disconnect_node_from_graph(state.low_res_masks); + ggml_disconnect_node_from_graph(state.iou_predictions); + + return true; +} + +bool sam_write_masks(const sam_hparams& hparams, int nx, int ny, const sam_state & state, const std::string & fname, const bool multimask_output) { + if (state.low_res_masks->ne[2] == 0) return true; + if (state.low_res_masks->ne[2] != state.iou_predictions->ne[0]) { + printf("Error: number of masks (%d) does not match number of iou predictions (%d)\n", (int)state.low_res_masks->ne[2], (int)state.iou_predictions->ne[0]); + return false; + } + + const int n_img_size = hparams.n_img_size(); + const float mask_threshold = hparams.mask_threshold; + const float iou_threshold = hparams.iou_threshold; + const float stability_score_threshold = hparams.stability_score_threshold; + const float intersection_threshold = mask_threshold + hparams.stability_score_offset; + const float union_threshold = mask_threshold - hparams.stability_score_offset; + + const int ne0 = state.low_res_masks->ne[0]; + const int ne1 = state.low_res_masks->ne[1]; + const int ne2 = multimask_output ? state.low_res_masks->ne[2] : 1; + + // Remove padding and upscale masks to the original image size. + // ref: https://github.com/facebookresearch/segment-anything/blob/efeab7296ab579d4a261e554eca80faf6b33924a/segment_anything/modeling/sam.py#L140 + + const float preprocess_scale = std::max(nx, ny) / float(n_img_size); + const int cropped_nx = int(nx / preprocess_scale + 0.5f); + const int cropped_ny = int(ny / preprocess_scale + 0.5f); + + const float scale_x_1 = (float)ne0 / (float)n_img_size; + const float scale_y_1 = (float)ne1 / (float)n_img_size; + + const float scale_x_2 = float(cropped_nx) / float(nx); + const float scale_y_2 = float(cropped_ny) / float(ny); + + const auto iou_data = (float*)state.iou_predictions->data; + + for (int i = 0; i < ne2; ++i) { + if (iou_threshold > 0.f && iou_data[i] < iou_threshold) { + printf("Skipping mask %d with iou %f below threshold %f\n", i, iou_data[i], iou_threshold); + continue; // Filtering masks with iou below the threshold + } + + std::vector mask_data(n_img_size*n_img_size); + { + const float* data = (float *) state.low_res_masks->data + i*ne0*ne1; + + for (int iy = 0; iy < n_img_size; ++iy) { + for (int ix = 0; ix < n_img_size; ++ix) { + const float sx = std::max(scale_x_1*(ix + 0.5f) - 0.5f, 0.0f); + const float sy = std::max(scale_y_1*(iy + 0.5f) - 0.5f, 0.0f); + + const int x0 = std::max(0, (int)sx); + const int y0 = std::max(0, (int)sy); + + const int x1 = std::min(x0 + 1, ne0 - 1); + const int y1 = std::min(y0 + 1, ne1 - 1); + + const float dx = sx - x0; + const float dy = sy - y0; + + const int j00 = y0*ne0 + x0; + const int j01 = y0*ne0 + x1; + const int j10 = y1*ne0 + x0; + const int j11 = y1*ne0 + x1; + + const float v00 = data[j00]; + const float v01 = data[j01]; + const float v10 = data[j10]; + const float v11 = data[j11]; + + const float v0 = (1-dx)*v00 + dx*v01; + const float v1 = (1-dx)*v10 + dx*v11; + + const float v = (1-dy)*v0 + dy*v1; + + mask_data[iy*n_img_size + ix] = v; + } + } + } + + int intersections = 0; + int unions = 0; + sam_image_u8 res; + int min_iy = ny; + int max_iy = 0; + int min_ix = nx; + int max_ix = 0; + { + const float* data = mask_data.data(); + + res.nx = nx; + res.ny = ny; + res.data.resize(nx*ny); + + for (int iy = 0; iy < ny; ++iy) { + for (int ix = 0; ix < nx; ++ix) { + const float sx = std::max(scale_x_2*(ix + 0.5f) - 0.5f, 0.0f); + const float sy = std::max(scale_y_2*(iy + 0.5f) - 0.5f, 0.0f); + + const int x0 = std::max(0, (int)sx); + const int y0 = std::max(0, (int)sy); + + const int x1 = std::min(x0 + 1, cropped_nx - 1); + const int y1 = std::min(y0 + 1, cropped_ny - 1); + + const float dx = sx - x0; + const float dy = sy - y0; + + const int j00 = y0*n_img_size + x0; + const int j01 = y0*n_img_size + x1; + const int j10 = y1*n_img_size + x0; + const int j11 = y1*n_img_size + x1; + + const float v00 = data[j00]; + const float v01 = data[j01]; + const float v10 = data[j10]; + const float v11 = data[j11]; + + const float v0 = (1-dx)*v00 + dx*v01; + const float v1 = (1-dx)*v10 + dx*v11; + + const float v = (1-dy)*v0 + dy*v1; + + if (v > intersection_threshold) { + intersections++; + } + if (v > union_threshold) { + unions++; + } + if (v > mask_threshold) { + min_iy = std::min(min_iy, iy); + max_iy = std::max(max_iy, iy); + min_ix = std::min(min_ix, ix); + max_ix = std::max(max_ix, ix); + + res.data[iy*nx + ix] = 255; + } + } + } + } + + const float stability_score = float(intersections) / float(unions); + if (stability_score_threshold > 0.f && stability_score < stability_score_threshold) { + printf("Skipping mask %d with stability score %f below threshold %f\n", i, stability_score, stability_score_threshold); + continue; // Filtering masks with stability score below the threshold + } + + printf("Mask %d: iou = %f, stability_score = %f, bbox (%d, %d), (%d, %d)\n", + i, iou_data[i], stability_score, min_ix, max_ix, min_iy, max_iy); + + const std::string filename = multimask_output ? fname + std::to_string(i) + ".png" : fname + ".png"; + if (!stbi_write_png(filename.c_str(), res.nx, res.ny, 1, res.data.data(), res.nx)) { + printf("%s: failed to write mask %s\n", __func__, filename.c_str()); + return false; + } + } + + + return true; +} + + +struct ggml_cgraph * sam_build_fast_graph( + const sam_model & model, + sam_state & state, + const int nx, + const int ny, + const sam_prompt & prompt, + const bool multimask_output) { + + struct ggml_init_params ggml_params = { + /*.mem_size =*/ state.buf_compute_fast.size(), + /*.mem_buffer =*/ state.buf_compute_fast.data(), + /*.no_alloc =*/ true, // skip allocating as we use ggml_alloc to allocate exact memory requirements + }; + + struct ggml_context * ctx0 = ggml_init(ggml_params); + struct ggml_cgraph * gf = ggml_new_graph(ctx0); + + prompt_encoder_result enc_res = sam_encode_prompt(model, ctx0, gf, state, prompt); + if (!enc_res.embd_prompt_sparse || !enc_res.embd_prompt_dense) { + fprintf(stderr, "%s: failed to encode prompt\n", __func__); + return {}; + } + + struct ggml_tensor * pe_img_dense = sam_fill_dense_pe(model, ctx0, gf, state); + if (!pe_img_dense) { + fprintf(stderr, "%s: failed to get dense positional encoding\n", __func__); + return {}; + } + + if (!sam_decode_mask(model, enc_res, pe_img_dense, ctx0, gf, state, multimask_output)) { + fprintf(stderr, "%s: failed to decode mask\n", __func__); + return {}; + } + + ggml_free(ctx0); + + ggml_gallocr_alloc_graph(state.allocr, gf); + + struct ggml_tensor * inp = ggml_graph_get_tensor(gf, "prompt_input"); + auto * data = (float *) inp->data; + + // Transform prompt (point or box) + { + // https://github.com/facebookresearch/segment-anything/blob/dca509fe793f601edb92606367a655c15ac00fdf/segment_anything/utils/transforms.py#L33 + // The point scaling here is greatly simplified but mathematically equivalent. + const auto scale = 1.0F / std::max(nx, ny); + + switch (prompt.prompt_type) { + case SAM_PROMPT_TYPE_POINT: { + const auto & pt = prompt.pt; + + // set the input by converting the [0, 1] coordinates to [-1, 1] + data[0] = 2.0f*pt.x*scale - 1.0f; + data[1] = 2.0f*pt.y*scale - 1.0f; + + // padding + // ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/prompt_encoder.py#L81-L85 + data[2] = 2.0f*(0.0f) - 1.0f; + data[3] = 2.0f*(0.0f) - 1.0f; + } break; + case SAM_PROMPT_TYPE_BOX: { + const auto & box = prompt.box; + + data[0] = 2.0f*box.x1*scale - 1.0f; + data[1] = 2.0f*box.y1*scale - 1.0f; + data[2] = 2.0f*box.x2*scale - 1.0f; + data[3] = 2.0f*box.y2*scale - 1.0f; + } break; + } + } + + // from sam_fill_dense_pe + { + struct ggml_tensor * xy_embed_stacked = ggml_graph_get_tensor(gf, "xy_embed_stacked"); + const int32_t n_img_embd = model.hparams.n_img_embd(); + const float n_img_embd_inv = 1.0f / n_img_embd; + float * data = (float *) ggml_get_data(xy_embed_stacked); + for (int i = 0; i < n_img_embd; ++i) { + const int row = 2*i*n_img_embd; + const float y_val = 2 * (i + 0.5f) * n_img_embd_inv - 1; + for (int j = 0; j < n_img_embd; ++j) { + const float x_val = 2 * (j + 0.5f) * n_img_embd_inv - 1; + data[row + 2*j + 0] = x_val; + data[row + 2*j + 1] = y_val; + } + } + } + + return gf; +} + +void sam_print_usage(int argc, char ** argv, const sam_params & params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n"); + fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); + fprintf(stderr, " -m FNAME, --model FNAME\n"); + fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); + fprintf(stderr, " -i FNAME, --inp FNAME\n"); + fprintf(stderr, " input file (default: %s)\n", params.fname_inp.c_str()); + fprintf(stderr, " -o FNAME, --out FNAME\n"); + fprintf(stderr, " mask file name prefix (default: %s)\n", params.fname_out.c_str()); + fprintf(stderr, " -sm, --single-mask\n"); + fprintf(stderr, " single mask output (default multi mask output)\n"); + fprintf(stderr, "SAM hyperparameters:\n"); + fprintf(stderr, " -mt FLOAT, --mask-threshold\n"); + fprintf(stderr, " mask threshold (default: %f)\n", params.mask_threshold); + fprintf(stderr, " -it FLOAT, --iou-threshold\n"); + fprintf(stderr, " iou threshold (default: %f)\n", params.iou_threshold); + fprintf(stderr, " -st FLOAT, --score-threshold\n"); + fprintf(stderr, " score threshold (default: %f)\n", params.stability_score_threshold); + fprintf(stderr, " -so FLOAT, --score-offset\n"); + fprintf(stderr, " score offset (default: %f)\n", params.stability_score_offset); + fprintf(stderr, " -e FLOAT, --epsilon\n"); + fprintf(stderr, " epsilon (default: %f)\n", params.eps); + fprintf(stderr, " -ed FLOAT, --epsilon-decoder-transformer\n"); + fprintf(stderr, " epsilon decoder transformer (default: %f)\n", params.eps_decoder_transformer); + fprintf(stderr, "SAM prompt:\n"); + fprintf(stderr, " -p [x,y], --point-prompt\n"); + fprintf(stderr, " point to be used as prompt for SAM (default: %f,%f). Must be in a format FLOAT,FLOAT \n", params.prompt.pt.x, params.prompt.pt.y); + fprintf(stderr, " -b [x1,y1,x2,y2], --box-prompt\n"); + fprintf(stderr, " box to be used as prompt for SAM (default: %f,%f,%f,%f). Must be in a format FLOAT,FLOAT,FLOAT,FLOAT \n", + params.prompt.box.x1, params.prompt.box.y1, params.prompt.box.x2, params.prompt.box.y2); + fprintf(stderr, "\n"); +} + +bool sam_params_parse(int argc, char ** argv, sam_params & params) { + + bool use_point_prompt = false; + bool use_box_prompt = false; + + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + + if (arg == "-s" || arg == "--seed") { + params.seed = std::stoi(argv[++i]); + } else if (arg == "-t" || arg == "--threads") { + params.n_threads = std::stoi(argv[++i]); + } else if (arg == "-m" || arg == "--model") { + params.model = argv[++i]; + } else if (arg == "-i" || arg == "--inp") { + params.fname_inp = argv[++i]; + } else if (arg == "-o" || arg == "--out") { + params.fname_out = argv[++i]; + } else if (arg == "-sm" || arg == "--single-mask") { + params.multimask_output = false; + } else if (arg == "-mt" || arg == "--mask-threshold") { + params.mask_threshold = std::stof(argv[++i]); + } else if (arg == "-it" || arg == "--iou-threshold") { + params.iou_threshold = std::stof(argv[++i]); + } else if (arg == "-st" || arg == "--score-threshold") { + params.stability_score_threshold = std::stof(argv[++i]); + } else if (arg == "-so" || arg == "--score-offset") { + params.stability_score_offset = std::stof(argv[++i]); + } else if (arg == "-e" || arg == "--epsilon") { + params.eps = std::stof(argv[++i]); + } else if (arg == "-ed" || arg == "--epsilon-decoder-transformer") { + params.eps_decoder_transformer = std::stof(argv[++i]); + } else if (arg == "-p" || arg == "--point-prompt") { + // TODO multiple points per model invocation + use_point_prompt = true; + char* point = argv[++i]; + + char* coord = strtok(point, ","); + if (!coord){ + fprintf(stderr, "Error while parsing prompt!\n"); + exit(1); + } + params.prompt.pt.x = std::stof(coord); + + coord = strtok(NULL, ","); + if (!coord){ + fprintf(stderr, "Error while parsing prompt!\n"); + exit(1); + } + params.prompt.pt.y = std::stof(coord); + } else if (arg == "-b" || arg == "--box-prompt") { + use_box_prompt = true; + char * box_prompt = argv[++i]; + float box_vals[4]; + + char * val = strtok(box_prompt, ","); + if (!val) { + fprintf(stderr, "Error while parsing prompt!\n"); + exit(1); + } + box_vals[0] = std::stof(val); + + for (int j = 1; j < 4; ++j) { + char * val = strtok(NULL, ","); + if (!val) { + fprintf(stderr, "Error while parsing prompt!\n"); + exit(1); + } + box_vals[j] = std::stof(val); + } + + params.prompt.box.x1 = box_vals[0]; + params.prompt.box.y1 = box_vals[1]; + params.prompt.box.x2 = box_vals[2]; + params.prompt.box.y2 = box_vals[3]; + } else if (arg == "-h" || arg == "--help") { + sam_print_usage(argc, argv, params); + exit(0); + } else { + fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + sam_print_usage(argc, argv, params); + exit(0); + } + } + + if (use_box_prompt && use_point_prompt) { + fprintf(stderr, "Error: use either point or box prompt, not both.\n"); + exit(1); + } + + params.prompt.prompt_type = SAM_PROMPT_TYPE_POINT; + if (use_box_prompt) { + params.prompt.prompt_type = SAM_PROMPT_TYPE_BOX; + } + + return true; +} + + +int main(int argc, char ** argv) { + ggml_time_init(); + const int64_t t_main_start_us = ggml_time_us(); + + sam_params params; + params.model = "models/sam-vit-b/ggml-model-f16.bin"; + + sam_model model; + sam_state state; + int64_t t_load_us = 0; + + if (sam_params_parse(argc, argv, params) == false) { + return 1; + } + + if (params.seed < 0) { + params.seed = time(NULL); + } + fprintf(stderr, "%s: seed = %d\n", __func__, params.seed); + + // load the image + sam_image_u8 img0; + if (!sam_image_load_from_file(params.fname_inp, img0)) { + fprintf(stderr, "%s: failed to load image from '%s'\n", __func__, params.fname_inp.c_str()); + return 1; + } + fprintf(stderr, "%s: loaded image '%s' (%d x %d)\n", __func__, params.fname_inp.c_str(), img0.nx, img0.ny); + + // preprocess to f32 + sam_image_f32 img1; + if (!sam_image_preprocess(img0, img1)) { + fprintf(stderr, "%s: failed to preprocess image\n", __func__); + return 1; + } + fprintf(stderr, "%s: preprocessed image (%d x %d)\n", __func__, img1.nx, img1.ny); + + + // load the model + { + const int64_t t_start_us = ggml_time_us(); + + if (!sam_model_load(params, model)) { + fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str()); + return 1; + } + + t_load_us = ggml_time_us() - t_start_us; + } + + { + static size_t buf_size = 256u*1024*1024; + + struct ggml_init_params ggml_params = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, + }; + + state.ctx = ggml_init(ggml_params); + + state.embd_img = ggml_new_tensor_3d(state.ctx, GGML_TYPE_F32, + model.hparams.n_img_embd(), model.hparams.n_img_embd(), model.hparams.n_enc_out_chans); + + state.low_res_masks = ggml_new_tensor_3d(state.ctx, GGML_TYPE_F32, + model.hparams.n_enc_out_chans, model.hparams.n_enc_out_chans, 3); + + state.iou_predictions = ggml_new_tensor_1d(state.ctx, GGML_TYPE_F32, 3); + } + + // Encode image + { + state.buf_compute_img_enc.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead()); + state.allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + struct ggml_cgraph * gf = sam_encode_image(model, state, img1); + if (!gf) { + fprintf(stderr, "%s: failed to encode image\n", __func__); + return 1; + } + + ggml_graph_compute_helper(state.work_buffer, gf, params.n_threads); + + // print_t_f32("embd_img", state.embd_img); + + ggml_gallocr_free(state.allocr); + state.allocr = NULL; + state.work_buffer.clear(); + } + + // Encode prompt and decode mask + { + state.buf_compute_fast.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead()); + state.allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + switch (params.prompt.prompt_type) { + case SAM_PROMPT_TYPE_POINT: + fprintf(stderr, "Using point prompt: (%f, %f)\n", params.prompt.pt.x, params.prompt.pt.y); + break; + case SAM_PROMPT_TYPE_BOX: + fprintf(stderr, "Using box prompt: (%f, %f, %f, %f)\n", + params.prompt.box.x1, + params.prompt.box.y1, + params.prompt.box.x2, + params.prompt.box.y2); + break; + } + + struct ggml_cgraph * gf = sam_build_fast_graph(model, state, img0.nx, img0.ny, params.prompt, params.multimask_output); + if (!gf) { + fprintf(stderr, "%s: failed to build fast graph\n", __func__); + return 1; + } + + ggml_graph_compute_helper(state.work_buffer, gf, params.n_threads); + + //print_t_f32("iou_predictions", state.iou_predictions); + //print_t_f32("low_res_masks", state.low_res_masks); + ggml_gallocr_free(state.allocr); + state.allocr = NULL; + } + + if (!sam_write_masks(model.hparams, img0.nx, img0.ny, state, params.fname_out, params.multimask_output)) { + fprintf(stderr, "%s: failed to write masks\n", __func__); + return 1; + } + + // report timing + { + const int64_t t_main_end_us = ggml_time_us(); + + fprintf(stderr, "\n\n"); + fprintf(stderr, "%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f); + fprintf(stderr, "%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f); + } + + ggml_free(model.ctx); + + return 0; +} diff --git a/ggml/examples/simple/CMakeLists.txt b/ggml/examples/simple/CMakeLists.txt new file mode 100644 index 0000000..7d3aae0 --- /dev/null +++ b/ggml/examples/simple/CMakeLists.txt @@ -0,0 +1,21 @@ +# +# simple-ctx + +set(TEST_TARGET simple-ctx) +add_executable(${TEST_TARGET} simple-ctx.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml) + +# +# simple-backend + +set(TEST_TARGET simple-backend) +add_executable(${TEST_TARGET} simple-backend.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml) + +if (GGML_CUDA) + add_compile_definitions(GGML_USE_CUDA) +endif() + +if (GGML_METAL) + add_compile_definitions(GGML_USE_METAL) +endif() diff --git a/ggml/examples/simple/README.md b/ggml/examples/simple/README.md new file mode 100644 index 0000000..28c549f --- /dev/null +++ b/ggml/examples/simple/README.md @@ -0,0 +1,61 @@ +## Simple + +This example simply performs a matrix multiplication, solely for the purpose of demonstrating a basic usage of ggml and backend handling. The code is commented to help understand what each part does. + +Traditional matrix multiplication goes like this (multiply row-by-column): + +$$ +A \times B = C +$$ + +$$ +\begin{bmatrix} +2 & 8 \\ +5 & 1 \\ +4 & 2 \\ +8 & 6 \\ +\end{bmatrix} +\times +\begin{bmatrix} +10 & 9 & 5 \\ +5 & 9 & 4 \\ +\end{bmatrix} +\= +\begin{bmatrix} +60 & 90 & 42 \\ +55 & 54 & 29 \\ +50 & 54 & 28 \\ +110 & 126 & 64 \\ +\end{bmatrix} +$$ + +In `ggml`, we pass the matrix $B$ in transposed form and multiply row-by-row. The result $C$ is also transposed: + +$$ +ggml\\_mul\\_mat(A, B^T) = C^T +$$ + +$$ +ggml\\_mul\\_mat( +\begin{bmatrix} +2 & 8 \\ +5 & 1 \\ +4 & 2 \\ +8 & 6 \\ +\end{bmatrix} +, +\begin{bmatrix} +10 & 5 \\ +9 & 9 \\ +5 & 4 \\ +\end{bmatrix} +) +\= +\begin{bmatrix} +60 & 55 & 50 & 110 \\ +90 & 54 & 54 & 126 \\ +42 & 29 & 28 & 64 \\ +\end{bmatrix} +$$ + +The `simple-ctx` doesn't support gpu acceleration. `simple-backend` demonstrates how to use other backends like CUDA and Metal. diff --git a/ggml/examples/simple/simple-backend.cpp b/ggml/examples/simple/simple-backend.cpp new file mode 100644 index 0000000..c7292b9 --- /dev/null +++ b/ggml/examples/simple/simple-backend.cpp @@ -0,0 +1,153 @@ +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include + +static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) { + (void) level; + (void) user_data; + fputs(text, stderr); + fflush(stderr); +} + +// This is a simple model with two tensors a and b +struct simple_model { + struct ggml_tensor * a {}; + struct ggml_tensor * b {}; + + // the backend to perform the computation (CPU, CUDA, METAL) + ggml_backend_t backend {}; + ggml_backend_t cpu_backend {}; + ggml_backend_sched_t sched {}; + + // storage for the graph and tensors + std::vector buf; +}; + +// initialize data of matrices to perform matrix multiplication +const int rows_A = 4, cols_A = 2; + +float matrix_A[rows_A * cols_A] = { + 2, 8, + 5, 1, + 4, 2, + 8, 6 +}; + +const int rows_B = 3, cols_B = 2; +/* Transpose([ + 10, 9, 5, + 5, 9, 4 +]) 2 rows, 3 cols */ +float matrix_B[rows_B * cols_B] = { + 10, 5, + 9, 9, + 5, 4 +}; + + +// initialize the tensors of the model in this case two matrices 2x2 +void init_model(simple_model & model) { + ggml_log_set(ggml_log_callback_default, nullptr); + + ggml_backend_load_all(); + + model.backend = ggml_backend_init_best(); + model.cpu_backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); + + ggml_backend_t backends[2] = { model.backend, model.cpu_backend }; + model.sched = ggml_backend_sched_new(backends, nullptr, 2, GGML_DEFAULT_GRAPH_SIZE, false, true); +} + +// build the compute graph to perform a matrix multiplication +struct ggml_cgraph * build_graph(simple_model& model) { + size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(); + model.buf.resize(buf_size); + + struct ggml_init_params params0 = { + /*.mem_size =*/ buf_size, + /*.mem_buffer =*/ model.buf.data(), + /*.no_alloc =*/ true, // the tensors will be allocated later + }; + + // create a context to build the graph + struct ggml_context * ctx = ggml_init(params0); + + struct ggml_cgraph * gf = ggml_new_graph(ctx); + + // create tensors + model.a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_A, rows_A); + model.b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, cols_B, rows_B); + + // result = a*b^T + struct ggml_tensor * result = ggml_mul_mat(ctx, model.a, model.b); + + // build operations nodes + ggml_build_forward_expand(gf, result); + + ggml_free(ctx); + + return gf; +} + +// compute with backend +struct ggml_tensor * compute(simple_model & model, struct ggml_cgraph * gf) { + ggml_backend_sched_reset(model.sched); + ggml_backend_sched_alloc_graph(model.sched, gf); + + // load data from cpu memory to backend buffer + ggml_backend_tensor_set(model.a, matrix_A, 0, ggml_nbytes(model.a)); + ggml_backend_tensor_set(model.b, matrix_B, 0, ggml_nbytes(model.b)); + + // compute the graph + ggml_backend_sched_graph_compute(model.sched, gf); + + // in this case, the output tensor is the last one in the graph + return ggml_graph_node(gf, -1); +} + +int main(void) { + ggml_time_init(); + + simple_model model; + init_model(model); + + struct ggml_cgraph * gf = build_graph(model); + + // perform computation + struct ggml_tensor * result = compute(model, gf); + + // create a array to print result + std::vector out_data(ggml_nelements(result)); + + // bring the data from the backend memory + ggml_backend_tensor_get(result, out_data.data(), 0, ggml_nbytes(result)); + + // expected result: + // [ 60.00 55.00 50.00 110.00 + // 90.00 54.00 54.00 126.00 + // 42.00 29.00 28.00 64.00 ] + + printf("mul mat (%d x %d) (transposed result):\n[", (int) result->ne[0], (int) result->ne[1]); + for (int j = 0; j < result->ne[1] /* rows */; j++) { + if (j > 0) { + printf("\n"); + } + + for (int i = 0; i < result->ne[0] /* cols */; i++) { + printf(" %.2f", out_data[j * result->ne[0] + i]); + } + } + printf(" ]\n"); + + // release backend memory and free backend + ggml_backend_sched_free(model.sched); + ggml_backend_free(model.backend); + ggml_backend_free(model.cpu_backend); + return 0; +} diff --git a/ggml/examples/simple/simple-ctx.cpp b/ggml/examples/simple/simple-ctx.cpp new file mode 100644 index 0000000..fea8f40 --- /dev/null +++ b/ggml/examples/simple/simple-ctx.cpp @@ -0,0 +1,127 @@ +#include "ggml.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +// This is a simple model with two tensors a and b +struct simple_model { + struct ggml_tensor * a; + struct ggml_tensor * b; + + // the context to define the tensor information (dimensions, size, memory data) + struct ggml_context * ctx; +}; + +// initialize the tensors of the model in this case two matrices 2x2 +void load_model(simple_model & model, float * a, float * b, int rows_A, int cols_A, int rows_B, int cols_B) { + size_t ctx_size = 0; + { + ctx_size += rows_A * cols_A * ggml_type_size(GGML_TYPE_F32); // tensor a + ctx_size += rows_B * cols_B * ggml_type_size(GGML_TYPE_F32); // tensor b + ctx_size += 2 * ggml_tensor_overhead(), // tensors + ctx_size += ggml_graph_overhead(); // compute graph + ctx_size += 1024; // some overhead + } + + struct ggml_init_params params { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ false, // NOTE: this should be false when using the legacy API + }; + + // create context + model.ctx = ggml_init(params); + + // create tensors + model.a = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_A, rows_A); + model.b = ggml_new_tensor_2d(model.ctx, GGML_TYPE_F32, cols_B, rows_B); + + memcpy(model.a->data, a, ggml_nbytes(model.a)); + memcpy(model.b->data, b, ggml_nbytes(model.b)); +} + +// build the compute graph to perform a matrix multiplication +struct ggml_cgraph * build_graph(const simple_model& model) { + struct ggml_cgraph * gf = ggml_new_graph(model.ctx); + + // result = a*b^T + struct ggml_tensor * result = ggml_mul_mat(model.ctx, model.a, model.b); + + ggml_build_forward_expand(gf, result); + return gf; +} + +// compute with backend +struct ggml_tensor * compute(const simple_model & model) { + struct ggml_cgraph * gf = build_graph(model); + + int n_threads = 1; // number of threads to perform some operations with multi-threading + + ggml_graph_compute_with_ctx(model.ctx, gf, n_threads); + + // in this case, the output tensor is the last one in the graph + return ggml_graph_node(gf, -1); +} + +int main(void) { + ggml_time_init(); + + // initialize data of matrices to perform matrix multiplication + const int rows_A = 4, cols_A = 2; + + float matrix_A[rows_A * cols_A] = { + 2, 8, + 5, 1, + 4, 2, + 8, 6 + }; + + const int rows_B = 3, cols_B = 2; + /* Transpose([ + 10, 9, 5, + 5, 9, 4 + ]) 2 rows, 3 cols */ + float matrix_B[rows_B * cols_B] = { + 10, 5, + 9, 9, + 5, 4 + }; + + simple_model model; + load_model(model, matrix_A, matrix_B, rows_A, cols_A, rows_B, cols_B); + + // perform computation in cpu + struct ggml_tensor * result = compute(model); + + // get the result data pointer as a float array to print + std::vector out_data(ggml_nelements(result)); + memcpy(out_data.data(), result->data, ggml_nbytes(result)); + + // expected result: + // [ 60.00 55.00 50.00 110.00 + // 90.00 54.00 54.00 126.00 + // 42.00 29.00 28.00 64.00 ] + + printf("mul mat (%d x %d) (transposed result):\n[", (int) result->ne[0], (int) result->ne[1]); + for (int j = 0; j < result->ne[1] /* rows */; j++) { + if (j > 0) { + printf("\n"); + } + + for (int i = 0; i < result->ne[0] /* cols */; i++) { + printf(" %.2f", out_data[j * result->ne[0] + i]); + } + } + printf(" ]\n"); + + // free memory + ggml_free(model.ctx); + return 0; +} diff --git a/ggml/examples/stb_image.h b/ggml/examples/stb_image.h new file mode 100644 index 0000000..5e807a0 --- /dev/null +++ b/ggml/examples/stb_image.h @@ -0,0 +1,7987 @@ +/* stb_image - v2.28 - public domain image loader - http://nothings.org/stb + no warranty implied; use at your own risk + + Do this: + #define STB_IMAGE_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + + // i.e. it should look like this: + #include ... + #include ... + #include ... + #define STB_IMAGE_IMPLEMENTATION + #include "stb_image.h" + + You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. + And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free + + + QUICK NOTES: + Primarily of interest to game developers and other people who can + avoid problematic images and only need the trivial interface + + JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) + PNG 1/2/4/8/16-bit-per-channel + + TGA (not sure what subset, if a subset) + BMP non-1bpp, non-RLE + PSD (composited view only, no extra channels, 8/16 bit-per-channel) + + GIF (*comp always reports as 4-channel) + HDR (radiance rgbE format) + PIC (Softimage PIC) + PNM (PPM and PGM binary only) + + Animated GIF still needs a proper API, but here's one way to do it: + http://gist.github.com/urraka/685d9a6340b26b830d49 + + - decode from memory or through FILE (define STBI_NO_STDIO to remove code) + - decode from arbitrary I/O callbacks + - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) + + Full documentation under "DOCUMENTATION" below. + + +LICENSE + + See end of file for license information. + +RECENT REVISION HISTORY: + + 2.28 (2023-01-29) many error fixes, security errors, just tons of stuff + 2.27 (2021-07-11) document stbi_info better, 16-bit PNM support, bug fixes + 2.26 (2020-07-13) many minor fixes + 2.25 (2020-02-02) fix warnings + 2.24 (2020-02-02) fix warnings; thread-local failure_reason and flip_vertically + 2.23 (2019-08-11) fix clang static analysis warning + 2.22 (2019-03-04) gif fixes, fix warnings + 2.21 (2019-02-25) fix typo in comment + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings + 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes + 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 + RGB-format JPEG; remove white matting in PSD; + allocate large structures on the stack; + correct channel count for PNG & BMP + 2.10 (2016-01-22) avoid warning introduced in 2.09 + 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED + + See end of file for full revision history. + + + ============================ Contributors ========================= + + Image formats Extensions, features + Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) + Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) + Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) + Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) + Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) + Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) + Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) + github:urraka (animated gif) Junggon Kim (PNM comments) + Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) + socks-the-fox (16-bit PNG) + Jeremy Sawicki (handle all ImageNet JPGs) + Optimizations & bugfixes Mikhail Morozov (1-bit BMP) + Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) + Arseny Kapoulkine Simon Breuss (16-bit PNM) + John-Mark Allen + Carmelo J Fdez-Aguera + + Bug & warning fixes + Marc LeBlanc David Woo Guillaume George Martins Mozeiko + Christpher Lloyd Jerry Jansson Joseph Thomson Blazej Dariusz Roszkowski + Phil Jordan Dave Moore Roy Eltham + Hayaki Saito Nathan Reed Won Chun + Luke Graham Johan Duparc Nick Verigakis the Horde3D community + Thomas Ruf Ronny Chevalier github:rlyeh + Janez Zemva John Bartholomew Michal Cichon github:romigrou + Jonathan Blow Ken Hamada Tero Hanninen github:svdijk + Eugene Golushkov Laurent Gomila Cort Stratton github:snagar + Aruelien Pocheville Sergio Gonzalez Thibault Reuille github:Zelex + Cass Everitt Ryamond Barbiero github:grim210 + Paul Du Bois Engin Manap Aldo Culquicondor github:sammyhw + Philipp Wiesemann Dale Weiler Oriol Ferrer Mesia github:phprus + Josh Tobin Neil Bickford Matthew Gregan github:poppolopoppo + Julian Raschke Gregory Mullen Christian Floisand github:darealshinji + Baldur Karlsson Kevin Schmidt JR Smith github:Michaelangel007 + Brad Weinberger Matvey Cherevko github:mosra + Luca Sas Alexander Veselov Zack Middleton [reserved] + Ryan C. Gordon [reserved] [reserved] + DO NOT ADD YOUR NAME HERE + + Jacko Dirks + + To add your name to the credits, pick a random blank space in the middle and fill it. + 80% of merge conflicts on stb PRs are due to people adding their name at the end + of the credits. +*/ + +#ifndef STBI_INCLUDE_STB_IMAGE_H +#define STBI_INCLUDE_STB_IMAGE_H + +// DOCUMENTATION +// +// Limitations: +// - no 12-bit-per-channel JPEG +// - no JPEGs with arithmetic coding +// - GIF always returns *comp=4 +// +// Basic usage (see HDR discussion below for HDR usage): +// int x,y,n; +// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); +// // ... process data if not NULL ... +// // ... x = width, y = height, n = # 8-bit components per pixel ... +// // ... replace '0' with '1'..'4' to force that many components per pixel +// // ... but 'n' will always be the number that it would have been if you said 0 +// stbi_image_free(data); +// +// Standard parameters: +// int *x -- outputs image width in pixels +// int *y -- outputs image height in pixels +// int *channels_in_file -- outputs # of image components in image file +// int desired_channels -- if non-zero, # of image components requested in result +// +// The return value from an image loader is an 'unsigned char *' which points +// to the pixel data, or NULL on an allocation failure or if the image is +// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, +// with each pixel consisting of N interleaved 8-bit components; the first +// pixel pointed to is top-left-most in the image. There is no padding between +// image scanlines or between pixels, regardless of format. The number of +// components N is 'desired_channels' if desired_channels is non-zero, or +// *channels_in_file otherwise. If desired_channels is non-zero, +// *channels_in_file has the number of components that _would_ have been +// output otherwise. E.g. if you set desired_channels to 4, you will always +// get RGBA output, but you can check *channels_in_file to see if it's trivially +// opaque because e.g. there were only 3 channels in the source image. +// +// An output image with N components has the following components interleaved +// in this order in each pixel: +// +// N=#comp components +// 1 grey +// 2 grey, alpha +// 3 red, green, blue +// 4 red, green, blue, alpha +// +// If image loading fails for any reason, the return value will be NULL, +// and *x, *y, *channels_in_file will be unchanged. The function +// stbi_failure_reason() can be queried for an extremely brief, end-user +// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS +// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly +// more user-friendly ones. +// +// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. +// +// To query the width, height and component count of an image without having to +// decode the full file, you can use the stbi_info family of functions: +// +// int x,y,n,ok; +// ok = stbi_info(filename, &x, &y, &n); +// // returns ok=1 and sets x, y, n if image is a supported format, +// // 0 otherwise. +// +// Note that stb_image pervasively uses ints in its public API for sizes, +// including sizes of memory buffers. This is now part of the API and thus +// hard to change without causing breakage. As a result, the various image +// loaders all have certain limits on image size; these differ somewhat +// by format but generally boil down to either just under 2GB or just under +// 1GB. When the decoded image would be larger than this, stb_image decoding +// will fail. +// +// Additionally, stb_image will reject image files that have any of their +// dimensions set to a larger value than the configurable STBI_MAX_DIMENSIONS, +// which defaults to 2**24 = 16777216 pixels. Due to the above memory limit, +// the only way to have an image with such dimensions load correctly +// is for it to have a rather extreme aspect ratio. Either way, the +// assumption here is that such larger images are likely to be malformed +// or malicious. If you do need to load an image with individual dimensions +// larger than that, and it still fits in the overall size limit, you can +// #define STBI_MAX_DIMENSIONS on your own to be something larger. +// +// =========================================================================== +// +// UNICODE: +// +// If compiling for Windows and you wish to use Unicode filenames, compile +// with +// #define STBI_WINDOWS_UTF8 +// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert +// Windows wchar_t filenames to utf8. +// +// =========================================================================== +// +// Philosophy +// +// stb libraries are designed with the following priorities: +// +// 1. easy to use +// 2. easy to maintain +// 3. good performance +// +// Sometimes I let "good performance" creep up in priority over "easy to maintain", +// and for best performance I may provide less-easy-to-use APIs that give higher +// performance, in addition to the easy-to-use ones. Nevertheless, it's important +// to keep in mind that from the standpoint of you, a client of this library, +// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. +// +// Some secondary priorities arise directly from the first two, some of which +// provide more explicit reasons why performance can't be emphasized. +// +// - Portable ("ease of use") +// - Small source code footprint ("easy to maintain") +// - No dependencies ("ease of use") +// +// =========================================================================== +// +// I/O callbacks +// +// I/O callbacks allow you to read from arbitrary sources, like packaged +// files or some other source. Data read from callbacks are processed +// through a small internal buffer (currently 128 bytes) to try to reduce +// overhead. +// +// The three functions you must define are "read" (reads some bytes of data), +// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). +// +// =========================================================================== +// +// SIMD support +// +// The JPEG decoder will try to automatically use SIMD kernels on x86 when +// supported by the compiler. For ARM Neon support, you must explicitly +// request it. +// +// (The old do-it-yourself SIMD API is no longer supported in the current +// code.) +// +// On x86, SSE2 will automatically be used when available based on a run-time +// test; if not, the generic C versions are used as a fall-back. On ARM targets, +// the typical path is to have separate builds for NEON and non-NEON devices +// (at least this is true for iOS and Android). Therefore, the NEON support is +// toggled by a build flag: define STBI_NEON to get NEON loops. +// +// If for some reason you do not want to use any of SIMD code, or if +// you have issues compiling it, you can disable it entirely by +// defining STBI_NO_SIMD. +// +// =========================================================================== +// +// HDR image support (disable by defining STBI_NO_HDR) +// +// stb_image supports loading HDR images in general, and currently the Radiance +// .HDR file format specifically. You can still load any file through the existing +// interface; if you attempt to load an HDR file, it will be automatically remapped +// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; +// both of these constants can be reconfigured through this interface: +// +// stbi_hdr_to_ldr_gamma(2.2f); +// stbi_hdr_to_ldr_scale(1.0f); +// +// (note, do not use _inverse_ constants; stbi_image will invert them +// appropriately). +// +// Additionally, there is a new, parallel interface for loading files as +// (linear) floats to preserve the full dynamic range: +// +// float *data = stbi_loadf(filename, &x, &y, &n, 0); +// +// If you load LDR images through this interface, those images will +// be promoted to floating point values, run through the inverse of +// constants corresponding to the above: +// +// stbi_ldr_to_hdr_scale(1.0f); +// stbi_ldr_to_hdr_gamma(2.2f); +// +// Finally, given a filename (or an open file or memory block--see header +// file for details) containing image data, you can query for the "most +// appropriate" interface to use (that is, whether the image is HDR or +// not), using: +// +// stbi_is_hdr(char *filename); +// +// =========================================================================== +// +// iPhone PNG support: +// +// We optionally support converting iPhone-formatted PNGs (which store +// premultiplied BGRA) back to RGB, even though they're internally encoded +// differently. To enable this conversion, call +// stbi_convert_iphone_png_to_rgb(1). +// +// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per +// pixel to remove any premultiplied alpha *only* if the image file explicitly +// says there's premultiplied data (currently only happens in iPhone images, +// and only if iPhone convert-to-rgb processing is on). +// +// =========================================================================== +// +// ADDITIONAL CONFIGURATION +// +// - You can suppress implementation of any of the decoders to reduce +// your code footprint by #defining one or more of the following +// symbols before creating the implementation. +// +// STBI_NO_JPEG +// STBI_NO_PNG +// STBI_NO_BMP +// STBI_NO_PSD +// STBI_NO_TGA +// STBI_NO_GIF +// STBI_NO_HDR +// STBI_NO_PIC +// STBI_NO_PNM (.ppm and .pgm) +// +// - You can request *only* certain decoders and suppress all other ones +// (this will be more forward-compatible, as addition of new decoders +// doesn't require you to disable them explicitly): +// +// STBI_ONLY_JPEG +// STBI_ONLY_PNG +// STBI_ONLY_BMP +// STBI_ONLY_PSD +// STBI_ONLY_TGA +// STBI_ONLY_GIF +// STBI_ONLY_HDR +// STBI_ONLY_PIC +// STBI_ONLY_PNM (.ppm and .pgm) +// +// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still +// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB +// +// - If you define STBI_MAX_DIMENSIONS, stb_image will reject images greater +// than that size (in either width or height) without further processing. +// This is to let programs in the wild set an upper bound to prevent +// denial-of-service attacks on untrusted data, as one could generate a +// valid image of gigantic dimensions and force stb_image to allocate a +// huge block of memory and spend disproportionate time decoding it. By +// default this is set to (1 << 24), which is 16777216, but that's still +// very big. + +#ifndef STBI_NO_STDIO +#include +#endif // STBI_NO_STDIO + +#define STBI_VERSION 1 + +enum +{ + STBI_default = 0, // only used for desired_channels + + STBI_grey = 1, + STBI_grey_alpha = 2, + STBI_rgb = 3, + STBI_rgb_alpha = 4 +}; + +#include +typedef unsigned char stbi_uc; +typedef unsigned short stbi_us; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef STBIDEF +#ifdef STB_IMAGE_STATIC +#define STBIDEF static +#else +#define STBIDEF extern +#endif +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// PRIMARY API - works on images of any type +// + +// +// load image by filename, open file, or memory buffer +// + +typedef struct +{ + int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read + void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative + int (*eof) (void *user); // returns nonzero if we are at end of file/data +} stbi_io_callbacks; + +//////////////////////////////////// +// +// 8-bits-per-channel interface +// + +STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +// for stbi_load_from_file, file pointer is left pointing immediately after image +#endif + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +#endif + +#ifdef STBI_WINDOWS_UTF8 +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif + +//////////////////////////////////// +// +// 16-bits-per-channel interface +// + +STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +#endif + +//////////////////////////////////// +// +// float-per-channel interface +// +#ifndef STBI_NO_LINEAR + STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + + #ifndef STBI_NO_STDIO + STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); + #endif +#endif + +#ifndef STBI_NO_HDR + STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); + STBIDEF void stbi_hdr_to_ldr_scale(float scale); +#endif // STBI_NO_HDR + +#ifndef STBI_NO_LINEAR + STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); + STBIDEF void stbi_ldr_to_hdr_scale(float scale); +#endif // STBI_NO_LINEAR + +// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename); +STBIDEF int stbi_is_hdr_from_file(FILE *f); +#endif // STBI_NO_STDIO + + +// get a VERY brief reason for failure +// on most compilers (and ALL modern mainstream compilers) this is threadsafe +STBIDEF const char *stbi_failure_reason (void); + +// free the loaded image -- this is just free() +STBIDEF void stbi_image_free (void *retval_from_stbi_load); + +// get image dimensions & components without fully decoding +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit (char const *filename); +STBIDEF int stbi_is_16_bit_from_file(FILE *f); +#endif + + + +// for image formats that explicitly notate that they have premultiplied alpha, +// we just return the colors as stored in the file. set this flag to force +// unpremultiplication. results are undefined if the unpremultiply overflow. +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); + +// indicate whether we should process iphone images back to canonical format, +// or just pass them through "as-is" +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); + +// flip the image vertically, so the first pixel in the output array is the bottom left +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); + +// as above, but only applies to images loaded on the thread that calls the function +// this function is only available if your compiler supports thread-local variables; +// calling it will fail to link if your compiler doesn't +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply); +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert); +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip); + +// ZLIB client - used by PNG, available for other purposes + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); +STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + +STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + + +#ifdef __cplusplus +} +#endif + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBI_INCLUDE_STB_IMAGE_H + +#ifdef STB_IMAGE_IMPLEMENTATION + +#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ + || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ + || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ + || defined(STBI_ONLY_ZLIB) + #ifndef STBI_ONLY_JPEG + #define STBI_NO_JPEG + #endif + #ifndef STBI_ONLY_PNG + #define STBI_NO_PNG + #endif + #ifndef STBI_ONLY_BMP + #define STBI_NO_BMP + #endif + #ifndef STBI_ONLY_PSD + #define STBI_NO_PSD + #endif + #ifndef STBI_ONLY_TGA + #define STBI_NO_TGA + #endif + #ifndef STBI_ONLY_GIF + #define STBI_NO_GIF + #endif + #ifndef STBI_ONLY_HDR + #define STBI_NO_HDR + #endif + #ifndef STBI_ONLY_PIC + #define STBI_NO_PIC + #endif + #ifndef STBI_ONLY_PNM + #define STBI_NO_PNM + #endif +#endif + +#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) +#define STBI_NO_ZLIB +#endif + + +#include +#include // ptrdiff_t on osx +#include +#include +#include + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +#include // ldexp, pow +#endif + +#ifndef STBI_NO_STDIO +#include +#endif + +#ifndef STBI_ASSERT +#include +#define STBI_ASSERT(x) assert(x) +#endif + +#ifdef __cplusplus +#define STBI_EXTERN extern "C" +#else +#define STBI_EXTERN extern +#endif + + +#ifndef _MSC_VER + #ifdef __cplusplus + #define stbi_inline inline + #else + #define stbi_inline + #endif +#else + #define stbi_inline __forceinline +#endif + +#ifndef STBI_NO_THREAD_LOCALS + #if defined(__cplusplus) && __cplusplus >= 201103L + #define STBI_THREAD_LOCAL thread_local + #elif defined(__GNUC__) && __GNUC__ < 5 + #define STBI_THREAD_LOCAL __thread + #elif defined(_MSC_VER) + #define STBI_THREAD_LOCAL __declspec(thread) + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) + #define STBI_THREAD_LOCAL _Thread_local + #endif + + #ifndef STBI_THREAD_LOCAL + #if defined(__GNUC__) + #define STBI_THREAD_LOCAL __thread + #endif + #endif +#endif + +#if defined(_MSC_VER) || defined(__SYMBIAN32__) +typedef unsigned short stbi__uint16; +typedef signed short stbi__int16; +typedef unsigned int stbi__uint32; +typedef signed int stbi__int32; +#else +#include +typedef uint16_t stbi__uint16; +typedef int16_t stbi__int16; +typedef uint32_t stbi__uint32; +typedef int32_t stbi__int32; +#endif + +// should produce compiler error if size is wrong +typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBI_NOTUSED(v) (void)(v) +#else +#define STBI_NOTUSED(v) (void)sizeof(v) +#endif + +#ifdef _MSC_VER +#define STBI_HAS_LROTL +#endif + +#ifdef STBI_HAS_LROTL + #define stbi_lrot(x,y) _lrotl(x,y) +#else + #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (-(y) & 31))) +#endif + +#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) +// ok +#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." +#endif + +#ifndef STBI_MALLOC +#define STBI_MALLOC(sz) malloc(sz) +#define STBI_REALLOC(p,newsz) realloc(p,newsz) +#define STBI_FREE(p) free(p) +#endif + +#ifndef STBI_REALLOC_SIZED +#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) +#endif + +// x86/x64 detection +#if defined(__x86_64__) || defined(_M_X64) +#define STBI__X64_TARGET +#elif defined(__i386) || defined(_M_IX86) +#define STBI__X86_TARGET +#endif + +#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) +// gcc doesn't support sse2 intrinsics unless you compile with -msse2, +// which in turn means it gets to use SSE2 everywhere. This is unfortunate, +// but previous attempts to provide the SSE2 functions with runtime +// detection caused numerous issues. The way architecture extensions are +// exposed in GCC/Clang is, sadly, not really suited for one-file libs. +// New behavior: if compiled with -msse2, we use SSE2 without any +// detection; if not, we don't use it at all. +#define STBI_NO_SIMD +#endif + +#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) +// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET +// +// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the +// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. +// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not +// simultaneously enabling "-mstackrealign". +// +// See https://github.com/nothings/stb/issues/81 for more information. +// +// So default to no SSE2 on 32-bit MinGW. If you've read this far and added +// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. +#define STBI_NO_SIMD +#endif + +#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) +#define STBI_SSE2 +#include + +#ifdef _MSC_VER + +#if _MSC_VER >= 1400 // not VC6 +#include // __cpuid +static int stbi__cpuid3(void) +{ + int info[4]; + __cpuid(info,1); + return info[3]; +} +#else +static int stbi__cpuid3(void) +{ + int res; + __asm { + mov eax,1 + cpuid + mov res,edx + } + return res; +} +#endif + +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + int info3 = stbi__cpuid3(); + return ((info3 >> 26) & 1) != 0; +} +#endif + +#else // assume GCC-style if not VC++ +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + // If we're even attempting to compile this on GCC/Clang, that means + // -msse2 is on, which means the compiler is allowed to use SSE2 + // instructions at will, and so are we. + return 1; +} +#endif + +#endif +#endif + +// ARM NEON +#if defined(STBI_NO_SIMD) && defined(STBI_NEON) +#undef STBI_NEON +#endif + +#ifdef STBI_NEON +#include +#ifdef _MSC_VER +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name +#else +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) +#endif +#endif + +#ifndef STBI_SIMD_ALIGN +#define STBI_SIMD_ALIGN(type, name) type name +#endif + +#ifndef STBI_MAX_DIMENSIONS +#define STBI_MAX_DIMENSIONS (1 << 24) +#endif + +/////////////////////////////////////////////// +// +// stbi__context struct and start_xxx functions + +// stbi__context structure is our basic context used by all images, so it +// contains all the IO context, plus some basic image information +typedef struct +{ + stbi__uint32 img_x, img_y; + int img_n, img_out_n; + + stbi_io_callbacks io; + void *io_user_data; + + int read_from_callbacks; + int buflen; + stbi_uc buffer_start[128]; + int callback_already_read; + + stbi_uc *img_buffer, *img_buffer_end; + stbi_uc *img_buffer_original, *img_buffer_original_end; +} stbi__context; + + +static void stbi__refill_buffer(stbi__context *s); + +// initialize a memory-decode context +static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) +{ + s->io.read = NULL; + s->read_from_callbacks = 0; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; + s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; +} + +// initialize a callback-based context +static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) +{ + s->io = *c; + s->io_user_data = user; + s->buflen = sizeof(s->buffer_start); + s->read_from_callbacks = 1; + s->callback_already_read = 0; + s->img_buffer = s->img_buffer_original = s->buffer_start; + stbi__refill_buffer(s); + s->img_buffer_original_end = s->img_buffer_end; +} + +#ifndef STBI_NO_STDIO + +static int stbi__stdio_read(void *user, char *data, int size) +{ + return (int) fread(data,1,size,(FILE*) user); +} + +static void stbi__stdio_skip(void *user, int n) +{ + int ch; + fseek((FILE*) user, n, SEEK_CUR); + ch = fgetc((FILE*) user); /* have to read a byte to reset feof()'s flag */ + if (ch != EOF) { + ungetc(ch, (FILE *) user); /* push byte back onto stream if valid. */ + } +} + +static int stbi__stdio_eof(void *user) +{ + return feof((FILE*) user) || ferror((FILE *) user); +} + +static stbi_io_callbacks stbi__stdio_callbacks = +{ + stbi__stdio_read, + stbi__stdio_skip, + stbi__stdio_eof, +}; + +static void stbi__start_file(stbi__context *s, FILE *f) +{ + stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); +} + +//static void stop_file(stbi__context *s) { } + +#endif // !STBI_NO_STDIO + +static void stbi__rewind(stbi__context *s) +{ + // conceptually rewind SHOULD rewind to the beginning of the stream, + // but we just rewind to the beginning of the initial buffer, because + // we only use it after doing 'test', which only ever looks at at most 92 bytes + s->img_buffer = s->img_buffer_original; + s->img_buffer_end = s->img_buffer_original_end; +} + +enum +{ + STBI_ORDER_RGB, + STBI_ORDER_BGR +}; + +typedef struct +{ + int bits_per_channel; + int num_channels; + int channel_order; +} stbi__result_info; + +#ifndef STBI_NO_JPEG +static int stbi__jpeg_test(stbi__context *s); +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNG +static int stbi__png_test(stbi__context *s); +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__png_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_BMP +static int stbi__bmp_test(stbi__context *s); +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_TGA +static int stbi__tga_test(stbi__context *s); +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s); +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__psd_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_HDR +static int stbi__hdr_test(stbi__context *s); +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_test(stbi__context *s); +static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_GIF +static int stbi__gif_test(stbi__context *s); +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNM +static int stbi__pnm_test(stbi__context *s); +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__pnm_is16(stbi__context *s); +#endif + +static +#ifdef STBI_THREAD_LOCAL +STBI_THREAD_LOCAL +#endif +const char *stbi__g_failure_reason; + +STBIDEF const char *stbi_failure_reason(void) +{ + return stbi__g_failure_reason; +} + +#ifndef STBI_NO_FAILURE_STRINGS +static int stbi__err(const char *str) +{ + stbi__g_failure_reason = str; + return 0; +} +#endif + +static void *stbi__malloc(size_t size) +{ + return STBI_MALLOC(size); +} + +// stb_image uses ints pervasively, including for offset calculations. +// therefore the largest decoded image size we can support with the +// current code, even on 64-bit targets, is INT_MAX. this is not a +// significant limitation for the intended use case. +// +// we do, however, need to make sure our size calculations don't +// overflow. hence a few helper functions for size calculations that +// multiply integers together, making sure that they're non-negative +// and no overflow occurs. + +// return 1 if the sum is valid, 0 on overflow. +// negative terms are considered invalid. +static int stbi__addsizes_valid(int a, int b) +{ + if (b < 0) return 0; + // now 0 <= b <= INT_MAX, hence also + // 0 <= INT_MAX - b <= INTMAX. + // And "a + b <= INT_MAX" (which might overflow) is the + // same as a <= INT_MAX - b (no overflow) + return a <= INT_MAX - b; +} + +// returns 1 if the product is valid, 0 on overflow. +// negative factors are considered invalid. +static int stbi__mul2sizes_valid(int a, int b) +{ + if (a < 0 || b < 0) return 0; + if (b == 0) return 1; // mul-by-0 is always safe + // portable way to check for no overflows in a*b + return a <= INT_MAX/b; +} + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow +static int stbi__mad2sizes_valid(int a, int b, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); +} +#endif + +// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow +static int stbi__mad3sizes_valid(int a, int b, int c, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__addsizes_valid(a*b*c, add); +} + +// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); +} +#endif + +#if !defined(STBI_NO_JPEG) || !defined(STBI_NO_PNG) || !defined(STBI_NO_TGA) || !defined(STBI_NO_HDR) +// mallocs with size overflow checking +static void *stbi__malloc_mad2(int a, int b, int add) +{ + if (!stbi__mad2sizes_valid(a, b, add)) return NULL; + return stbi__malloc(a*b + add); +} +#endif + +static void *stbi__malloc_mad3(int a, int b, int c, int add) +{ + if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; + return stbi__malloc(a*b*c + add); +} + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) || !defined(STBI_NO_PNM) +static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) +{ + if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; + return stbi__malloc(a*b*c*d + add); +} +#endif + +// returns 1 if the sum of two signed ints is valid (between -2^31 and 2^31-1 inclusive), 0 on overflow. +static int stbi__addints_valid(int a, int b) +{ + if ((a >= 0) != (b >= 0)) return 1; // a and b have different signs, so no overflow + if (a < 0 && b < 0) return a >= INT_MIN - b; // same as a + b >= INT_MIN; INT_MIN - b cannot overflow since b < 0. + return a <= INT_MAX - b; +} + +// returns 1 if the product of two signed shorts is valid, 0 on overflow. +static int stbi__mul2shorts_valid(short a, short b) +{ + if (b == 0 || b == -1) return 1; // multiplication by 0 is always 0; check for -1 so SHRT_MIN/b doesn't overflow + if ((a >= 0) == (b >= 0)) return a <= SHRT_MAX/b; // product is positive, so similar to mul2sizes_valid + if (b < 0) return a <= SHRT_MIN / b; // same as a * b >= SHRT_MIN + return a >= SHRT_MIN / b; +} + +// stbi__err - error +// stbi__errpf - error returning pointer to float +// stbi__errpuc - error returning pointer to unsigned char + +#ifdef STBI_NO_FAILURE_STRINGS + #define stbi__err(x,y) 0 +#elif defined(STBI_FAILURE_USERMSG) + #define stbi__err(x,y) stbi__err(y) +#else + #define stbi__err(x,y) stbi__err(x) +#endif + +#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) +#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) + +STBIDEF void stbi_image_free(void *retval_from_stbi_load) +{ + STBI_FREE(retval_from_stbi_load); +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); +#endif + +#ifndef STBI_NO_HDR +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); +#endif + +static int stbi__vertically_flip_on_load_global = 0; + +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_global = flag_true_if_should_flip; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__vertically_flip_on_load stbi__vertically_flip_on_load_global +#else +static STBI_THREAD_LOCAL int stbi__vertically_flip_on_load_local, stbi__vertically_flip_on_load_set; + +STBIDEF void stbi_set_flip_vertically_on_load_thread(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load_local = flag_true_if_should_flip; + stbi__vertically_flip_on_load_set = 1; +} + +#define stbi__vertically_flip_on_load (stbi__vertically_flip_on_load_set \ + ? stbi__vertically_flip_on_load_local \ + : stbi__vertically_flip_on_load_global) +#endif // STBI_THREAD_LOCAL + +static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields + ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed + ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order + ri->num_channels = 0; + + // test the formats with a very explicit header first (at least a FOURCC + // or distinctive magic number first) + #ifndef STBI_NO_PNG + if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_BMP + if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_GIF + if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PSD + if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); + #else + STBI_NOTUSED(bpc); + #endif + #ifndef STBI_NO_PIC + if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); + #endif + + // then the formats that can end up attempting to load with just 1 or 2 + // bytes matching expectations; these are prone to false positives, so + // try them later + #ifndef STBI_NO_JPEG + if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNM + if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); + return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); + } + #endif + + #ifndef STBI_NO_TGA + // test tga last because it's a crappy test! + if (stbi__tga_test(s)) + return stbi__tga_load(s,x,y,comp,req_comp, ri); + #endif + + return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); +} + +static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi_uc *reduced; + + reduced = (stbi_uc *) stbi__malloc(img_len); + if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling + + STBI_FREE(orig); + return reduced; +} + +static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi__uint16 *enlarged; + + enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); + if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff + + STBI_FREE(orig); + return enlarged; +} + +static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) +{ + int row; + size_t bytes_per_row = (size_t)w * bytes_per_pixel; + stbi_uc temp[2048]; + stbi_uc *bytes = (stbi_uc *)image; + + for (row = 0; row < (h>>1); row++) { + stbi_uc *row0 = bytes + row*bytes_per_row; + stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; + // swap row0 with row1 + size_t bytes_left = bytes_per_row; + while (bytes_left) { + size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); + memcpy(temp, row0, bytes_copy); + memcpy(row0, row1, bytes_copy); + memcpy(row1, temp, bytes_copy); + row0 += bytes_copy; + row1 += bytes_copy; + bytes_left -= bytes_copy; + } + } +} + +#ifndef STBI_NO_GIF +static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) +{ + int slice; + int slice_size = w * h * bytes_per_pixel; + + stbi_uc *bytes = (stbi_uc *)image; + for (slice = 0; slice < z; ++slice) { + stbi__vertical_flip(bytes, w, h, bytes_per_pixel); + bytes += slice_size; + } +} +#endif + +static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 8) { + result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 8; + } + + // @TODO: move stbi__convert_format to here + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); + } + + return (unsigned char *) result; +} + +static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); + + if (result == NULL) + return NULL; + + // it is the responsibility of the loaders to make sure we get either 8 or 16 bit. + STBI_ASSERT(ri.bits_per_channel == 8 || ri.bits_per_channel == 16); + + if (ri.bits_per_channel != 16) { + result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 16; + } + + // @TODO: move stbi__convert_format16 to here + // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); + } + + return (stbi__uint16 *) result; +} + +#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) +static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) +{ + if (stbi__vertically_flip_on_load && result != NULL) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); + } +} +#endif + +#ifndef STBI_NO_STDIO + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); +#endif + +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbi__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + + +STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + unsigned char *result; + if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__uint16 *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + stbi__uint16 *result; + if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file_16(f,x,y,comp,req_comp); + fclose(f); + return result; +} + + +#endif //!STBI_NO_STDIO + +STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_mem(&s,buffer,len); + + result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); + if (stbi__vertically_flip_on_load) { + stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); + } + + return result; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *data; + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + stbi__result_info ri; + float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); + if (hdr_data) + stbi__float_postprocess(hdr_data,x,y,comp,req_comp); + return hdr_data; + } + #endif + data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); + if (data) + return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); + return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); +} + +STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_STDIO +STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + float *result; + FILE *f = stbi__fopen(filename, "rb"); + if (!f) return stbi__errpf("can't fopen", "Unable to open file"); + result = stbi_loadf_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_file(&s,f); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} +#endif // !STBI_NO_STDIO + +#endif // !STBI_NO_LINEAR + +// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is +// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always +// reports false! + +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(buffer); + STBI_NOTUSED(len); + return 0; + #endif +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result=0; + if (f) { + result = stbi_is_hdr_from_file(f); + fclose(f); + } + return result; +} + +STBIDEF int stbi_is_hdr_from_file(FILE *f) +{ + #ifndef STBI_NO_HDR + long pos = ftell(f); + int res; + stbi__context s; + stbi__start_file(&s,f); + res = stbi__hdr_test(&s); + fseek(f, pos, SEEK_SET); + return res; + #else + STBI_NOTUSED(f); + return 0; + #endif +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(clbk); + STBI_NOTUSED(user); + return 0; + #endif +} + +#ifndef STBI_NO_LINEAR +static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; + +STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } +STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } +#endif + +static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; + +STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } +STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } + + +////////////////////////////////////////////////////////////////////////////// +// +// Common code used by all image loaders +// + +enum +{ + STBI__SCAN_load=0, + STBI__SCAN_type, + STBI__SCAN_header +}; + +static void stbi__refill_buffer(stbi__context *s) +{ + int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); + s->callback_already_read += (int) (s->img_buffer - s->img_buffer_original); + if (n == 0) { + // at end of file, treat same as if from memory, but need to handle case + // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file + s->read_from_callbacks = 0; + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start+1; + *s->img_buffer = 0; + } else { + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start + n; + } +} + +stbi_inline static stbi_uc stbi__get8(stbi__context *s) +{ + if (s->img_buffer < s->img_buffer_end) + return *s->img_buffer++; + if (s->read_from_callbacks) { + stbi__refill_buffer(s); + return *s->img_buffer++; + } + return 0; +} + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_HDR) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +stbi_inline static int stbi__at_eof(stbi__context *s) +{ + if (s->io.read) { + if (!(s->io.eof)(s->io_user_data)) return 0; + // if feof() is true, check if buffer = end + // special case: we've only got the special 0 character at the end + if (s->read_from_callbacks == 0) return 1; + } + + return s->img_buffer >= s->img_buffer_end; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) +// nothing +#else +static void stbi__skip(stbi__context *s, int n) +{ + if (n == 0) return; // already there! + if (n < 0) { + s->img_buffer = s->img_buffer_end; + return; + } + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + s->img_buffer = s->img_buffer_end; + (s->io.skip)(s->io_user_data, n - blen); + return; + } + } + s->img_buffer += n; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_TGA) && defined(STBI_NO_HDR) && defined(STBI_NO_PNM) +// nothing +#else +static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) +{ + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + int res, count; + + memcpy(buffer, s->img_buffer, blen); + + count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); + res = (count == (n-blen)); + s->img_buffer = s->img_buffer_end; + return res; + } + } + + if (s->img_buffer+n <= s->img_buffer_end) { + memcpy(buffer, s->img_buffer, n); + s->img_buffer += n; + return 1; + } else + return 0; +} +#endif + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static int stbi__get16be(stbi__context *s) +{ + int z = stbi__get8(s); + return (z << 8) + stbi__get8(s); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) && defined(STBI_NO_PIC) +// nothing +#else +static stbi__uint32 stbi__get32be(stbi__context *s) +{ + stbi__uint32 z = stbi__get16be(s); + return (z << 16) + stbi__get16be(s); +} +#endif + +#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) +// nothing +#else +static int stbi__get16le(stbi__context *s) +{ + int z = stbi__get8(s); + return z + (stbi__get8(s) << 8); +} +#endif + +#ifndef STBI_NO_BMP +static stbi__uint32 stbi__get32le(stbi__context *s) +{ + stbi__uint32 z = stbi__get16le(s); + z += (stbi__uint32)stbi__get16le(s) << 16; + return z; +} +#endif + +#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings + +#if defined(STBI_NO_JPEG) && defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +////////////////////////////////////////////////////////////////////////////// +// +// generic converter from built-in img_n to req_comp +// individual types do this automatically as much as possible (e.g. jpeg +// does all cases internally since it needs to colorspace convert anyway, +// and it never has alpha, so very few cases ). png can automatically +// interleave an alpha=255 channel, but falls back to this for other cases +// +// assume data buffer is malloced, so malloc a new one and free that one +// only failure mode is malloc failing + +static stbi_uc stbi__compute_y(int r, int g, int b) +{ + return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_BMP) && defined(STBI_NO_PSD) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) && defined(STBI_NO_PIC) && defined(STBI_NO_PNM) +// nothing +#else +static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + unsigned char *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); + if (good == NULL) { + STBI_FREE(data); + return stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + unsigned char *src = data + j * x * img_n ; + unsigned char *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 stbi__compute_y_16(int r, int g, int b) +{ + return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); +} +#endif + +#if defined(STBI_NO_PNG) && defined(STBI_NO_PSD) +// nothing +#else +static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + stbi__uint16 *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); + if (good == NULL) { + STBI_FREE(data); + return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + stbi__uint16 *src = data + j * x * img_n ; + stbi__uint16 *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); STBI_FREE(data); STBI_FREE(good); return (stbi__uint16*) stbi__errpuc("unsupported", "Unsupported format conversion"); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) +{ + int i,k,n; + float *output; + if (!data) return NULL; + output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); + } + } + if (n < comp) { + for (i=0; i < x*y; ++i) { + output[i*comp + n] = data[i*comp + n]/255.0f; + } + } + STBI_FREE(data); + return output; +} +#endif + +#ifndef STBI_NO_HDR +#define stbi__float2int(x) ((int) (x)) +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) +{ + int i,k,n; + stbi_uc *output; + if (!data) return NULL; + output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + if (k < comp) { + float z = data[i*comp+k] * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + } + STBI_FREE(data); + return output; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// "baseline" JPEG/JFIF decoder +// +// simple implementation +// - doesn't support delayed output of y-dimension +// - simple interface (only one output format: 8-bit interleaved RGB) +// - doesn't try to recover corrupt jpegs +// - doesn't allow partial loading, loading multiple at once +// - still fast on x86 (copying globals into locals doesn't help x86) +// - allocates lots of intermediate memory (full size of all components) +// - non-interleaved case requires this anyway +// - allows good upsampling (see next) +// high-quality +// - upsampled channels are bilinearly interpolated, even across blocks +// - quality integer IDCT derived from IJG's 'slow' +// performance +// - fast huffman; reasonable integer IDCT +// - some SIMD kernels for common paths on targets with SSE2/NEON +// - uses a lot of intermediate memory, could cache poorly + +#ifndef STBI_NO_JPEG + +// huffman decoding acceleration +#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache + +typedef struct +{ + stbi_uc fast[1 << FAST_BITS]; + // weirdly, repacking this into AoS is a 10% speed loss, instead of a win + stbi__uint16 code[256]; + stbi_uc values[256]; + stbi_uc size[257]; + unsigned int maxcode[18]; + int delta[17]; // old 'firstsymbol' - old 'firstcode' +} stbi__huffman; + +typedef struct +{ + stbi__context *s; + stbi__huffman huff_dc[4]; + stbi__huffman huff_ac[4]; + stbi__uint16 dequant[4][64]; + stbi__int16 fast_ac[4][1 << FAST_BITS]; + +// sizes for components, interleaved MCUs + int img_h_max, img_v_max; + int img_mcu_x, img_mcu_y; + int img_mcu_w, img_mcu_h; + +// definition of jpeg image component + struct + { + int id; + int h,v; + int tq; + int hd,ha; + int dc_pred; + + int x,y,w2,h2; + stbi_uc *data; + void *raw_data, *raw_coeff; + stbi_uc *linebuf; + short *coeff; // progressive only + int coeff_w, coeff_h; // number of 8x8 coefficient blocks + } img_comp[4]; + + stbi__uint32 code_buffer; // jpeg entropy-coded buffer + int code_bits; // number of valid bits + unsigned char marker; // marker seen while filling entropy buffer + int nomore; // flag if we saw a marker so must stop + + int progressive; + int spec_start; + int spec_end; + int succ_high; + int succ_low; + int eob_run; + int jfif; + int app14_color_transform; // Adobe APP14 tag + int rgb; + + int scan_n, order[4]; + int restart_interval, todo; + +// kernels + void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); + void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); + stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); +} stbi__jpeg; + +static int stbi__build_huffman(stbi__huffman *h, int *count) +{ + int i,j,k=0; + unsigned int code; + // build size list for each symbol (from JPEG spec) + for (i=0; i < 16; ++i) { + for (j=0; j < count[i]; ++j) { + h->size[k++] = (stbi_uc) (i+1); + if(k >= 257) return stbi__err("bad size list","Corrupt JPEG"); + } + } + h->size[k] = 0; + + // compute actual symbols (from jpeg spec) + code = 0; + k = 0; + for(j=1; j <= 16; ++j) { + // compute delta to add to code to compute symbol id + h->delta[j] = k - code; + if (h->size[k] == j) { + while (h->size[k] == j) + h->code[k++] = (stbi__uint16) (code++); + if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); + } + // compute largest code + 1 for this size, preshifted as needed later + h->maxcode[j] = code << (16-j); + code <<= 1; + } + h->maxcode[j] = 0xffffffff; + + // build non-spec acceleration table; 255 is flag for not-accelerated + memset(h->fast, 255, 1 << FAST_BITS); + for (i=0; i < k; ++i) { + int s = h->size[i]; + if (s <= FAST_BITS) { + int c = h->code[i] << (FAST_BITS-s); + int m = 1 << (FAST_BITS-s); + for (j=0; j < m; ++j) { + h->fast[c+j] = (stbi_uc) i; + } + } + } + return 1; +} + +// build a table that decodes both magnitude and value of small ACs in +// one go. +static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) +{ + int i; + for (i=0; i < (1 << FAST_BITS); ++i) { + stbi_uc fast = h->fast[i]; + fast_ac[i] = 0; + if (fast < 255) { + int rs = h->values[fast]; + int run = (rs >> 4) & 15; + int magbits = rs & 15; + int len = h->size[fast]; + + if (magbits && len + magbits <= FAST_BITS) { + // magnitude code followed by receive_extend code + int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); + int m = 1 << (magbits - 1); + if (k < m) k += (~0U << magbits) + 1; + // if the result is small enough, we can fit it in fast_ac table + if (k >= -128 && k <= 127) + fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); + } + } + } +} + +static void stbi__grow_buffer_unsafe(stbi__jpeg *j) +{ + do { + unsigned int b = j->nomore ? 0 : stbi__get8(j->s); + if (b == 0xff) { + int c = stbi__get8(j->s); + while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes + if (c != 0) { + j->marker = (unsigned char) c; + j->nomore = 1; + return; + } + } + j->code_buffer |= b << (24 - j->code_bits); + j->code_bits += 8; + } while (j->code_bits <= 24); +} + +// (1 << n) - 1 +static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; + +// decode a jpeg huffman value from the bitstream +stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) +{ + unsigned int temp; + int c,k; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + // look at the top FAST_BITS and determine what symbol ID it is, + // if the code is <= FAST_BITS + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + k = h->fast[c]; + if (k < 255) { + int s = h->size[k]; + if (s > j->code_bits) + return -1; + j->code_buffer <<= s; + j->code_bits -= s; + return h->values[k]; + } + + // naive test is to shift the code_buffer down so k bits are + // valid, then test against maxcode. To speed this up, we've + // preshifted maxcode left so that it has (16-k) 0s at the + // end; in other words, regardless of the number of bits, it + // wants to be compared against something shifted to have 16; + // that way we don't need to shift inside the loop. + temp = j->code_buffer >> 16; + for (k=FAST_BITS+1 ; ; ++k) + if (temp < h->maxcode[k]) + break; + if (k == 17) { + // error! code not found + j->code_bits -= 16; + return -1; + } + + if (k > j->code_bits) + return -1; + + // convert the huffman code to the symbol id + c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; + if(c < 0 || c >= 256) // symbol id out of bounds! + return -1; + STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); + + // convert the id to a symbol + j->code_bits -= k; + j->code_buffer <<= k; + return h->values[c]; +} + +// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + + sgn = j->code_buffer >> 31; // sign bit always in MSB; 0 if MSB clear (positive), 1 if MSB set (negative) + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k + (stbi__jbias[n] & (sgn - 1)); +} + +// get some unsigned bits +stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) +{ + unsigned int k; + if (j->code_bits < n) stbi__grow_buffer_unsafe(j); + if (j->code_bits < n) return 0; // ran out of bits from stream, return 0s intead of continuing + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k; +} + +stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) +{ + unsigned int k; + if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); + if (j->code_bits < 1) return 0; // ran out of bits from stream, return 0s intead of continuing + k = j->code_buffer; + j->code_buffer <<= 1; + --j->code_bits; + return k & 0x80000000; +} + +// given a value that's at position X in the zigzag stream, +// where does it appear in the 8x8 matrix coded as row-major? +static const stbi_uc stbi__jpeg_dezigzag[64+15] = +{ + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63, + // let corrupt input sample past end + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63 +}; + +// decode one 64-entry block-- +static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) +{ + int diff,dc,k; + int t; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("bad huffman code","Corrupt JPEG"); + + // 0 all the ac values now so we can do it 32-bits at a time + memset(data,0,64*sizeof(data[0])); + + diff = t ? stbi__extend_receive(j, t) : 0; + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta","Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, dequant[0])) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * dequant[0]); + + // decode AC components, see JPEG spec + k = 1; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * dequant[zig]); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (rs != 0xf0) break; // end block + k += 16; + } else { + k += r; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); + } + } + } while (k < 64); + return 1; +} + +static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) +{ + int diff,dc; + int t; + if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + if (j->succ_high == 0) { + // first scan for DC coefficient, must be first + memset(data,0,64*sizeof(data[0])); // 0 all the ac values now + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0 || t > 15) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + diff = t ? stbi__extend_receive(j, t) : 0; + + if (!stbi__addints_valid(j->img_comp[b].dc_pred, diff)) return stbi__err("bad delta", "Corrupt JPEG"); + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + if (!stbi__mul2shorts_valid(dc, 1 << j->succ_low)) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + data[0] = (short) (dc * (1 << j->succ_low)); + } else { + // refinement scan for DC coefficient + if (stbi__jpeg_get_bit(j)) + data[0] += (short) (1 << j->succ_low); + } + return 1; +} + +// @OPTIMIZE: store non-zigzagged during the decode passes, +// and only de-zigzag when dequantizing +static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) +{ + int k; + if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->succ_high == 0) { + int shift = j->succ_low; + + if (j->eob_run) { + --j->eob_run; + return 1; + } + + k = j->spec_start; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + if (s > j->code_bits) return stbi__err("bad huffman code", "Combined length longer than code bits available"); + j->code_buffer <<= s; + j->code_bits -= s; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * (1 << shift)); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r); + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + --j->eob_run; + break; + } + k += 16; + } else { + k += r; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * (1 << shift)); + } + } + } while (k <= j->spec_end); + } else { + // refinement scan for these AC coefficients + + short bit = (short) (1 << j->succ_low); + + if (j->eob_run) { + --j->eob_run; + for (k = j->spec_start; k <= j->spec_end; ++k) { + short *p = &data[stbi__jpeg_dezigzag[k]]; + if (*p != 0) + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } + } else { + k = j->spec_start; + do { + int r,s; + int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r) - 1; + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + r = 64; // force end of block + } else { + // r=15 s=0 should write 16 0s, so we just do + // a run of 15 0s and then write s (which is 0), + // so we don't have to do anything special here + } + } else { + if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); + // sign bit + if (stbi__jpeg_get_bit(j)) + s = bit; + else + s = -bit; + } + + // advance by r + while (k <= j->spec_end) { + short *p = &data[stbi__jpeg_dezigzag[k++]]; + if (*p != 0) { + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } else { + if (r == 0) { + *p = (short) s; + break; + } + --r; + } + } + } while (k <= j->spec_end); + } + } + return 1; +} + +// take a -128..127 value and stbi__clamp it and convert to 0..255 +stbi_inline static stbi_uc stbi__clamp(int x) +{ + // trick to use a single test to catch both cases + if ((unsigned int) x > 255) { + if (x < 0) return 0; + if (x > 255) return 255; + } + return (stbi_uc) x; +} + +#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) +#define stbi__fsh(x) ((x) * 4096) + +// derived from jidctint -- DCT_ISLOW +#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ + int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ + p2 = s2; \ + p3 = s6; \ + p1 = (p2+p3) * stbi__f2f(0.5411961f); \ + t2 = p1 + p3*stbi__f2f(-1.847759065f); \ + t3 = p1 + p2*stbi__f2f( 0.765366865f); \ + p2 = s0; \ + p3 = s4; \ + t0 = stbi__fsh(p2+p3); \ + t1 = stbi__fsh(p2-p3); \ + x0 = t0+t3; \ + x3 = t0-t3; \ + x1 = t1+t2; \ + x2 = t1-t2; \ + t0 = s7; \ + t1 = s5; \ + t2 = s3; \ + t3 = s1; \ + p3 = t0+t2; \ + p4 = t1+t3; \ + p1 = t0+t3; \ + p2 = t1+t2; \ + p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ + t0 = t0*stbi__f2f( 0.298631336f); \ + t1 = t1*stbi__f2f( 2.053119869f); \ + t2 = t2*stbi__f2f( 3.072711026f); \ + t3 = t3*stbi__f2f( 1.501321110f); \ + p1 = p5 + p1*stbi__f2f(-0.899976223f); \ + p2 = p5 + p2*stbi__f2f(-2.562915447f); \ + p3 = p3*stbi__f2f(-1.961570560f); \ + p4 = p4*stbi__f2f(-0.390180644f); \ + t3 += p1+p4; \ + t2 += p2+p3; \ + t1 += p2+p4; \ + t0 += p1+p3; + +static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) +{ + int i,val[64],*v=val; + stbi_uc *o; + short *d = data; + + // columns + for (i=0; i < 8; ++i,++d, ++v) { + // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing + if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 + && d[40]==0 && d[48]==0 && d[56]==0) { + // no shortcut 0 seconds + // (1|2|3|4|5|6|7)==0 0 seconds + // all separate -0.047 seconds + // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds + int dcterm = d[0]*4; + v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; + } else { + STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) + // constants scaled things up by 1<<12; let's bring them back + // down, but keep 2 extra bits of precision + x0 += 512; x1 += 512; x2 += 512; x3 += 512; + v[ 0] = (x0+t3) >> 10; + v[56] = (x0-t3) >> 10; + v[ 8] = (x1+t2) >> 10; + v[48] = (x1-t2) >> 10; + v[16] = (x2+t1) >> 10; + v[40] = (x2-t1) >> 10; + v[24] = (x3+t0) >> 10; + v[32] = (x3-t0) >> 10; + } + } + + for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { + // no fast case since the first 1D IDCT spread components out + STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) + // constants scaled things up by 1<<12, plus we had 1<<2 from first + // loop, plus horizontal and vertical each scale by sqrt(8) so together + // we've got an extra 1<<3, so 1<<17 total we need to remove. + // so we want to round that, which means adding 0.5 * 1<<17, + // aka 65536. Also, we'll end up with -128 to 127 that we want + // to encode as 0..255 by adding 128, so we'll add that before the shift + x0 += 65536 + (128<<17); + x1 += 65536 + (128<<17); + x2 += 65536 + (128<<17); + x3 += 65536 + (128<<17); + // tried computing the shifts into temps, or'ing the temps to see + // if any were out of range, but that was slower + o[0] = stbi__clamp((x0+t3) >> 17); + o[7] = stbi__clamp((x0-t3) >> 17); + o[1] = stbi__clamp((x1+t2) >> 17); + o[6] = stbi__clamp((x1-t2) >> 17); + o[2] = stbi__clamp((x2+t1) >> 17); + o[5] = stbi__clamp((x2-t1) >> 17); + o[3] = stbi__clamp((x3+t0) >> 17); + o[4] = stbi__clamp((x3-t0) >> 17); + } +} + +#ifdef STBI_SSE2 +// sse2 integer IDCT. not the fastest possible implementation but it +// produces bit-identical results to the generic C version so it's +// fully "transparent". +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + // This is constructed to match our regular (generic) integer IDCT exactly. + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i tmp; + + // dot product constant: even elems=x, odd elems=y + #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) + + // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) + // out(1) = c1[even]*x + c1[odd]*y + #define dct_rot(out0,out1, x,y,c0,c1) \ + __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ + __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ + __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ + __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ + __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ + __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) + + // out = in << 12 (in 16-bit, out 32-bit) + #define dct_widen(out, in) \ + __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ + __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) + + // wide add + #define dct_wadd(out, a, b) \ + __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_add_epi32(a##_h, b##_h) + + // wide sub + #define dct_wsub(out, a, b) \ + __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) + + // butterfly a/b, add bias, then shift by "s" and pack + #define dct_bfly32o(out0, out1, a,b,bias,s) \ + { \ + __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ + __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ + dct_wadd(sum, abiased, b); \ + dct_wsub(dif, abiased, b); \ + out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ + out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ + } + + // 8-bit interleave step (for transposes) + #define dct_interleave8(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi8(a, b); \ + b = _mm_unpackhi_epi8(tmp, b) + + // 16-bit interleave step (for transposes) + #define dct_interleave16(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi16(a, b); \ + b = _mm_unpackhi_epi16(tmp, b) + + #define dct_pass(bias,shift) \ + { \ + /* even part */ \ + dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ + __m128i sum04 = _mm_add_epi16(row0, row4); \ + __m128i dif04 = _mm_sub_epi16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ + dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ + __m128i sum17 = _mm_add_epi16(row1, row7); \ + __m128i sum35 = _mm_add_epi16(row3, row5); \ + dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ + dct_wadd(x4, y0o, y4o); \ + dct_wadd(x5, y1o, y5o); \ + dct_wadd(x6, y2o, y5o); \ + dct_wadd(x7, y3o, y4o); \ + dct_bfly32o(row0,row7, x0,x7,bias,shift); \ + dct_bfly32o(row1,row6, x1,x6,bias,shift); \ + dct_bfly32o(row2,row5, x2,x5,bias,shift); \ + dct_bfly32o(row3,row4, x3,x4,bias,shift); \ + } + + __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); + __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); + __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); + __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); + __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); + __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); + __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); + __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); + + // rounding biases in column/row passes, see stbi__idct_block for explanation. + __m128i bias_0 = _mm_set1_epi32(512); + __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); + + // load + row0 = _mm_load_si128((const __m128i *) (data + 0*8)); + row1 = _mm_load_si128((const __m128i *) (data + 1*8)); + row2 = _mm_load_si128((const __m128i *) (data + 2*8)); + row3 = _mm_load_si128((const __m128i *) (data + 3*8)); + row4 = _mm_load_si128((const __m128i *) (data + 4*8)); + row5 = _mm_load_si128((const __m128i *) (data + 5*8)); + row6 = _mm_load_si128((const __m128i *) (data + 6*8)); + row7 = _mm_load_si128((const __m128i *) (data + 7*8)); + + // column pass + dct_pass(bias_0, 10); + + { + // 16bit 8x8 transpose pass 1 + dct_interleave16(row0, row4); + dct_interleave16(row1, row5); + dct_interleave16(row2, row6); + dct_interleave16(row3, row7); + + // transpose pass 2 + dct_interleave16(row0, row2); + dct_interleave16(row1, row3); + dct_interleave16(row4, row6); + dct_interleave16(row5, row7); + + // transpose pass 3 + dct_interleave16(row0, row1); + dct_interleave16(row2, row3); + dct_interleave16(row4, row5); + dct_interleave16(row6, row7); + } + + // row pass + dct_pass(bias_1, 17); + + { + // pack + __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 + __m128i p1 = _mm_packus_epi16(row2, row3); + __m128i p2 = _mm_packus_epi16(row4, row5); + __m128i p3 = _mm_packus_epi16(row6, row7); + + // 8bit 8x8 transpose pass 1 + dct_interleave8(p0, p2); // a0e0a1e1... + dct_interleave8(p1, p3); // c0g0c1g1... + + // transpose pass 2 + dct_interleave8(p0, p1); // a0c0e0g0... + dct_interleave8(p2, p3); // b0d0f0h0... + + // transpose pass 3 + dct_interleave8(p0, p2); // a0b0c0d0... + dct_interleave8(p1, p3); // a4b4c4d4... + + // store + _mm_storel_epi64((__m128i *) out, p0); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p2); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p1); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p3); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); + } + +#undef dct_const +#undef dct_rot +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_interleave8 +#undef dct_interleave16 +#undef dct_pass +} + +#endif // STBI_SSE2 + +#ifdef STBI_NEON + +// NEON integer IDCT. should produce bit-identical +// results to the generic C version. +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; + + int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); + int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); + int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); + int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); + int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); + int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); + int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); + int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); + int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); + int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); + int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); + int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); + +#define dct_long_mul(out, inq, coeff) \ + int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) + +#define dct_long_mac(out, acc, inq, coeff) \ + int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) + +#define dct_widen(out, inq) \ + int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ + int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) + +// wide add +#define dct_wadd(out, a, b) \ + int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vaddq_s32(a##_h, b##_h) + +// wide sub +#define dct_wsub(out, a, b) \ + int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vsubq_s32(a##_h, b##_h) + +// butterfly a/b, then shift using "shiftop" by "s" and pack +#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ + { \ + dct_wadd(sum, a, b); \ + dct_wsub(dif, a, b); \ + out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ + out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ + } + +#define dct_pass(shiftop, shift) \ + { \ + /* even part */ \ + int16x8_t sum26 = vaddq_s16(row2, row6); \ + dct_long_mul(p1e, sum26, rot0_0); \ + dct_long_mac(t2e, p1e, row6, rot0_1); \ + dct_long_mac(t3e, p1e, row2, rot0_2); \ + int16x8_t sum04 = vaddq_s16(row0, row4); \ + int16x8_t dif04 = vsubq_s16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + int16x8_t sum15 = vaddq_s16(row1, row5); \ + int16x8_t sum17 = vaddq_s16(row1, row7); \ + int16x8_t sum35 = vaddq_s16(row3, row5); \ + int16x8_t sum37 = vaddq_s16(row3, row7); \ + int16x8_t sumodd = vaddq_s16(sum17, sum35); \ + dct_long_mul(p5o, sumodd, rot1_0); \ + dct_long_mac(p1o, p5o, sum17, rot1_1); \ + dct_long_mac(p2o, p5o, sum35, rot1_2); \ + dct_long_mul(p3o, sum37, rot2_0); \ + dct_long_mul(p4o, sum15, rot2_1); \ + dct_wadd(sump13o, p1o, p3o); \ + dct_wadd(sump24o, p2o, p4o); \ + dct_wadd(sump23o, p2o, p3o); \ + dct_wadd(sump14o, p1o, p4o); \ + dct_long_mac(x4, sump13o, row7, rot3_0); \ + dct_long_mac(x5, sump24o, row5, rot3_1); \ + dct_long_mac(x6, sump23o, row3, rot3_2); \ + dct_long_mac(x7, sump14o, row1, rot3_3); \ + dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ + dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ + dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ + dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ + } + + // load + row0 = vld1q_s16(data + 0*8); + row1 = vld1q_s16(data + 1*8); + row2 = vld1q_s16(data + 2*8); + row3 = vld1q_s16(data + 3*8); + row4 = vld1q_s16(data + 4*8); + row5 = vld1q_s16(data + 5*8); + row6 = vld1q_s16(data + 6*8); + row7 = vld1q_s16(data + 7*8); + + // add DC bias + row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); + + // column pass + dct_pass(vrshrn_n_s32, 10); + + // 16bit 8x8 transpose + { +// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. +// whether compilers actually get this is another story, sadly. +#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } +#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } + + // pass 1 + dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 + dct_trn16(row2, row3); + dct_trn16(row4, row5); + dct_trn16(row6, row7); + + // pass 2 + dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 + dct_trn32(row1, row3); + dct_trn32(row4, row6); + dct_trn32(row5, row7); + + // pass 3 + dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 + dct_trn64(row1, row5); + dct_trn64(row2, row6); + dct_trn64(row3, row7); + +#undef dct_trn16 +#undef dct_trn32 +#undef dct_trn64 + } + + // row pass + // vrshrn_n_s32 only supports shifts up to 16, we need + // 17. so do a non-rounding shift of 16 first then follow + // up with a rounding shift by 1. + dct_pass(vshrn_n_s32, 16); + + { + // pack and round + uint8x8_t p0 = vqrshrun_n_s16(row0, 1); + uint8x8_t p1 = vqrshrun_n_s16(row1, 1); + uint8x8_t p2 = vqrshrun_n_s16(row2, 1); + uint8x8_t p3 = vqrshrun_n_s16(row3, 1); + uint8x8_t p4 = vqrshrun_n_s16(row4, 1); + uint8x8_t p5 = vqrshrun_n_s16(row5, 1); + uint8x8_t p6 = vqrshrun_n_s16(row6, 1); + uint8x8_t p7 = vqrshrun_n_s16(row7, 1); + + // again, these can translate into one instruction, but often don't. +#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } +#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } + + // sadly can't use interleaved stores here since we only write + // 8 bytes to each scan line! + + // 8x8 8-bit transpose pass 1 + dct_trn8_8(p0, p1); + dct_trn8_8(p2, p3); + dct_trn8_8(p4, p5); + dct_trn8_8(p6, p7); + + // pass 2 + dct_trn8_16(p0, p2); + dct_trn8_16(p1, p3); + dct_trn8_16(p4, p6); + dct_trn8_16(p5, p7); + + // pass 3 + dct_trn8_32(p0, p4); + dct_trn8_32(p1, p5); + dct_trn8_32(p2, p6); + dct_trn8_32(p3, p7); + + // store + vst1_u8(out, p0); out += out_stride; + vst1_u8(out, p1); out += out_stride; + vst1_u8(out, p2); out += out_stride; + vst1_u8(out, p3); out += out_stride; + vst1_u8(out, p4); out += out_stride; + vst1_u8(out, p5); out += out_stride; + vst1_u8(out, p6); out += out_stride; + vst1_u8(out, p7); + +#undef dct_trn8_8 +#undef dct_trn8_16 +#undef dct_trn8_32 + } + +#undef dct_long_mul +#undef dct_long_mac +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_pass +} + +#endif // STBI_NEON + +#define STBI__MARKER_none 0xff +// if there's a pending marker from the entropy stream, return that +// otherwise, fetch from the stream and get a marker. if there's no +// marker, return 0xff, which is never a valid marker value +static stbi_uc stbi__get_marker(stbi__jpeg *j) +{ + stbi_uc x; + if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } + x = stbi__get8(j->s); + if (x != 0xff) return STBI__MARKER_none; + while (x == 0xff) + x = stbi__get8(j->s); // consume repeated 0xff fill bytes + return x; +} + +// in each scan, we'll have scan_n components, and the order +// of the components is specified by order[] +#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) + +// after a restart interval, stbi__jpeg_reset the entropy decoder and +// the dc prediction +static void stbi__jpeg_reset(stbi__jpeg *j) +{ + j->code_bits = 0; + j->code_buffer = 0; + j->nomore = 0; + j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; + j->marker = STBI__MARKER_none; + j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; + j->eob_run = 0; + // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, + // since we don't even allow 1<<30 pixels +} + +static int stbi__parse_entropy_coded_data(stbi__jpeg *z) +{ + stbi__jpeg_reset(z); + if (!z->progressive) { + if (z->scan_n == 1) { + int i,j; + STBI_SIMD_ALIGN(short, data[64]); + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + // if it's NOT a restart, then just bail, so we get corrupt data + // rather than no data + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + STBI_SIMD_ALIGN(short, data[64]); + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x)*8; + int y2 = (j*z->img_comp[n].v + y)*8; + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } else { + if (z->scan_n == 1) { + int i,j; + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + if (z->spec_start == 0) { + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } else { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) + return 0; + } + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x); + int y2 = (j*z->img_comp[n].v + y); + short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } +} + +static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) +{ + int i; + for (i=0; i < 64; ++i) + data[i] *= dequant[i]; +} + +static void stbi__jpeg_finish(stbi__jpeg *z) +{ + if (z->progressive) { + // dequantize and idct the data + int i,j,n; + for (n=0; n < z->s->img_n; ++n) { + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + } + } + } + } +} + +static int stbi__process_marker(stbi__jpeg *z, int m) +{ + int L; + switch (m) { + case STBI__MARKER_none: // no marker found + return stbi__err("expected marker","Corrupt JPEG"); + + case 0xDD: // DRI - specify restart interval + if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); + z->restart_interval = stbi__get16be(z->s); + return 1; + + case 0xDB: // DQT - define quantization table + L = stbi__get16be(z->s)-2; + while (L > 0) { + int q = stbi__get8(z->s); + int p = q >> 4, sixteen = (p != 0); + int t = q & 15,i; + if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); + if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); + + for (i=0; i < 64; ++i) + z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); + L -= (sixteen ? 129 : 65); + } + return L==0; + + case 0xC4: // DHT - define huffman table + L = stbi__get16be(z->s)-2; + while (L > 0) { + stbi_uc *v; + int sizes[16],i,n=0; + int q = stbi__get8(z->s); + int tc = q >> 4; + int th = q & 15; + if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); + for (i=0; i < 16; ++i) { + sizes[i] = stbi__get8(z->s); + n += sizes[i]; + } + if(n > 256) return stbi__err("bad DHT header","Corrupt JPEG"); // Loop over i < n would write past end of values! + L -= 17; + if (tc == 0) { + if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; + v = z->huff_dc[th].values; + } else { + if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; + v = z->huff_ac[th].values; + } + for (i=0; i < n; ++i) + v[i] = stbi__get8(z->s); + if (tc != 0) + stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); + L -= n; + } + return L==0; + } + + // check for comment block or APP blocks + if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { + L = stbi__get16be(z->s); + if (L < 2) { + if (m == 0xFE) + return stbi__err("bad COM len","Corrupt JPEG"); + else + return stbi__err("bad APP len","Corrupt JPEG"); + } + L -= 2; + + if (m == 0xE0 && L >= 5) { // JFIF APP0 segment + static const unsigned char tag[5] = {'J','F','I','F','\0'}; + int ok = 1; + int i; + for (i=0; i < 5; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 5; + if (ok) + z->jfif = 1; + } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment + static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; + int ok = 1; + int i; + for (i=0; i < 6; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 6; + if (ok) { + stbi__get8(z->s); // version + stbi__get16be(z->s); // flags0 + stbi__get16be(z->s); // flags1 + z->app14_color_transform = stbi__get8(z->s); // color transform + L -= 6; + } + } + + stbi__skip(z->s, L); + return 1; + } + + return stbi__err("unknown marker","Corrupt JPEG"); +} + +// after we see SOS +static int stbi__process_scan_header(stbi__jpeg *z) +{ + int i; + int Ls = stbi__get16be(z->s); + z->scan_n = stbi__get8(z->s); + if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); + if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); + for (i=0; i < z->scan_n; ++i) { + int id = stbi__get8(z->s), which; + int q = stbi__get8(z->s); + for (which = 0; which < z->s->img_n; ++which) + if (z->img_comp[which].id == id) + break; + if (which == z->s->img_n) return 0; // no match + z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); + z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); + z->order[i] = which; + } + + { + int aa; + z->spec_start = stbi__get8(z->s); + z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 + aa = stbi__get8(z->s); + z->succ_high = (aa >> 4); + z->succ_low = (aa & 15); + if (z->progressive) { + if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) + return stbi__err("bad SOS", "Corrupt JPEG"); + } else { + if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); + if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); + z->spec_end = 63; + } + } + + return 1; +} + +static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) +{ + int i; + for (i=0; i < ncomp; ++i) { + if (z->img_comp[i].raw_data) { + STBI_FREE(z->img_comp[i].raw_data); + z->img_comp[i].raw_data = NULL; + z->img_comp[i].data = NULL; + } + if (z->img_comp[i].raw_coeff) { + STBI_FREE(z->img_comp[i].raw_coeff); + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].coeff = 0; + } + if (z->img_comp[i].linebuf) { + STBI_FREE(z->img_comp[i].linebuf); + z->img_comp[i].linebuf = NULL; + } + } + return why; +} + +static int stbi__process_frame_header(stbi__jpeg *z, int scan) +{ + stbi__context *s = z->s; + int Lf,p,i,q, h_max=1,v_max=1,c; + Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG + p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline + s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG + s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + c = stbi__get8(s); + if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); + s->img_n = c; + for (i=0; i < c; ++i) { + z->img_comp[i].data = NULL; + z->img_comp[i].linebuf = NULL; + } + + if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); + + z->rgb = 0; + for (i=0; i < s->img_n; ++i) { + static const unsigned char rgb[3] = { 'R', 'G', 'B' }; + z->img_comp[i].id = stbi__get8(s); + if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) + ++z->rgb; + q = stbi__get8(s); + z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); + z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); + z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); + } + + if (scan != STBI__SCAN_load) return 1; + + if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); + + for (i=0; i < s->img_n; ++i) { + if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; + if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; + } + + // check that plane subsampling factors are integer ratios; our resamplers can't deal with fractional ratios + // and I've never seen a non-corrupted JPEG file actually use them + for (i=0; i < s->img_n; ++i) { + if (h_max % z->img_comp[i].h != 0) return stbi__err("bad H","Corrupt JPEG"); + if (v_max % z->img_comp[i].v != 0) return stbi__err("bad V","Corrupt JPEG"); + } + + // compute interleaved mcu info + z->img_h_max = h_max; + z->img_v_max = v_max; + z->img_mcu_w = h_max * 8; + z->img_mcu_h = v_max * 8; + // these sizes can't be more than 17 bits + z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; + z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; + + for (i=0; i < s->img_n; ++i) { + // number of effective pixels (e.g. for non-interleaved MCU) + z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; + z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; + // to simplify generation, we'll allocate enough memory to decode + // the bogus oversized data from using interleaved MCUs and their + // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't + // discard the extra data until colorspace conversion + // + // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) + // so these muls can't overflow with 32-bit ints (which we require) + z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; + z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; + z->img_comp[i].coeff = 0; + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].linebuf = NULL; + z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); + if (z->img_comp[i].raw_data == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + // align blocks for idct using mmx/sse + z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); + if (z->progressive) { + // w2, h2 are multiples of 8 (see above) + z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; + z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; + z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); + if (z->img_comp[i].raw_coeff == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); + } + } + + return 1; +} + +// use comparisons since in some cases we handle more than one case (e.g. SOF) +#define stbi__DNL(x) ((x) == 0xdc) +#define stbi__SOI(x) ((x) == 0xd8) +#define stbi__EOI(x) ((x) == 0xd9) +#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) +#define stbi__SOS(x) ((x) == 0xda) + +#define stbi__SOF_progressive(x) ((x) == 0xc2) + +static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) +{ + int m; + z->jfif = 0; + z->app14_color_transform = -1; // valid values are 0,1,2 + z->marker = STBI__MARKER_none; // initialize cached marker to empty + m = stbi__get_marker(z); + if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); + if (scan == STBI__SCAN_type) return 1; + m = stbi__get_marker(z); + while (!stbi__SOF(m)) { + if (!stbi__process_marker(z,m)) return 0; + m = stbi__get_marker(z); + while (m == STBI__MARKER_none) { + // some files have extra padding after their blocks, so ok, we'll scan + if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); + m = stbi__get_marker(z); + } + } + z->progressive = stbi__SOF_progressive(m); + if (!stbi__process_frame_header(z, scan)) return 0; + return 1; +} + +static int stbi__skip_jpeg_junk_at_end(stbi__jpeg *j) +{ + // some JPEGs have junk at end, skip over it but if we find what looks + // like a valid marker, resume there + while (!stbi__at_eof(j->s)) { + int x = stbi__get8(j->s); + while (x == 255) { // might be a marker + if (stbi__at_eof(j->s)) return STBI__MARKER_none; + x = stbi__get8(j->s); + if (x != 0x00 && x != 0xff) { + // not a stuffed zero or lead-in to another marker, looks + // like an actual marker, return it + return x; + } + // stuffed zero has x=0 now which ends the loop, meaning we go + // back to regular scan loop. + // repeated 0xff keeps trying to read the next byte of the marker. + } + } + return STBI__MARKER_none; +} + +// decode image to YCbCr format +static int stbi__decode_jpeg_image(stbi__jpeg *j) +{ + int m; + for (m = 0; m < 4; m++) { + j->img_comp[m].raw_data = NULL; + j->img_comp[m].raw_coeff = NULL; + } + j->restart_interval = 0; + if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; + m = stbi__get_marker(j); + while (!stbi__EOI(m)) { + if (stbi__SOS(m)) { + if (!stbi__process_scan_header(j)) return 0; + if (!stbi__parse_entropy_coded_data(j)) return 0; + if (j->marker == STBI__MARKER_none ) { + j->marker = stbi__skip_jpeg_junk_at_end(j); + // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 + } + m = stbi__get_marker(j); + if (STBI__RESTART(m)) + m = stbi__get_marker(j); + } else if (stbi__DNL(m)) { + int Ld = stbi__get16be(j->s); + stbi__uint32 NL = stbi__get16be(j->s); + if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); + if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); + m = stbi__get_marker(j); + } else { + if (!stbi__process_marker(j, m)) return 1; + m = stbi__get_marker(j); + } + } + if (j->progressive) + stbi__jpeg_finish(j); + return 1; +} + +// static jfif-centered resampling (across block boundaries) + +typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, + int w, int hs); + +#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) + +static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + STBI_NOTUSED(out); + STBI_NOTUSED(in_far); + STBI_NOTUSED(w); + STBI_NOTUSED(hs); + return in_near; +} + +static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples vertically for every one in input + int i; + STBI_NOTUSED(hs); + for (i=0; i < w; ++i) + out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); + return out; +} + +static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples horizontally for every one in input + int i; + stbi_uc *input = in_near; + + if (w == 1) { + // if only one sample, can't do any interpolation + out[0] = out[1] = input[0]; + return out; + } + + out[0] = input[0]; + out[1] = stbi__div4(input[0]*3 + input[1] + 2); + for (i=1; i < w-1; ++i) { + int n = 3*input[i]+2; + out[i*2+0] = stbi__div4(n+input[i-1]); + out[i*2+1] = stbi__div4(n+input[i+1]); + } + out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); + out[i*2+1] = input[w-1]; + + STBI_NOTUSED(in_far); + STBI_NOTUSED(hs); + + return out; +} + +#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) + +static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i,t0,t1; + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + out[0] = stbi__div4(t1+2); + for (i=1; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i=0,t0,t1; + + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + // process groups of 8 pixels for as long as we can. + // note we can't handle the last pixel in a row in this loop + // because we need to handle the filter boundary conditions. + for (; i < ((w-1) & ~7); i += 8) { +#if defined(STBI_SSE2) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + __m128i zero = _mm_setzero_si128(); + __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); + __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); + __m128i farw = _mm_unpacklo_epi8(farb, zero); + __m128i nearw = _mm_unpacklo_epi8(nearb, zero); + __m128i diff = _mm_sub_epi16(farw, nearw); + __m128i nears = _mm_slli_epi16(nearw, 2); + __m128i curr = _mm_add_epi16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + __m128i prv0 = _mm_slli_si128(curr, 2); + __m128i nxt0 = _mm_srli_si128(curr, 2); + __m128i prev = _mm_insert_epi16(prv0, t1, 0); + __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + __m128i bias = _mm_set1_epi16(8); + __m128i curs = _mm_slli_epi16(curr, 2); + __m128i prvd = _mm_sub_epi16(prev, curr); + __m128i nxtd = _mm_sub_epi16(next, curr); + __m128i curb = _mm_add_epi16(curs, bias); + __m128i even = _mm_add_epi16(prvd, curb); + __m128i odd = _mm_add_epi16(nxtd, curb); + + // interleave even and odd pixels, then undo scaling. + __m128i int0 = _mm_unpacklo_epi16(even, odd); + __m128i int1 = _mm_unpackhi_epi16(even, odd); + __m128i de0 = _mm_srli_epi16(int0, 4); + __m128i de1 = _mm_srli_epi16(int1, 4); + + // pack and write output + __m128i outv = _mm_packus_epi16(de0, de1); + _mm_storeu_si128((__m128i *) (out + i*2), outv); +#elif defined(STBI_NEON) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + uint8x8_t farb = vld1_u8(in_far + i); + uint8x8_t nearb = vld1_u8(in_near + i); + int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); + int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); + int16x8_t curr = vaddq_s16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + int16x8_t prv0 = vextq_s16(curr, curr, 7); + int16x8_t nxt0 = vextq_s16(curr, curr, 1); + int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); + int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + int16x8_t curs = vshlq_n_s16(curr, 2); + int16x8_t prvd = vsubq_s16(prev, curr); + int16x8_t nxtd = vsubq_s16(next, curr); + int16x8_t even = vaddq_s16(curs, prvd); + int16x8_t odd = vaddq_s16(curs, nxtd); + + // undo scaling and round, then store with even/odd phases interleaved + uint8x8x2_t o; + o.val[0] = vqrshrun_n_s16(even, 4); + o.val[1] = vqrshrun_n_s16(odd, 4); + vst2_u8(out + i*2, o); +#endif + + // "previous" value for next iter + t1 = 3*in_near[i+7] + in_far[i+7]; + } + + t0 = t1; + t1 = 3*in_near[i] + in_far[i]; + out[i*2] = stbi__div16(3*t1 + t0 + 8); + + for (++i; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} +#endif + +static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // resample with nearest-neighbor + int i,j; + STBI_NOTUSED(in_far); + for (i=0; i < w; ++i) + for (j=0; j < hs; ++j) + out[i*hs+j] = in_near[i]; + return out; +} + +// this is a reduced-precision calculation of YCbCr-to-RGB introduced +// to make sure the code produces the same results in both SIMD and scalar +#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) +static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) +{ + int i; + for (i=0; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) +{ + int i = 0; + +#ifdef STBI_SSE2 + // step == 3 is pretty ugly on the final interleave, and i'm not convinced + // it's useful in practice (you wouldn't use it for textures, for example). + // so just accelerate step == 4 case. + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + __m128i signflip = _mm_set1_epi8(-0x80); + __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); + __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); + __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); + __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); + __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); + __m128i xw = _mm_set1_epi16(255); // alpha channel + + for (; i+7 < count; i += 8) { + // load + __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); + __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); + __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); + __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 + __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 + + // unpack to short (and left-shift cr, cb by 8) + __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); + __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); + __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); + + // color transform + __m128i yws = _mm_srli_epi16(yw, 4); + __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); + __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); + __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); + __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); + __m128i rws = _mm_add_epi16(cr0, yws); + __m128i gwt = _mm_add_epi16(cb0, yws); + __m128i bws = _mm_add_epi16(yws, cb1); + __m128i gws = _mm_add_epi16(gwt, cr1); + + // descale + __m128i rw = _mm_srai_epi16(rws, 4); + __m128i bw = _mm_srai_epi16(bws, 4); + __m128i gw = _mm_srai_epi16(gws, 4); + + // back to byte, set up for transpose + __m128i brb = _mm_packus_epi16(rw, bw); + __m128i gxb = _mm_packus_epi16(gw, xw); + + // transpose to interleave channels + __m128i t0 = _mm_unpacklo_epi8(brb, gxb); + __m128i t1 = _mm_unpackhi_epi8(brb, gxb); + __m128i o0 = _mm_unpacklo_epi16(t0, t1); + __m128i o1 = _mm_unpackhi_epi16(t0, t1); + + // store + _mm_storeu_si128((__m128i *) (out + 0), o0); + _mm_storeu_si128((__m128i *) (out + 16), o1); + out += 32; + } + } +#endif + +#ifdef STBI_NEON + // in this version, step=3 support would be easy to add. but is there demand? + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + uint8x8_t signflip = vdup_n_u8(0x80); + int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); + int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); + int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); + int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); + + for (; i+7 < count; i += 8) { + // load + uint8x8_t y_bytes = vld1_u8(y + i); + uint8x8_t cr_bytes = vld1_u8(pcr + i); + uint8x8_t cb_bytes = vld1_u8(pcb + i); + int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); + int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); + + // expand to s16 + int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); + int16x8_t crw = vshll_n_s8(cr_biased, 7); + int16x8_t cbw = vshll_n_s8(cb_biased, 7); + + // color transform + int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); + int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); + int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); + int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); + int16x8_t rws = vaddq_s16(yws, cr0); + int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); + int16x8_t bws = vaddq_s16(yws, cb1); + + // undo scaling, round, convert to byte + uint8x8x4_t o; + o.val[0] = vqrshrun_n_s16(rws, 4); + o.val[1] = vqrshrun_n_s16(gws, 4); + o.val[2] = vqrshrun_n_s16(bws, 4); + o.val[3] = vdup_n_u8(255); + + // store, interleaving r/g/b/a + vst4_u8(out, o); + out += 8*4; + } + } +#endif + + for (; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} +#endif + +// set up the kernels +static void stbi__setup_jpeg(stbi__jpeg *j) +{ + j->idct_block_kernel = stbi__idct_block; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; + +#ifdef STBI_SSE2 + if (stbi__sse2_available()) { + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; + } +#endif + +#ifdef STBI_NEON + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; +#endif +} + +// clean up the temporary component buffers +static void stbi__cleanup_jpeg(stbi__jpeg *j) +{ + stbi__free_jpeg_components(j, j->s->img_n, 0); +} + +typedef struct +{ + resample_row_func resample; + stbi_uc *line0,*line1; + int hs,vs; // expansion factor in each axis + int w_lores; // horizontal pixels pre-expansion + int ystep; // how far through vertical expansion we are + int ypos; // which pre-expansion row we're on +} stbi__resample; + +// fast 0..255 * 0..255 => 0..255 rounded multiplication +static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) +{ + unsigned int t = x*y + 128; + return (stbi_uc) ((t + (t >>8)) >> 8); +} + +static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) +{ + int n, decode_n, is_rgb; + z->s->img_n = 0; // make stbi__cleanup_jpeg safe + + // validate req_comp + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + + // load a jpeg image from whichever source, but leave in YCbCr format + if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } + + // determine actual number of components to generate + n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; + + is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); + + if (z->s->img_n == 3 && n < 3 && !is_rgb) + decode_n = 1; + else + decode_n = z->s->img_n; + + // nothing to do if no components requested; check this now to avoid + // accessing uninitialized coutput[0] later + if (decode_n <= 0) { stbi__cleanup_jpeg(z); return NULL; } + + // resample and color-convert + { + int k; + unsigned int i,j; + stbi_uc *output; + stbi_uc *coutput[4] = { NULL, NULL, NULL, NULL }; + + stbi__resample res_comp[4]; + + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + + // allocate line buffer big enough for upsampling off the edges + // with upsample factor of 4 + z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); + if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + r->hs = z->img_h_max / z->img_comp[k].h; + r->vs = z->img_v_max / z->img_comp[k].v; + r->ystep = r->vs >> 1; + r->w_lores = (z->s->img_x + r->hs-1) / r->hs; + r->ypos = 0; + r->line0 = r->line1 = z->img_comp[k].data; + + if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; + else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; + else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; + else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; + else r->resample = stbi__resample_row_generic; + } + + // can't error after this so, this is safe + output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); + if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + // now go ahead and resample + for (j=0; j < z->s->img_y; ++j) { + stbi_uc *out = output + n * z->s->img_x * j; + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + int y_bot = r->ystep >= (r->vs >> 1); + coutput[k] = r->resample(z->img_comp[k].linebuf, + y_bot ? r->line1 : r->line0, + y_bot ? r->line0 : r->line1, + r->w_lores, r->hs); + if (++r->ystep >= r->vs) { + r->ystep = 0; + r->line0 = r->line1; + if (++r->ypos < z->img_comp[k].y) + r->line1 += z->img_comp[k].w2; + } + } + if (n >= 3) { + stbi_uc *y = coutput[0]; + if (z->s->img_n == 3) { + if (is_rgb) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = y[i]; + out[1] = coutput[1][i]; + out[2] = coutput[2][i]; + out[3] = 255; + out += n; + } + } else { + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else if (z->s->img_n == 4) { + if (z->app14_color_transform == 0) { // CMYK + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(coutput[0][i], m); + out[1] = stbi__blinn_8x8(coutput[1][i], m); + out[2] = stbi__blinn_8x8(coutput[2][i], m); + out[3] = 255; + out += n; + } + } else if (z->app14_color_transform == 2) { // YCCK + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(255 - out[0], m); + out[1] = stbi__blinn_8x8(255 - out[1], m); + out[2] = stbi__blinn_8x8(255 - out[2], m); + out += n; + } + } else { // YCbCr + alpha? Ignore the fourth channel for now + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else + for (i=0; i < z->s->img_x; ++i) { + out[0] = out[1] = out[2] = y[i]; + out[3] = 255; // not used if n==3 + out += n; + } + } else { + if (is_rgb) { + if (n == 1) + for (i=0; i < z->s->img_x; ++i) + *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + else { + for (i=0; i < z->s->img_x; ++i, out += 2) { + out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + out[1] = 255; + } + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); + stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); + stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); + out[0] = stbi__compute_y(r, g, b); + out[1] = 255; + out += n; + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); + out[1] = 255; + out += n; + } + } else { + stbi_uc *y = coutput[0]; + if (n == 1) + for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; + else + for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } + } + } + } + stbi__cleanup_jpeg(z); + *out_x = z->s->img_x; + *out_y = z->s->img_y; + if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output + return output; + } +} + +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + unsigned char* result; + stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__errpuc("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + STBI_NOTUSED(ri); + j->s = s; + stbi__setup_jpeg(j); + result = load_jpeg_image(j, x,y,comp,req_comp); + STBI_FREE(j); + return result; +} + +static int stbi__jpeg_test(stbi__context *s) +{ + int r; + stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + stbi__setup_jpeg(j); + r = stbi__decode_jpeg_header(j, STBI__SCAN_type); + stbi__rewind(s); + STBI_FREE(j); + return r; +} + +static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) +{ + if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { + stbi__rewind( j->s ); + return 0; + } + if (x) *x = j->s->img_x; + if (y) *y = j->s->img_y; + if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; + return 1; +} + +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) +{ + int result; + stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); + if (!j) return stbi__err("outofmem", "Out of memory"); + memset(j, 0, sizeof(stbi__jpeg)); + j->s = s; + result = stbi__jpeg_info_raw(j, x, y, comp); + STBI_FREE(j); + return result; +} +#endif + +// public domain zlib decode v0.2 Sean Barrett 2006-11-18 +// simple implementation +// - all input must be provided in an upfront buffer +// - all output is written to a single output buffer (can malloc/realloc) +// performance +// - fast huffman + +#ifndef STBI_NO_ZLIB + +// fast-way is faster to check than jpeg huffman, but slow way is slower +#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables +#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) +#define STBI__ZNSYMS 288 // number of symbols in literal/length alphabet + +// zlib-style huffman encoding +// (jpegs packs from left, zlib from right, so can't share code) +typedef struct +{ + stbi__uint16 fast[1 << STBI__ZFAST_BITS]; + stbi__uint16 firstcode[16]; + int maxcode[17]; + stbi__uint16 firstsymbol[16]; + stbi_uc size[STBI__ZNSYMS]; + stbi__uint16 value[STBI__ZNSYMS]; +} stbi__zhuffman; + +stbi_inline static int stbi__bitreverse16(int n) +{ + n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); + n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); + n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); + n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); + return n; +} + +stbi_inline static int stbi__bit_reverse(int v, int bits) +{ + STBI_ASSERT(bits <= 16); + // to bit reverse n bits, reverse 16 and shift + // e.g. 11 bits, bit reverse and shift away 5 + return stbi__bitreverse16(v) >> (16-bits); +} + +static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) +{ + int i,k=0; + int code, next_code[16], sizes[17]; + + // DEFLATE spec for generating codes + memset(sizes, 0, sizeof(sizes)); + memset(z->fast, 0, sizeof(z->fast)); + for (i=0; i < num; ++i) + ++sizes[sizelist[i]]; + sizes[0] = 0; + for (i=1; i < 16; ++i) + if (sizes[i] > (1 << i)) + return stbi__err("bad sizes", "Corrupt PNG"); + code = 0; + for (i=1; i < 16; ++i) { + next_code[i] = code; + z->firstcode[i] = (stbi__uint16) code; + z->firstsymbol[i] = (stbi__uint16) k; + code = (code + sizes[i]); + if (sizes[i]) + if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); + z->maxcode[i] = code << (16-i); // preshift for inner loop + code <<= 1; + k += sizes[i]; + } + z->maxcode[16] = 0x10000; // sentinel + for (i=0; i < num; ++i) { + int s = sizelist[i]; + if (s) { + int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; + stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); + z->size [c] = (stbi_uc ) s; + z->value[c] = (stbi__uint16) i; + if (s <= STBI__ZFAST_BITS) { + int j = stbi__bit_reverse(next_code[s],s); + while (j < (1 << STBI__ZFAST_BITS)) { + z->fast[j] = fastv; + j += (1 << s); + } + } + ++next_code[s]; + } + } + return 1; +} + +// zlib-from-memory implementation for PNG reading +// because PNG allows splitting the zlib stream arbitrarily, +// and it's annoying structurally to have PNG call ZLIB call PNG, +// we require PNG read all the IDATs and combine them into a single +// memory buffer + +typedef struct +{ + stbi_uc *zbuffer, *zbuffer_end; + int num_bits; + stbi__uint32 code_buffer; + + char *zout; + char *zout_start; + char *zout_end; + int z_expandable; + + stbi__zhuffman z_length, z_distance; +} stbi__zbuf; + +stbi_inline static int stbi__zeof(stbi__zbuf *z) +{ + return (z->zbuffer >= z->zbuffer_end); +} + +stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) +{ + return stbi__zeof(z) ? 0 : *z->zbuffer++; +} + +static void stbi__fill_bits(stbi__zbuf *z) +{ + do { + if (z->code_buffer >= (1U << z->num_bits)) { + z->zbuffer = z->zbuffer_end; /* treat this as EOF so we fail. */ + return; + } + z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; + z->num_bits += 8; + } while (z->num_bits <= 24); +} + +stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) +{ + unsigned int k; + if (z->num_bits < n) stbi__fill_bits(z); + k = z->code_buffer & ((1 << n) - 1); + z->code_buffer >>= n; + z->num_bits -= n; + return k; +} + +static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s,k; + // not resolved by fast table, so compute it the slow way + // use jpeg approach, which requires MSbits at top + k = stbi__bit_reverse(a->code_buffer, 16); + for (s=STBI__ZFAST_BITS+1; ; ++s) + if (k < z->maxcode[s]) + break; + if (s >= 16) return -1; // invalid code! + // code size is s, so: + b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; + if (b >= STBI__ZNSYMS) return -1; // some data was corrupt somewhere! + if (z->size[b] != s) return -1; // was originally an assert, but report failure instead. + a->code_buffer >>= s; + a->num_bits -= s; + return z->value[b]; +} + +stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s; + if (a->num_bits < 16) { + if (stbi__zeof(a)) { + return -1; /* report error for unexpected end of data. */ + } + stbi__fill_bits(a); + } + b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; + if (b) { + s = b >> 9; + a->code_buffer >>= s; + a->num_bits -= s; + return b & 511; + } + return stbi__zhuffman_decode_slowpath(a, z); +} + +static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes +{ + char *q; + unsigned int cur, limit, old_limit; + z->zout = zout; + if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); + cur = (unsigned int) (z->zout - z->zout_start); + limit = old_limit = (unsigned) (z->zout_end - z->zout_start); + if (UINT_MAX - cur < (unsigned) n) return stbi__err("outofmem", "Out of memory"); + while (cur + n > limit) { + if(limit > UINT_MAX / 2) return stbi__err("outofmem", "Out of memory"); + limit *= 2; + } + q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); + STBI_NOTUSED(old_limit); + if (q == NULL) return stbi__err("outofmem", "Out of memory"); + z->zout_start = q; + z->zout = q + cur; + z->zout_end = q + limit; + return 1; +} + +static const int stbi__zlength_base[31] = { + 3,4,5,6,7,8,9,10,11,13, + 15,17,19,23,27,31,35,43,51,59, + 67,83,99,115,131,163,195,227,258,0,0 }; + +static const int stbi__zlength_extra[31]= +{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; + +static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, +257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; + +static const int stbi__zdist_extra[32] = +{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +static int stbi__parse_huffman_block(stbi__zbuf *a) +{ + char *zout = a->zout; + for(;;) { + int z = stbi__zhuffman_decode(a, &a->z_length); + if (z < 256) { + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes + if (zout >= a->zout_end) { + if (!stbi__zexpand(a, zout, 1)) return 0; + zout = a->zout; + } + *zout++ = (char) z; + } else { + stbi_uc *p; + int len,dist; + if (z == 256) { + a->zout = zout; + return 1; + } + if (z >= 286) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, length codes 286 and 287 must not appear in compressed data + z -= 257; + len = stbi__zlength_base[z]; + if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); + z = stbi__zhuffman_decode(a, &a->z_distance); + if (z < 0 || z >= 30) return stbi__err("bad huffman code","Corrupt PNG"); // per DEFLATE, distance codes 30 and 31 must not appear in compressed data + dist = stbi__zdist_base[z]; + if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); + if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); + if (zout + len > a->zout_end) { + if (!stbi__zexpand(a, zout, len)) return 0; + zout = a->zout; + } + p = (stbi_uc *) (zout - dist); + if (dist == 1) { // run of one byte; common in images. + stbi_uc v = *p; + if (len) { do *zout++ = v; while (--len); } + } else { + if (len) { do *zout++ = *p++; while (--len); } + } + } + } +} + +static int stbi__compute_huffman_codes(stbi__zbuf *a) +{ + static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; + stbi__zhuffman z_codelength; + stbi_uc lencodes[286+32+137];//padding for maximum single op + stbi_uc codelength_sizes[19]; + int i,n; + + int hlit = stbi__zreceive(a,5) + 257; + int hdist = stbi__zreceive(a,5) + 1; + int hclen = stbi__zreceive(a,4) + 4; + int ntot = hlit + hdist; + + memset(codelength_sizes, 0, sizeof(codelength_sizes)); + for (i=0; i < hclen; ++i) { + int s = stbi__zreceive(a,3); + codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; + } + if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; + + n = 0; + while (n < ntot) { + int c = stbi__zhuffman_decode(a, &z_codelength); + if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); + if (c < 16) + lencodes[n++] = (stbi_uc) c; + else { + stbi_uc fill = 0; + if (c == 16) { + c = stbi__zreceive(a,2)+3; + if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); + fill = lencodes[n-1]; + } else if (c == 17) { + c = stbi__zreceive(a,3)+3; + } else if (c == 18) { + c = stbi__zreceive(a,7)+11; + } else { + return stbi__err("bad codelengths", "Corrupt PNG"); + } + if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); + memset(lencodes+n, fill, c); + n += c; + } + } + if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); + if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; + return 1; +} + +static int stbi__parse_uncompressed_block(stbi__zbuf *a) +{ + stbi_uc header[4]; + int len,nlen,k; + if (a->num_bits & 7) + stbi__zreceive(a, a->num_bits & 7); // discard + // drain the bit-packed data into header + k = 0; + while (a->num_bits > 0) { + header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check + a->code_buffer >>= 8; + a->num_bits -= 8; + } + if (a->num_bits < 0) return stbi__err("zlib corrupt","Corrupt PNG"); + // now fill header the normal way + while (k < 4) + header[k++] = stbi__zget8(a); + len = header[1] * 256 + header[0]; + nlen = header[3] * 256 + header[2]; + if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); + if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); + if (a->zout + len > a->zout_end) + if (!stbi__zexpand(a, a->zout, len)) return 0; + memcpy(a->zout, a->zbuffer, len); + a->zbuffer += len; + a->zout += len; + return 1; +} + +static int stbi__parse_zlib_header(stbi__zbuf *a) +{ + int cmf = stbi__zget8(a); + int cm = cmf & 15; + /* int cinfo = cmf >> 4; */ + int flg = stbi__zget8(a); + if (stbi__zeof(a)) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png + if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png + // window = 1 << (8 + cinfo)... but who cares, we fully buffer output + return 1; +} + +static const stbi_uc stbi__zdefault_length[STBI__ZNSYMS] = +{ + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 +}; +static const stbi_uc stbi__zdefault_distance[32] = +{ + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 +}; +/* +Init algorithm: +{ + int i; // use <= to match clearly with spec + for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; + for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; + for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; + for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; + + for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; +} +*/ + +static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) +{ + int final, type; + if (parse_header) + if (!stbi__parse_zlib_header(a)) return 0; + a->num_bits = 0; + a->code_buffer = 0; + do { + final = stbi__zreceive(a,1); + type = stbi__zreceive(a,2); + if (type == 0) { + if (!stbi__parse_uncompressed_block(a)) return 0; + } else if (type == 3) { + return 0; + } else { + if (type == 1) { + // use fixed code lengths + if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , STBI__ZNSYMS)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; + } else { + if (!stbi__compute_huffman_codes(a)) return 0; + } + if (!stbi__parse_huffman_block(a)) return 0; + } + } while (!final); + return 1; +} + +static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) +{ + a->zout_start = obuf; + a->zout = obuf; + a->zout_end = obuf + olen; + a->z_expandable = exp; + + return stbi__parse_zlib(a, parse_header); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) +{ + return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) + return (int) (a.zout - a.zout_start); + else + return -1; +} + +STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(16384); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer+len; + if (stbi__do_zlib(&a, p, 16384, 1, 0)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) + return (int) (a.zout - a.zout_start); + else + return -1; +} +#endif + +// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 +// simple implementation +// - only 8-bit samples +// - no CRC checking +// - allocates lots of intermediate memory +// - avoids problem of streaming data between subsystems +// - avoids explicit window management +// performance +// - uses stb_zlib, a PD zlib implementation with fast huffman decoding + +#ifndef STBI_NO_PNG +typedef struct +{ + stbi__uint32 length; + stbi__uint32 type; +} stbi__pngchunk; + +static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) +{ + stbi__pngchunk c; + c.length = stbi__get32be(s); + c.type = stbi__get32be(s); + return c; +} + +static int stbi__check_png_header(stbi__context *s) +{ + static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; + int i; + for (i=0; i < 8; ++i) + if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); + return 1; +} + +typedef struct +{ + stbi__context *s; + stbi_uc *idata, *expanded, *out; + int depth; +} stbi__png; + + +enum { + STBI__F_none=0, + STBI__F_sub=1, + STBI__F_up=2, + STBI__F_avg=3, + STBI__F_paeth=4, + // synthetic filters used for first scanline to avoid needing a dummy row of 0s + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static stbi_uc first_row_filter[5] = +{ + STBI__F_none, + STBI__F_sub, + STBI__F_none, + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static int stbi__paeth(int a, int b, int c) +{ + int p = a + b - c; + int pa = abs(p-a); + int pb = abs(p-b); + int pc = abs(p-c); + if (pa <= pb && pa <= pc) return a; + if (pb <= pc) return b; + return c; +} + +static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; + +// create the png data from post-deflated data +static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) +{ + int bytes = (depth == 16? 2 : 1); + stbi__context *s = a->s; + stbi__uint32 i,j,stride = x*out_n*bytes; + stbi__uint32 img_len, img_width_bytes; + int k; + int img_n = s->img_n; // copy it into a local for later + + int output_bytes = out_n*bytes; + int filter_bytes = img_n*bytes; + int width = x; + + STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); + a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into + if (!a->out) return stbi__err("outofmem", "Out of memory"); + + if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); + img_width_bytes = (((img_n * x * depth) + 7) >> 3); + img_len = (img_width_bytes + 1) * y; + + // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, + // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), + // so just check for raw_len < img_len always. + if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); + + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *prior; + int filter = *raw++; + + if (filter > 4) + return stbi__err("invalid filter","Corrupt PNG"); + + if (depth < 8) { + if (img_width_bytes > x) return stbi__err("invalid width","Corrupt PNG"); + cur += x*out_n - img_width_bytes; // store output to the rightmost img_len bytes, so we can decode in place + filter_bytes = 1; + width = img_width_bytes; + } + prior = cur - stride; // bugfix: need to compute this after 'cur +=' computation above + + // if first row, use special filter that doesn't sample previous row + if (j == 0) filter = first_row_filter[filter]; + + // handle first byte explicitly + for (k=0; k < filter_bytes; ++k) { + switch (filter) { + case STBI__F_none : cur[k] = raw[k]; break; + case STBI__F_sub : cur[k] = raw[k]; break; + case STBI__F_up : cur[k] = STBI__BYTECAST(raw[k] + prior[k]); break; + case STBI__F_avg : cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); break; + case STBI__F_paeth : cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(0,prior[k],0)); break; + case STBI__F_avg_first : cur[k] = raw[k]; break; + case STBI__F_paeth_first: cur[k] = raw[k]; break; + } + } + + if (depth == 8) { + if (img_n != out_n) + cur[img_n] = 255; // first pixel + raw += img_n; + cur += out_n; + prior += out_n; + } else if (depth == 16) { + if (img_n != out_n) { + cur[filter_bytes] = 255; // first pixel top byte + cur[filter_bytes+1] = 255; // first pixel bottom byte + } + raw += filter_bytes; + cur += output_bytes; + prior += output_bytes; + } else { + raw += 1; + cur += 1; + prior += 1; + } + + // this is a little gross, so that we don't switch per-pixel or per-component + if (depth < 8 || img_n == out_n) { + int nk = (width - 1)*filter_bytes; + #define STBI__CASE(f) \ + case f: \ + for (k=0; k < nk; ++k) + switch (filter) { + // "none" filter turns into a memcpy here; make that explicit. + case STBI__F_none: memcpy(cur, raw, nk); break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],prior[k],prior[k-filter_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],0,0)); } break; + } + #undef STBI__CASE + raw += nk; + } else { + STBI_ASSERT(img_n+1 == out_n); + #define STBI__CASE(f) \ + case f: \ + for (i=x-1; i >= 1; --i, cur[filter_bytes]=255,raw+=filter_bytes,cur+=output_bytes,prior+=output_bytes) \ + for (k=0; k < filter_bytes; ++k) + switch (filter) { + STBI__CASE(STBI__F_none) { cur[k] = raw[k]; } break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k- output_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k- output_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],prior[k],prior[k- output_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k- output_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],0,0)); } break; + } + #undef STBI__CASE + + // the loop above sets the high byte of the pixels' alpha, but for + // 16 bit png files we also need the low byte set. we'll do that here. + if (depth == 16) { + cur = a->out + stride*j; // start at the beginning of the row again + for (i=0; i < x; ++i,cur+=output_bytes) { + cur[filter_bytes+1] = 255; + } + } + } + } + + // we make a separate pass to expand bits to pixels; for performance, + // this could run two scanlines behind the above code, so it won't + // intefere with filtering but will still be in the cache. + if (depth < 8) { + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *in = a->out + stride*j + x*out_n - img_width_bytes; + // unpack 1/2/4-bit into a 8-bit buffer. allows us to keep the common 8-bit path optimal at minimal cost for 1/2/4-bit + // png guarante byte alignment, if width is not multiple of 8/4/2 we'll decode dummy trailing data that will be skipped in the later loop + stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range + + // note that the final byte might overshoot and write more data than desired. + // we can allocate enough data that this never writes out of memory, but it + // could also overwrite the next scanline. can it overwrite non-empty data + // on the next scanline? yes, consider 1-pixel-wide scanlines with 1-bit-per-pixel. + // so we need to explicitly clamp the final ones + + if (depth == 4) { + for (k=x*img_n; k >= 2; k-=2, ++in) { + *cur++ = scale * ((*in >> 4) ); + *cur++ = scale * ((*in ) & 0x0f); + } + if (k > 0) *cur++ = scale * ((*in >> 4) ); + } else if (depth == 2) { + for (k=x*img_n; k >= 4; k-=4, ++in) { + *cur++ = scale * ((*in >> 6) ); + *cur++ = scale * ((*in >> 4) & 0x03); + *cur++ = scale * ((*in >> 2) & 0x03); + *cur++ = scale * ((*in ) & 0x03); + } + if (k > 0) *cur++ = scale * ((*in >> 6) ); + if (k > 1) *cur++ = scale * ((*in >> 4) & 0x03); + if (k > 2) *cur++ = scale * ((*in >> 2) & 0x03); + } else if (depth == 1) { + for (k=x*img_n; k >= 8; k-=8, ++in) { + *cur++ = scale * ((*in >> 7) ); + *cur++ = scale * ((*in >> 6) & 0x01); + *cur++ = scale * ((*in >> 5) & 0x01); + *cur++ = scale * ((*in >> 4) & 0x01); + *cur++ = scale * ((*in >> 3) & 0x01); + *cur++ = scale * ((*in >> 2) & 0x01); + *cur++ = scale * ((*in >> 1) & 0x01); + *cur++ = scale * ((*in ) & 0x01); + } + if (k > 0) *cur++ = scale * ((*in >> 7) ); + if (k > 1) *cur++ = scale * ((*in >> 6) & 0x01); + if (k > 2) *cur++ = scale * ((*in >> 5) & 0x01); + if (k > 3) *cur++ = scale * ((*in >> 4) & 0x01); + if (k > 4) *cur++ = scale * ((*in >> 3) & 0x01); + if (k > 5) *cur++ = scale * ((*in >> 2) & 0x01); + if (k > 6) *cur++ = scale * ((*in >> 1) & 0x01); + } + if (img_n != out_n) { + int q; + // insert alpha = 255 + cur = a->out + stride*j; + if (img_n == 1) { + for (q=x-1; q >= 0; --q) { + cur[q*2+1] = 255; + cur[q*2+0] = cur[q]; + } + } else { + STBI_ASSERT(img_n == 3); + for (q=x-1; q >= 0; --q) { + cur[q*4+3] = 255; + cur[q*4+2] = cur[q*3+2]; + cur[q*4+1] = cur[q*3+1]; + cur[q*4+0] = cur[q*3+0]; + } + } + } + } + } else if (depth == 16) { + // force the image data from big-endian to platform-native. + // this is done in a separate pass due to the decoding relying + // on the data being untouched, but could probably be done + // per-line during decode if care is taken. + stbi_uc *cur = a->out; + stbi__uint16 *cur16 = (stbi__uint16*)cur; + + for(i=0; i < x*y*out_n; ++i,cur16++,cur+=2) { + *cur16 = (cur[0] << 8) | cur[1]; + } + } + + return 1; +} + +static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) +{ + int bytes = (depth == 16 ? 2 : 1); + int out_bytes = out_n * bytes; + stbi_uc *final; + int p; + if (!interlaced) + return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); + + // de-interlacing + final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); + if (!final) return stbi__err("outofmem", "Out of memory"); + for (p=0; p < 7; ++p) { + int xorig[] = { 0,4,0,2,0,1,0 }; + int yorig[] = { 0,0,4,0,2,0,1 }; + int xspc[] = { 8,8,4,4,2,2,1 }; + int yspc[] = { 8,8,8,4,4,2,2 }; + int i,j,x,y; + // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 + x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; + y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; + if (x && y) { + stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; + if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { + STBI_FREE(final); + return 0; + } + for (j=0; j < y; ++j) { + for (i=0; i < x; ++i) { + int out_y = j*yspc[p]+yorig[p]; + int out_x = i*xspc[p]+xorig[p]; + memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, + a->out + (j*x+i)*out_bytes, out_bytes); + } + } + STBI_FREE(a->out); + image_data += img_len; + image_data_len -= img_len; + } + } + a->out = final; + + return 1; +} + +static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + // compute color-based transparency, assuming we've + // already got 255 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i=0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 255); + p += 2; + } + } else { + for (i=0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi__uint16 *p = (stbi__uint16*) z->out; + + // compute color-based transparency, assuming we've + // already got 65535 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i = 0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 65535); + p += 2; + } + } else { + for (i = 0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) +{ + stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; + stbi_uc *p, *temp_out, *orig = a->out; + + p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); + if (p == NULL) return stbi__err("outofmem", "Out of memory"); + + // between here and free(out) below, exitting would leak + temp_out = p; + + if (pal_img_n == 3) { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p += 3; + } + } else { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p[3] = palette[n+3]; + p += 4; + } + } + STBI_FREE(a->out); + a->out = temp_out; + + STBI_NOTUSED(len); + + return 1; +} + +static int stbi__unpremultiply_on_load_global = 0; +static int stbi__de_iphone_flag_global = 0; + +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_global = flag_true_if_should_unpremultiply; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_global = flag_true_if_should_convert; +} + +#ifndef STBI_THREAD_LOCAL +#define stbi__unpremultiply_on_load stbi__unpremultiply_on_load_global +#define stbi__de_iphone_flag stbi__de_iphone_flag_global +#else +static STBI_THREAD_LOCAL int stbi__unpremultiply_on_load_local, stbi__unpremultiply_on_load_set; +static STBI_THREAD_LOCAL int stbi__de_iphone_flag_local, stbi__de_iphone_flag_set; + +STBIDEF void stbi_set_unpremultiply_on_load_thread(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load_local = flag_true_if_should_unpremultiply; + stbi__unpremultiply_on_load_set = 1; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb_thread(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag_local = flag_true_if_should_convert; + stbi__de_iphone_flag_set = 1; +} + +#define stbi__unpremultiply_on_load (stbi__unpremultiply_on_load_set \ + ? stbi__unpremultiply_on_load_local \ + : stbi__unpremultiply_on_load_global) +#define stbi__de_iphone_flag (stbi__de_iphone_flag_set \ + ? stbi__de_iphone_flag_local \ + : stbi__de_iphone_flag_global) +#endif // STBI_THREAD_LOCAL + +static void stbi__de_iphone(stbi__png *z) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + if (s->img_out_n == 3) { // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 3; + } + } else { + STBI_ASSERT(s->img_out_n == 4); + if (stbi__unpremultiply_on_load) { + // convert bgr to rgb and unpremultiply + for (i=0; i < pixel_count; ++i) { + stbi_uc a = p[3]; + stbi_uc t = p[0]; + if (a) { + stbi_uc half = a / 2; + p[0] = (p[2] * 255 + half) / a; + p[1] = (p[1] * 255 + half) / a; + p[2] = ( t * 255 + half) / a; + } else { + p[0] = p[2]; + p[2] = t; + } + p += 4; + } + } else { + // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 4; + } + } + } +} + +#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) + +static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) +{ + stbi_uc palette[1024], pal_img_n=0; + stbi_uc has_trans=0, tc[3]={0}; + stbi__uint16 tc16[3]; + stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; + int first=1,k,interlace=0, color=0, is_iphone=0; + stbi__context *s = z->s; + + z->expanded = NULL; + z->idata = NULL; + z->out = NULL; + + if (!stbi__check_png_header(s)) return 0; + + if (scan == STBI__SCAN_type) return 1; + + for (;;) { + stbi__pngchunk c = stbi__get_chunk_header(s); + switch (c.type) { + case STBI__PNG_TYPE('C','g','B','I'): + is_iphone = 1; + stbi__skip(s, c.length); + break; + case STBI__PNG_TYPE('I','H','D','R'): { + int comp,filter; + if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); + first = 0; + if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); + s->img_x = stbi__get32be(s); + s->img_y = stbi__get32be(s); + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); + color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); + comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); + filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); + interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); + if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); + if (!pal_img_n) { + s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); + if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); + } else { + // if paletted, then pal_n is our final components, and + // img_n is # components to decompress/filter. + s->img_n = 1; + if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); + } + // even with SCAN_header, have to scan to see if we have a tRNS + break; + } + + case STBI__PNG_TYPE('P','L','T','E'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); + pal_len = c.length / 3; + if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); + for (i=0; i < pal_len; ++i) { + palette[i*4+0] = stbi__get8(s); + palette[i*4+1] = stbi__get8(s); + palette[i*4+2] = stbi__get8(s); + palette[i*4+3] = 255; + } + break; + } + + case STBI__PNG_TYPE('t','R','N','S'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); + if (pal_img_n) { + if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } + if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); + if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); + pal_img_n = 4; + for (i=0; i < c.length; ++i) + palette[i*4+3] = stbi__get8(s); + } else { + if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); + if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); + has_trans = 1; + // non-paletted with tRNS = constant alpha. if header-scanning, we can stop now. + if (scan == STBI__SCAN_header) { ++s->img_n; return 1; } + if (z->depth == 16) { + for (k = 0; k < s->img_n; ++k) tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is + } else { + for (k = 0; k < s->img_n; ++k) tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger + } + } + break; + } + + case STBI__PNG_TYPE('I','D','A','T'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); + if (scan == STBI__SCAN_header) { + // header scan definitely stops at first IDAT + if (pal_img_n) + s->img_n = pal_img_n; + return 1; + } + if (c.length > (1u << 30)) return stbi__err("IDAT size limit", "IDAT section larger than 2^30 bytes"); + if ((int)(ioff + c.length) < (int)ioff) return 0; + if (ioff + c.length > idata_limit) { + stbi__uint32 idata_limit_old = idata_limit; + stbi_uc *p; + if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; + while (ioff + c.length > idata_limit) + idata_limit *= 2; + STBI_NOTUSED(idata_limit_old); + p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); + z->idata = p; + } + if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); + ioff += c.length; + break; + } + + case STBI__PNG_TYPE('I','E','N','D'): { + stbi__uint32 raw_len, bpl; + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (scan != STBI__SCAN_load) return 1; + if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); + // initial guess for decoded data size to avoid unnecessary reallocs + bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component + raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; + z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); + if (z->expanded == NULL) return 0; // zlib should set error + STBI_FREE(z->idata); z->idata = NULL; + if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) + s->img_out_n = s->img_n+1; + else + s->img_out_n = s->img_n; + if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; + if (has_trans) { + if (z->depth == 16) { + if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; + } else { + if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; + } + } + if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) + stbi__de_iphone(z); + if (pal_img_n) { + // pal_img_n == 3 or 4 + s->img_n = pal_img_n; // record the actual colors we had + s->img_out_n = pal_img_n; + if (req_comp >= 3) s->img_out_n = req_comp; + if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) + return 0; + } else if (has_trans) { + // non-paletted image with tRNS -> source image has (constant) alpha + ++s->img_n; + } + STBI_FREE(z->expanded); z->expanded = NULL; + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + return 1; + } + + default: + // if critical, fail + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if ((c.type & (1 << 29)) == 0) { + #ifndef STBI_NO_FAILURE_STRINGS + // not threadsafe + static char invalid_chunk[] = "XXXX PNG chunk not known"; + invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); + invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); + invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); + invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); + #endif + return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); + } + stbi__skip(s, c.length); + break; + } + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + } +} + +static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) +{ + void *result=NULL; + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { + if (p->depth <= 8) + ri->bits_per_channel = 8; + else if (p->depth == 16) + ri->bits_per_channel = 16; + else + return stbi__errpuc("bad bits_per_channel", "PNG not supported: unsupported color depth"); + result = p->out; + p->out = NULL; + if (req_comp && req_comp != p->s->img_out_n) { + if (ri->bits_per_channel == 8) + result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + else + result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + p->s->img_out_n = req_comp; + if (result == NULL) return result; + } + *x = p->s->img_x; + *y = p->s->img_y; + if (n) *n = p->s->img_n; + } + STBI_FREE(p->out); p->out = NULL; + STBI_FREE(p->expanded); p->expanded = NULL; + STBI_FREE(p->idata); p->idata = NULL; + + return result; +} + +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi__png p; + p.s = s; + return stbi__do_png(&p, x,y,comp,req_comp, ri); +} + +static int stbi__png_test(stbi__context *s) +{ + int r; + r = stbi__check_png_header(s); + stbi__rewind(s); + return r; +} + +static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) +{ + if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { + stbi__rewind( p->s ); + return 0; + } + if (x) *x = p->s->img_x; + if (y) *y = p->s->img_y; + if (comp) *comp = p->s->img_n; + return 1; +} + +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__png p; + p.s = s; + return stbi__png_info_raw(&p, x, y, comp); +} + +static int stbi__png_is16(stbi__context *s) +{ + stbi__png p; + p.s = s; + if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) + return 0; + if (p.depth != 16) { + stbi__rewind(p.s); + return 0; + } + return 1; +} +#endif + +// Microsoft/Windows BMP image + +#ifndef STBI_NO_BMP +static int stbi__bmp_test_raw(stbi__context *s) +{ + int r; + int sz; + if (stbi__get8(s) != 'B') return 0; + if (stbi__get8(s) != 'M') return 0; + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + stbi__get32le(s); // discard data offset + sz = stbi__get32le(s); + r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); + return r; +} + +static int stbi__bmp_test(stbi__context *s) +{ + int r = stbi__bmp_test_raw(s); + stbi__rewind(s); + return r; +} + + +// returns 0..31 for the highest set bit +static int stbi__high_bit(unsigned int z) +{ + int n=0; + if (z == 0) return -1; + if (z >= 0x10000) { n += 16; z >>= 16; } + if (z >= 0x00100) { n += 8; z >>= 8; } + if (z >= 0x00010) { n += 4; z >>= 4; } + if (z >= 0x00004) { n += 2; z >>= 2; } + if (z >= 0x00002) { n += 1;/* >>= 1;*/ } + return n; +} + +static int stbi__bitcount(unsigned int a) +{ + a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 + a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits + a = (a + (a >> 8)); // max 16 per 8 bits + a = (a + (a >> 16)); // max 32 per 8 bits + return a & 0xff; +} + +// extract an arbitrarily-aligned N-bit value (N=bits) +// from v, and then make it 8-bits long and fractionally +// extend it to full full range. +static int stbi__shiftsigned(unsigned int v, int shift, int bits) +{ + static unsigned int mul_table[9] = { + 0, + 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, + 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, + }; + static unsigned int shift_table[9] = { + 0, 0,0,1,0,2,4,6,0, + }; + if (shift < 0) + v <<= -shift; + else + v >>= shift; + STBI_ASSERT(v < 256); + v >>= (8-bits); + STBI_ASSERT(bits >= 0 && bits <= 8); + return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; +} + +typedef struct +{ + int bpp, offset, hsz; + unsigned int mr,mg,mb,ma, all_a; + int extra_read; +} stbi__bmp_data; + +static int stbi__bmp_set_mask_defaults(stbi__bmp_data *info, int compress) +{ + // BI_BITFIELDS specifies masks explicitly, don't override + if (compress == 3) + return 1; + + if (compress == 0) { + if (info->bpp == 16) { + info->mr = 31u << 10; + info->mg = 31u << 5; + info->mb = 31u << 0; + } else if (info->bpp == 32) { + info->mr = 0xffu << 16; + info->mg = 0xffu << 8; + info->mb = 0xffu << 0; + info->ma = 0xffu << 24; + info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 + } else { + // otherwise, use defaults, which is all-0 + info->mr = info->mg = info->mb = info->ma = 0; + } + return 1; + } + return 0; // error +} + +static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) +{ + int hsz; + if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + info->offset = stbi__get32le(s); + info->hsz = hsz = stbi__get32le(s); + info->mr = info->mg = info->mb = info->ma = 0; + info->extra_read = 14; + + if (info->offset < 0) return stbi__errpuc("bad BMP", "bad BMP"); + + if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); + if (hsz == 12) { + s->img_x = stbi__get16le(s); + s->img_y = stbi__get16le(s); + } else { + s->img_x = stbi__get32le(s); + s->img_y = stbi__get32le(s); + } + if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); + info->bpp = stbi__get16le(s); + if (hsz != 12) { + int compress = stbi__get32le(s); + if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); + if (compress >= 4) return stbi__errpuc("BMP JPEG/PNG", "BMP type not supported: unsupported compression"); // this includes PNG/JPEG modes + if (compress == 3 && info->bpp != 16 && info->bpp != 32) return stbi__errpuc("bad BMP", "bad BMP"); // bitfields requires 16 or 32 bits/pixel + stbi__get32le(s); // discard sizeof + stbi__get32le(s); // discard hres + stbi__get32le(s); // discard vres + stbi__get32le(s); // discard colorsused + stbi__get32le(s); // discard max important + if (hsz == 40 || hsz == 56) { + if (hsz == 56) { + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + } + if (info->bpp == 16 || info->bpp == 32) { + if (compress == 0) { + stbi__bmp_set_mask_defaults(info, compress); + } else if (compress == 3) { + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->extra_read += 12; + // not documented, but generated by photoshop and handled by mspaint + if (info->mr == info->mg && info->mg == info->mb) { + // ?!?!? + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else { + // V4/V5 header + int i; + if (hsz != 108 && hsz != 124) + return stbi__errpuc("bad BMP", "bad BMP"); + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->ma = stbi__get32le(s); + if (compress != 3) // override mr/mg/mb unless in BI_BITFIELDS mode, as per docs + stbi__bmp_set_mask_defaults(info, compress); + stbi__get32le(s); // discard color space + for (i=0; i < 12; ++i) + stbi__get32le(s); // discard color space parameters + if (hsz == 124) { + stbi__get32le(s); // discard rendering intent + stbi__get32le(s); // discard offset of profile data + stbi__get32le(s); // discard size of profile data + stbi__get32le(s); // discard reserved + } + } + } + return (void *) 1; +} + + +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + unsigned int mr=0,mg=0,mb=0,ma=0, all_a; + stbi_uc pal[256][4]; + int psize=0,i,j,width; + int flip_vertically, pad, target; + stbi__bmp_data info; + STBI_NOTUSED(ri); + + info.all_a = 255; + if (stbi__bmp_parse_header(s, &info) == NULL) + return NULL; // error code already set + + flip_vertically = ((int) s->img_y) > 0; + s->img_y = abs((int) s->img_y); + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + mr = info.mr; + mg = info.mg; + mb = info.mb; + ma = info.ma; + all_a = info.all_a; + + if (info.hsz == 12) { + if (info.bpp < 24) + psize = (info.offset - info.extra_read - 24) / 3; + } else { + if (info.bpp < 16) + psize = (info.offset - info.extra_read - info.hsz) >> 2; + } + if (psize == 0) { + // accept some number of extra bytes after the header, but if the offset points either to before + // the header ends or implies a large amount of extra data, reject the file as malformed + int bytes_read_so_far = s->callback_already_read + (int)(s->img_buffer - s->img_buffer_original); + int header_limit = 1024; // max we actually read is below 256 bytes currently. + int extra_data_limit = 256*4; // what ordinarily goes here is a palette; 256 entries*4 bytes is its max size. + if (bytes_read_so_far <= 0 || bytes_read_so_far > header_limit) { + return stbi__errpuc("bad header", "Corrupt BMP"); + } + // we established that bytes_read_so_far is positive and sensible. + // the first half of this test rejects offsets that are either too small positives, or + // negative, and guarantees that info.offset >= bytes_read_so_far > 0. this in turn + // ensures the number computed in the second half of the test can't overflow. + if (info.offset < bytes_read_so_far || info.offset - bytes_read_so_far > extra_data_limit) { + return stbi__errpuc("bad offset", "Corrupt BMP"); + } else { + stbi__skip(s, info.offset - bytes_read_so_far); + } + } + + if (info.bpp == 24 && ma == 0xff000000) + s->img_n = 3; + else + s->img_n = ma ? 4 : 3; + if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 + target = req_comp; + else + target = s->img_n; // if they want monochrome, we'll post-convert + + // sanity-check size + if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "Corrupt BMP"); + + out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (info.bpp < 16) { + int z=0; + if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } + for (i=0; i < psize; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + if (info.hsz != 12) stbi__get8(s); + pal[i][3] = 255; + } + stbi__skip(s, info.offset - info.extra_read - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); + if (info.bpp == 1) width = (s->img_x + 7) >> 3; + else if (info.bpp == 4) width = (s->img_x + 1) >> 1; + else if (info.bpp == 8) width = s->img_x; + else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } + pad = (-width)&3; + if (info.bpp == 1) { + for (j=0; j < (int) s->img_y; ++j) { + int bit_offset = 7, v = stbi__get8(s); + for (i=0; i < (int) s->img_x; ++i) { + int color = (v>>bit_offset)&0x1; + out[z++] = pal[color][0]; + out[z++] = pal[color][1]; + out[z++] = pal[color][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + if((--bit_offset) < 0) { + bit_offset = 7; + v = stbi__get8(s); + } + } + stbi__skip(s, pad); + } + } else { + for (j=0; j < (int) s->img_y; ++j) { + for (i=0; i < (int) s->img_x; i += 2) { + int v=stbi__get8(s),v2=0; + if (info.bpp == 4) { + v2 = v & 15; + v >>= 4; + } + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + v = (info.bpp == 8) ? stbi__get8(s) : v2; + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + } + stbi__skip(s, pad); + } + } + } else { + int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; + int z = 0; + int easy=0; + stbi__skip(s, info.offset - info.extra_read - info.hsz); + if (info.bpp == 24) width = 3 * s->img_x; + else if (info.bpp == 16) width = 2*s->img_x; + else /* bpp = 32 and pad = 0 */ width=0; + pad = (-width) & 3; + if (info.bpp == 24) { + easy = 1; + } else if (info.bpp == 32) { + if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) + easy = 2; + } + if (!easy) { + if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + // right shift amt to put high bit in position #7 + rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); + gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); + bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); + ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); + if (rcount > 8 || gcount > 8 || bcount > 8 || acount > 8) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + } + for (j=0; j < (int) s->img_y; ++j) { + if (easy) { + for (i=0; i < (int) s->img_x; ++i) { + unsigned char a; + out[z+2] = stbi__get8(s); + out[z+1] = stbi__get8(s); + out[z+0] = stbi__get8(s); + z += 3; + a = (easy == 2 ? stbi__get8(s) : 255); + all_a |= a; + if (target == 4) out[z++] = a; + } + } else { + int bpp = info.bpp; + for (i=0; i < (int) s->img_x; ++i) { + stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); + unsigned int a; + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); + a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); + all_a |= a; + if (target == 4) out[z++] = STBI__BYTECAST(a); + } + } + stbi__skip(s, pad); + } + } + + // if alpha channel is all 0s, replace with all 255s + if (target == 4 && all_a == 0) + for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) + out[i] = 255; + + if (flip_vertically) { + stbi_uc t; + for (j=0; j < (int) s->img_y>>1; ++j) { + stbi_uc *p1 = out + j *s->img_x*target; + stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; + for (i=0; i < (int) s->img_x*target; ++i) { + t = p1[i]; p1[i] = p2[i]; p2[i] = t; + } + } + } + + if (req_comp && req_comp != target) { + out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + return out; +} +#endif + +// Targa Truevision - TGA +// by Jonathan Dummer +#ifndef STBI_NO_TGA +// returns STBI_rgb or whatever, 0 on error +static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) +{ + // only RGB or RGBA (incl. 16bit) or grey allowed + if (is_rgb16) *is_rgb16 = 0; + switch(bits_per_pixel) { + case 8: return STBI_grey; + case 16: if(is_grey) return STBI_grey_alpha; + // fallthrough + case 15: if(is_rgb16) *is_rgb16 = 1; + return STBI_rgb; + case 24: // fallthrough + case 32: return bits_per_pixel/8; + default: return 0; + } +} + +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) +{ + int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; + int sz, tga_colormap_type; + stbi__get8(s); // discard Offset + tga_colormap_type = stbi__get8(s); // colormap type + if( tga_colormap_type > 1 ) { + stbi__rewind(s); + return 0; // only RGB or indexed allowed + } + tga_image_type = stbi__get8(s); // image type + if ( tga_colormap_type == 1 ) { // colormapped (paletted) image + if (tga_image_type != 1 && tga_image_type != 9) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip image x and y origin + tga_colormap_bpp = sz; + } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE + if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { + stbi__rewind(s); + return 0; // only RGB or grey allowed, +/- RLE + } + stbi__skip(s,9); // skip colormap specification and image x/y origin + tga_colormap_bpp = 0; + } + tga_w = stbi__get16le(s); + if( tga_w < 1 ) { + stbi__rewind(s); + return 0; // test width + } + tga_h = stbi__get16le(s); + if( tga_h < 1 ) { + stbi__rewind(s); + return 0; // test height + } + tga_bits_per_pixel = stbi__get8(s); // bits per pixel + stbi__get8(s); // ignore alpha bits + if (tga_colormap_bpp != 0) { + if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { + // when using a colormap, tga_bits_per_pixel is the size of the indexes + // I don't think anything but 8 or 16bit indexes makes sense + stbi__rewind(s); + return 0; + } + tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); + } else { + tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); + } + if(!tga_comp) { + stbi__rewind(s); + return 0; + } + if (x) *x = tga_w; + if (y) *y = tga_h; + if (comp) *comp = tga_comp; + return 1; // seems to have passed everything +} + +static int stbi__tga_test(stbi__context *s) +{ + int res = 0; + int sz, tga_color_type; + stbi__get8(s); // discard Offset + tga_color_type = stbi__get8(s); // color type + if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed + sz = stbi__get8(s); // image type + if ( tga_color_type == 1 ) { // colormapped (paletted) image + if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + stbi__skip(s,4); // skip image x and y origin + } else { // "normal" image w/o colormap + if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE + stbi__skip(s,9); // skip colormap specification and image x/y origin + } + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height + sz = stbi__get8(s); // bits per pixel + if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + + res = 1; // if we got this far, everything's good and we can return 1 instead of 0 + +errorEnd: + stbi__rewind(s); + return res; +} + +// read 16bit value and convert to 24bit RGB +static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) +{ + stbi__uint16 px = (stbi__uint16)stbi__get16le(s); + stbi__uint16 fiveBitMask = 31; + // we have 3 channels with 5bits each + int r = (px >> 10) & fiveBitMask; + int g = (px >> 5) & fiveBitMask; + int b = px & fiveBitMask; + // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later + out[0] = (stbi_uc)((r * 255)/31); + out[1] = (stbi_uc)((g * 255)/31); + out[2] = (stbi_uc)((b * 255)/31); + + // some people claim that the most significant bit might be used for alpha + // (possibly if an alpha-bit is set in the "image descriptor byte") + // but that only made 16bit test images completely translucent.. + // so let's treat all 15 and 16bit TGAs as RGB with no alpha. +} + +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + // read in the TGA header stuff + int tga_offset = stbi__get8(s); + int tga_indexed = stbi__get8(s); + int tga_image_type = stbi__get8(s); + int tga_is_RLE = 0; + int tga_palette_start = stbi__get16le(s); + int tga_palette_len = stbi__get16le(s); + int tga_palette_bits = stbi__get8(s); + int tga_x_origin = stbi__get16le(s); + int tga_y_origin = stbi__get16le(s); + int tga_width = stbi__get16le(s); + int tga_height = stbi__get16le(s); + int tga_bits_per_pixel = stbi__get8(s); + int tga_comp, tga_rgb16=0; + int tga_inverted = stbi__get8(s); + // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) + // image data + unsigned char *tga_data; + unsigned char *tga_palette = NULL; + int i, j; + unsigned char raw_data[4] = {0}; + int RLE_count = 0; + int RLE_repeating = 0; + int read_next_pixel = 1; + STBI_NOTUSED(ri); + STBI_NOTUSED(tga_x_origin); // @TODO + STBI_NOTUSED(tga_y_origin); // @TODO + + if (tga_height > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (tga_width > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // do a tiny bit of precessing + if ( tga_image_type >= 8 ) + { + tga_image_type -= 8; + tga_is_RLE = 1; + } + tga_inverted = 1 - ((tga_inverted >> 5) & 1); + + // If I'm paletted, then I'll use the number of bits from the palette + if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); + else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); + + if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency + return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); + + // tga info + *x = tga_width; + *y = tga_height; + if (comp) *comp = tga_comp; + + if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) + return stbi__errpuc("too large", "Corrupt TGA"); + + tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); + if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); + + // skip to the data's starting position (offset usually = 0) + stbi__skip(s, tga_offset ); + + if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { + for (i=0; i < tga_height; ++i) { + int row = tga_inverted ? tga_height -i - 1 : i; + stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; + stbi__getn(s, tga_row, tga_width * tga_comp); + } + } else { + // do I need to load a palette? + if ( tga_indexed) + { + if (tga_palette_len == 0) { /* you have to have at least one entry! */ + STBI_FREE(tga_data); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + + // any data to skip? (offset usually = 0) + stbi__skip(s, tga_palette_start ); + // load the palette + tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); + if (!tga_palette) { + STBI_FREE(tga_data); + return stbi__errpuc("outofmem", "Out of memory"); + } + if (tga_rgb16) { + stbi_uc *pal_entry = tga_palette; + STBI_ASSERT(tga_comp == STBI_rgb); + for (i=0; i < tga_palette_len; ++i) { + stbi__tga_read_rgb16(s, pal_entry); + pal_entry += tga_comp; + } + } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { + STBI_FREE(tga_data); + STBI_FREE(tga_palette); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + } + // load the data + for (i=0; i < tga_width * tga_height; ++i) + { + // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? + if ( tga_is_RLE ) + { + if ( RLE_count == 0 ) + { + // yep, get the next byte as a RLE command + int RLE_cmd = stbi__get8(s); + RLE_count = 1 + (RLE_cmd & 127); + RLE_repeating = RLE_cmd >> 7; + read_next_pixel = 1; + } else if ( !RLE_repeating ) + { + read_next_pixel = 1; + } + } else + { + read_next_pixel = 1; + } + // OK, if I need to read a pixel, do it now + if ( read_next_pixel ) + { + // load however much data we did have + if ( tga_indexed ) + { + // read in index, then perform the lookup + int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); + if ( pal_idx >= tga_palette_len ) { + // invalid index + pal_idx = 0; + } + pal_idx *= tga_comp; + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = tga_palette[pal_idx+j]; + } + } else if(tga_rgb16) { + STBI_ASSERT(tga_comp == STBI_rgb); + stbi__tga_read_rgb16(s, raw_data); + } else { + // read in the data raw + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = stbi__get8(s); + } + } + // clear the reading flag for the next pixel + read_next_pixel = 0; + } // end of reading a pixel + + // copy data + for (j = 0; j < tga_comp; ++j) + tga_data[i*tga_comp+j] = raw_data[j]; + + // in case we're in RLE mode, keep counting down + --RLE_count; + } + // do I need to invert the image? + if ( tga_inverted ) + { + for (j = 0; j*2 < tga_height; ++j) + { + int index1 = j * tga_width * tga_comp; + int index2 = (tga_height - 1 - j) * tga_width * tga_comp; + for (i = tga_width * tga_comp; i > 0; --i) + { + unsigned char temp = tga_data[index1]; + tga_data[index1] = tga_data[index2]; + tga_data[index2] = temp; + ++index1; + ++index2; + } + } + } + // clear my palette, if I had one + if ( tga_palette != NULL ) + { + STBI_FREE( tga_palette ); + } + } + + // swap RGB - if the source data was RGB16, it already is in the right order + if (tga_comp >= 3 && !tga_rgb16) + { + unsigned char* tga_pixel = tga_data; + for (i=0; i < tga_width * tga_height; ++i) + { + unsigned char temp = tga_pixel[0]; + tga_pixel[0] = tga_pixel[2]; + tga_pixel[2] = temp; + tga_pixel += tga_comp; + } + } + + // convert to target component count + if (req_comp && req_comp != tga_comp) + tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); + + // the things I do to get rid of an error message, and yet keep + // Microsoft's C compilers happy... [8^( + tga_palette_start = tga_palette_len = tga_palette_bits = + tga_x_origin = tga_y_origin = 0; + STBI_NOTUSED(tga_palette_start); + // OK, done + return tga_data; +} +#endif + +// ************************************************************************************************* +// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s) +{ + int r = (stbi__get32be(s) == 0x38425053); + stbi__rewind(s); + return r; +} + +static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) +{ + int count, nleft, len; + + count = 0; + while ((nleft = pixelCount - count) > 0) { + len = stbi__get8(s); + if (len == 128) { + // No-op. + } else if (len < 128) { + // Copy next len+1 bytes literally. + len++; + if (len > nleft) return 0; // corrupt data + count += len; + while (len) { + *p = stbi__get8(s); + p += 4; + len--; + } + } else if (len > 128) { + stbi_uc val; + // Next -len+1 bytes in the dest are replicated from next source byte. + // (Interpret len as a negative 8-bit int.) + len = 257 - len; + if (len > nleft) return 0; // corrupt data + val = stbi__get8(s); + count += len; + while (len) { + *p = val; + p += 4; + len--; + } + } + } + + return 1; +} + +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + int pixelCount; + int channelCount, compression; + int channel, i; + int bitdepth; + int w,h; + stbi_uc *out; + STBI_NOTUSED(ri); + + // Check identifier + if (stbi__get32be(s) != 0x38425053) // "8BPS" + return stbi__errpuc("not PSD", "Corrupt PSD image"); + + // Check file type version. + if (stbi__get16be(s) != 1) + return stbi__errpuc("wrong version", "Unsupported version of PSD image"); + + // Skip 6 reserved bytes. + stbi__skip(s, 6 ); + + // Read the number of channels (R, G, B, A, etc). + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) + return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); + + // Read the rows and columns of the image. + h = stbi__get32be(s); + w = stbi__get32be(s); + + if (h > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (w > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + // Make sure the depth is 8 bits. + bitdepth = stbi__get16be(s); + if (bitdepth != 8 && bitdepth != 16) + return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); + + // Make sure the color mode is RGB. + // Valid options are: + // 0: Bitmap + // 1: Grayscale + // 2: Indexed color + // 3: RGB color + // 4: CMYK color + // 7: Multichannel + // 8: Duotone + // 9: Lab color + if (stbi__get16be(s) != 3) + return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); + + // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) + stbi__skip(s,stbi__get32be(s) ); + + // Skip the image resources. (resolution, pen tool paths, etc) + stbi__skip(s, stbi__get32be(s) ); + + // Skip the reserved data. + stbi__skip(s, stbi__get32be(s) ); + + // Find out if the data is compressed. + // Known values: + // 0: no compression + // 1: RLE compressed + compression = stbi__get16be(s); + if (compression > 1) + return stbi__errpuc("bad compression", "PSD has an unknown compression format"); + + // Check size + if (!stbi__mad3sizes_valid(4, w, h, 0)) + return stbi__errpuc("too large", "Corrupt PSD"); + + // Create the destination image. + + if (!compression && bitdepth == 16 && bpc == 16) { + out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); + ri->bits_per_channel = 16; + } else + out = (stbi_uc *) stbi__malloc(4 * w*h); + + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + pixelCount = w*h; + + // Initialize the data to zero. + //memset( out, 0, pixelCount * 4 ); + + // Finally, the image data. + if (compression) { + // RLE as used by .PSD and .TIFF + // Loop until you get the number of unpacked bytes you are expecting: + // Read the next source byte into n. + // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. + // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. + // Else if n is 128, noop. + // Endloop + + // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, + // which we're going to just skip. + stbi__skip(s, h * channelCount * 2 ); + + // Read the RLE data by channel. + for (channel = 0; channel < 4; channel++) { + stbi_uc *p; + + p = out+channel; + if (channel >= channelCount) { + // Fill this channel with default data. + for (i = 0; i < pixelCount; i++, p += 4) + *p = (channel == 3 ? 255 : 0); + } else { + // Read the RLE data. + if (!stbi__psd_decode_rle(s, p, pixelCount)) { + STBI_FREE(out); + return stbi__errpuc("corrupt", "bad RLE data"); + } + } + } + + } else { + // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) + // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. + + // Read the data by channel. + for (channel = 0; channel < 4; channel++) { + if (channel >= channelCount) { + // Fill this channel with default data. + if (bitdepth == 16 && bpc == 16) { + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + stbi__uint16 val = channel == 3 ? 65535 : 0; + for (i = 0; i < pixelCount; i++, q += 4) + *q = val; + } else { + stbi_uc *p = out+channel; + stbi_uc val = channel == 3 ? 255 : 0; + for (i = 0; i < pixelCount; i++, p += 4) + *p = val; + } + } else { + if (ri->bits_per_channel == 16) { // output bpc + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + for (i = 0; i < pixelCount; i++, q += 4) + *q = (stbi__uint16) stbi__get16be(s); + } else { + stbi_uc *p = out+channel; + if (bitdepth == 16) { // input bpc + for (i = 0; i < pixelCount; i++, p += 4) + *p = (stbi_uc) (stbi__get16be(s) >> 8); + } else { + for (i = 0; i < pixelCount; i++, p += 4) + *p = stbi__get8(s); + } + } + } + } + } + + // remove weird white matte from PSD + if (channelCount >= 4) { + if (ri->bits_per_channel == 16) { + for (i=0; i < w*h; ++i) { + stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; + if (pixel[3] != 0 && pixel[3] != 65535) { + float a = pixel[3] / 65535.0f; + float ra = 1.0f / a; + float inv_a = 65535.0f * (1 - ra); + pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); + pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); + pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); + } + } + } else { + for (i=0; i < w*h; ++i) { + unsigned char *pixel = out + 4*i; + if (pixel[3] != 0 && pixel[3] != 255) { + float a = pixel[3] / 255.0f; + float ra = 1.0f / a; + float inv_a = 255.0f * (1 - ra); + pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); + pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); + pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); + } + } + } + } + + // convert to desired output format + if (req_comp && req_comp != 4) { + if (ri->bits_per_channel == 16) + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); + else + out = stbi__convert_format(out, 4, req_comp, w, h); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + if (comp) *comp = 4; + *y = h; + *x = w; + + return out; +} +#endif + +// ************************************************************************************************* +// Softimage PIC loader +// by Tom Seddon +// +// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format +// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ + +#ifndef STBI_NO_PIC +static int stbi__pic_is4(stbi__context *s,const char *str) +{ + int i; + for (i=0; i<4; ++i) + if (stbi__get8(s) != (stbi_uc)str[i]) + return 0; + + return 1; +} + +static int stbi__pic_test_core(stbi__context *s) +{ + int i; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) + return 0; + + for(i=0;i<84;++i) + stbi__get8(s); + + if (!stbi__pic_is4(s,"PICT")) + return 0; + + return 1; +} + +typedef struct +{ + stbi_uc size,type,channel; +} stbi__pic_packet; + +static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) +{ + int mask=0x80, i; + + for (i=0; i<4; ++i, mask>>=1) { + if (channel & mask) { + if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); + dest[i]=stbi__get8(s); + } + } + + return dest; +} + +static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) +{ + int mask=0x80,i; + + for (i=0;i<4; ++i, mask>>=1) + if (channel&mask) + dest[i]=src[i]; +} + +static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) +{ + int act_comp=0,num_packets=0,y,chained; + stbi__pic_packet packets[10]; + + // this will (should...) cater for even some bizarre stuff like having data + // for the same channel in multiple packets. + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return stbi__errpuc("bad format","too many packets"); + + packet = &packets[num_packets++]; + + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + + act_comp |= packet->channel; + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); + if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? + + for(y=0; ytype) { + default: + return stbi__errpuc("bad format","packet has bad compression type"); + + case 0: {//uncompressed + int x; + + for(x=0;xchannel,dest)) + return 0; + break; + } + + case 1://Pure RLE + { + int left=width, i; + + while (left>0) { + stbi_uc count,value[4]; + + count=stbi__get8(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); + + if (count > left) + count = (stbi_uc) left; + + if (!stbi__readval(s,packet->channel,value)) return 0; + + for(i=0; ichannel,dest,value); + left -= count; + } + } + break; + + case 2: {//Mixed RLE + int left=width; + while (left>0) { + int count = stbi__get8(s), i; + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); + + if (count >= 128) { // Repeated + stbi_uc value[4]; + + if (count==128) + count = stbi__get16be(s); + else + count -= 127; + if (count > left) + return stbi__errpuc("bad file","scanline overrun"); + + if (!stbi__readval(s,packet->channel,value)) + return 0; + + for(i=0;ichannel,dest,value); + } else { // Raw + ++count; + if (count>left) return stbi__errpuc("bad file","scanline overrun"); + + for(i=0;ichannel,dest)) + return 0; + } + left-=count; + } + break; + } + } + } + } + + return result; +} + +static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) +{ + stbi_uc *result; + int i, x,y, internal_comp; + STBI_NOTUSED(ri); + + if (!comp) comp = &internal_comp; + + for (i=0; i<92; ++i) + stbi__get8(s); + + x = stbi__get16be(s); + y = stbi__get16be(s); + + if (y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); + if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); + + stbi__get32be(s); //skip `ratio' + stbi__get16be(s); //skip `fields' + stbi__get16be(s); //skip `pad' + + // intermediate buffer is RGBA + result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); + if (!result) return stbi__errpuc("outofmem", "Out of memory"); + memset(result, 0xff, x*y*4); + + if (!stbi__pic_load_core(s,x,y,comp, result)) { + STBI_FREE(result); + result=0; + } + *px = x; + *py = y; + if (req_comp == 0) req_comp = *comp; + result=stbi__convert_format(result,4,req_comp,x,y); + + return result; +} + +static int stbi__pic_test(stbi__context *s) +{ + int r = stbi__pic_test_core(s); + stbi__rewind(s); + return r; +} +#endif + +// ************************************************************************************************* +// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb + +#ifndef STBI_NO_GIF +typedef struct +{ + stbi__int16 prefix; + stbi_uc first; + stbi_uc suffix; +} stbi__gif_lzw; + +typedef struct +{ + int w,h; + stbi_uc *out; // output buffer (always 4 components) + stbi_uc *background; // The current "background" as far as a gif is concerned + stbi_uc *history; + int flags, bgindex, ratio, transparent, eflags; + stbi_uc pal[256][4]; + stbi_uc lpal[256][4]; + stbi__gif_lzw codes[8192]; + stbi_uc *color_table; + int parse, step; + int lflags; + int start_x, start_y; + int max_x, max_y; + int cur_x, cur_y; + int line_size; + int delay; +} stbi__gif; + +static int stbi__gif_test_raw(stbi__context *s) +{ + int sz; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; + sz = stbi__get8(s); + if (sz != '9' && sz != '7') return 0; + if (stbi__get8(s) != 'a') return 0; + return 1; +} + +static int stbi__gif_test(stbi__context *s) +{ + int r = stbi__gif_test_raw(s); + stbi__rewind(s); + return r; +} + +static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) +{ + int i; + for (i=0; i < num_entries; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + pal[i][3] = transp == i ? 0 : 255; + } +} + +static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) +{ + stbi_uc version; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') + return stbi__err("not GIF", "Corrupt GIF"); + + version = stbi__get8(s); + if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); + if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); + + stbi__g_failure_reason = ""; + g->w = stbi__get16le(s); + g->h = stbi__get16le(s); + g->flags = stbi__get8(s); + g->bgindex = stbi__get8(s); + g->ratio = stbi__get8(s); + g->transparent = -1; + + if (g->w > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + if (g->h > STBI_MAX_DIMENSIONS) return stbi__err("too large","Very large image (corrupt?)"); + + if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments + + if (is_info) return 1; + + if (g->flags & 0x80) + stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); + + return 1; +} + +static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); + if (!g) return stbi__err("outofmem", "Out of memory"); + if (!stbi__gif_header(s, g, comp, 1)) { + STBI_FREE(g); + stbi__rewind( s ); + return 0; + } + if (x) *x = g->w; + if (y) *y = g->h; + STBI_FREE(g); + return 1; +} + +static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) +{ + stbi_uc *p, *c; + int idx; + + // recurse to decode the prefixes, since the linked-list is backwards, + // and working backwards through an interleaved image would be nasty + if (g->codes[code].prefix >= 0) + stbi__out_gif_code(g, g->codes[code].prefix); + + if (g->cur_y >= g->max_y) return; + + idx = g->cur_x + g->cur_y; + p = &g->out[idx]; + g->history[idx / 4] = 1; + + c = &g->color_table[g->codes[code].suffix * 4]; + if (c[3] > 128) { // don't render transparent pixels; + p[0] = c[2]; + p[1] = c[1]; + p[2] = c[0]; + p[3] = c[3]; + } + g->cur_x += 4; + + if (g->cur_x >= g->max_x) { + g->cur_x = g->start_x; + g->cur_y += g->step; + + while (g->cur_y >= g->max_y && g->parse > 0) { + g->step = (1 << g->parse) * g->line_size; + g->cur_y = g->start_y + (g->step >> 1); + --g->parse; + } + } +} + +static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) +{ + stbi_uc lzw_cs; + stbi__int32 len, init_code; + stbi__uint32 first; + stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; + stbi__gif_lzw *p; + + lzw_cs = stbi__get8(s); + if (lzw_cs > 12) return NULL; + clear = 1 << lzw_cs; + first = 1; + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + bits = 0; + valid_bits = 0; + for (init_code = 0; init_code < clear; init_code++) { + g->codes[init_code].prefix = -1; + g->codes[init_code].first = (stbi_uc) init_code; + g->codes[init_code].suffix = (stbi_uc) init_code; + } + + // support no starting clear code + avail = clear+2; + oldcode = -1; + + len = 0; + for(;;) { + if (valid_bits < codesize) { + if (len == 0) { + len = stbi__get8(s); // start new block + if (len == 0) + return g->out; + } + --len; + bits |= (stbi__int32) stbi__get8(s) << valid_bits; + valid_bits += 8; + } else { + stbi__int32 code = bits & codemask; + bits >>= codesize; + valid_bits -= codesize; + // @OPTIMIZE: is there some way we can accelerate the non-clear path? + if (code == clear) { // clear code + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + avail = clear + 2; + oldcode = -1; + first = 0; + } else if (code == clear + 1) { // end of stream code + stbi__skip(s, len); + while ((len = stbi__get8(s)) > 0) + stbi__skip(s,len); + return g->out; + } else if (code <= avail) { + if (first) { + return stbi__errpuc("no clear code", "Corrupt GIF"); + } + + if (oldcode >= 0) { + p = &g->codes[avail++]; + if (avail > 8192) { + return stbi__errpuc("too many codes", "Corrupt GIF"); + } + + p->prefix = (stbi__int16) oldcode; + p->first = g->codes[oldcode].first; + p->suffix = (code == avail) ? p->first : g->codes[code].first; + } else if (code == avail) + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + + stbi__out_gif_code(g, (stbi__uint16) code); + + if ((avail & codemask) == 0 && avail <= 0x0FFF) { + codesize++; + codemask = (1 << codesize) - 1; + } + + oldcode = code; + } else { + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + } + } + } +} + +// this function is designed to support animated gifs, although stb_image doesn't support it +// two back is the image from two frames ago, used for a very specific disposal format +static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) +{ + int dispose; + int first_frame; + int pi; + int pcount; + STBI_NOTUSED(req_comp); + + // on first frame, any non-written pixels get the background colour (non-transparent) + first_frame = 0; + if (g->out == 0) { + if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header + if (!stbi__mad3sizes_valid(4, g->w, g->h, 0)) + return stbi__errpuc("too large", "GIF image is too large"); + pcount = g->w * g->h; + g->out = (stbi_uc *) stbi__malloc(4 * pcount); + g->background = (stbi_uc *) stbi__malloc(4 * pcount); + g->history = (stbi_uc *) stbi__malloc(pcount); + if (!g->out || !g->background || !g->history) + return stbi__errpuc("outofmem", "Out of memory"); + + // image is treated as "transparent" at the start - ie, nothing overwrites the current background; + // background colour is only used for pixels that are not rendered first frame, after that "background" + // color refers to the color that was there the previous frame. + memset(g->out, 0x00, 4 * pcount); + memset(g->background, 0x00, 4 * pcount); // state of the background (starts transparent) + memset(g->history, 0x00, pcount); // pixels that were affected previous frame + first_frame = 1; + } else { + // second frame - how do we dispose of the previous one? + dispose = (g->eflags & 0x1C) >> 2; + pcount = g->w * g->h; + + if ((dispose == 3) && (two_back == 0)) { + dispose = 2; // if I don't have an image to revert back to, default to the old background + } + + if (dispose == 3) { // use previous graphic + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); + } + } + } else if (dispose == 2) { + // restore what was changed last frame to background before that frame; + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); + } + } + } else { + // This is a non-disposal case eithe way, so just + // leave the pixels as is, and they will become the new background + // 1: do not dispose + // 0: not specified. + } + + // background is what out is after the undoing of the previou frame; + memcpy( g->background, g->out, 4 * g->w * g->h ); + } + + // clear my history; + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + + for (;;) { + int tag = stbi__get8(s); + switch (tag) { + case 0x2C: /* Image Descriptor */ + { + stbi__int32 x, y, w, h; + stbi_uc *o; + + x = stbi__get16le(s); + y = stbi__get16le(s); + w = stbi__get16le(s); + h = stbi__get16le(s); + if (((x + w) > (g->w)) || ((y + h) > (g->h))) + return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); + + g->line_size = g->w * 4; + g->start_x = x * 4; + g->start_y = y * g->line_size; + g->max_x = g->start_x + w * 4; + g->max_y = g->start_y + h * g->line_size; + g->cur_x = g->start_x; + g->cur_y = g->start_y; + + // if the width of the specified rectangle is 0, that means + // we may not see *any* pixels or the image is malformed; + // to make sure this is caught, move the current y down to + // max_y (which is what out_gif_code checks). + if (w == 0) + g->cur_y = g->max_y; + + g->lflags = stbi__get8(s); + + if (g->lflags & 0x40) { + g->step = 8 * g->line_size; // first interlaced spacing + g->parse = 3; + } else { + g->step = g->line_size; + g->parse = 0; + } + + if (g->lflags & 0x80) { + stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); + g->color_table = (stbi_uc *) g->lpal; + } else if (g->flags & 0x80) { + g->color_table = (stbi_uc *) g->pal; + } else + return stbi__errpuc("missing color table", "Corrupt GIF"); + + o = stbi__process_gif_raster(s, g); + if (!o) return NULL; + + // if this was the first frame, + pcount = g->w * g->h; + if (first_frame && (g->bgindex > 0)) { + // if first frame, any pixel not drawn to gets the background color + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi] == 0) { + g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; + memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); + } + } + } + + return o; + } + + case 0x21: // Comment Extension. + { + int len; + int ext = stbi__get8(s); + if (ext == 0xF9) { // Graphic Control Extension. + len = stbi__get8(s); + if (len == 4) { + g->eflags = stbi__get8(s); + g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. + + // unset old transparent + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 255; + } + if (g->eflags & 0x01) { + g->transparent = stbi__get8(s); + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 0; + } + } else { + // don't need transparent + stbi__skip(s, 1); + g->transparent = -1; + } + } else { + stbi__skip(s, len); + break; + } + } + while ((len = stbi__get8(s)) != 0) { + stbi__skip(s, len); + } + break; + } + + case 0x3B: // gif stream termination code + return (stbi_uc *) s; // using '1' causes warning on some compilers + + default: + return stbi__errpuc("unknown code", "Corrupt GIF"); + } + } +} + +static void *stbi__load_gif_main_outofmem(stbi__gif *g, stbi_uc *out, int **delays) +{ + STBI_FREE(g->out); + STBI_FREE(g->history); + STBI_FREE(g->background); + + if (out) STBI_FREE(out); + if (delays && *delays) STBI_FREE(*delays); + return stbi__errpuc("outofmem", "Out of memory"); +} + +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + if (stbi__gif_test(s)) { + int layers = 0; + stbi_uc *u = 0; + stbi_uc *out = 0; + stbi_uc *two_back = 0; + stbi__gif g; + int stride; + int out_size = 0; + int delays_size = 0; + + STBI_NOTUSED(out_size); + STBI_NOTUSED(delays_size); + + memset(&g, 0, sizeof(g)); + if (delays) { + *delays = 0; + } + + do { + u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + + if (u) { + *x = g.w; + *y = g.h; + ++layers; + stride = g.w * g.h * 4; + + if (out) { + void *tmp = (stbi_uc*) STBI_REALLOC_SIZED( out, out_size, layers * stride ); + if (!tmp) + return stbi__load_gif_main_outofmem(&g, out, delays); + else { + out = (stbi_uc*) tmp; + out_size = layers * stride; + } + + if (delays) { + int *new_delays = (int*) STBI_REALLOC_SIZED( *delays, delays_size, sizeof(int) * layers ); + if (!new_delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + *delays = new_delays; + delays_size = layers * sizeof(int); + } + } else { + out = (stbi_uc*)stbi__malloc( layers * stride ); + if (!out) + return stbi__load_gif_main_outofmem(&g, out, delays); + out_size = layers * stride; + if (delays) { + *delays = (int*) stbi__malloc( layers * sizeof(int) ); + if (!*delays) + return stbi__load_gif_main_outofmem(&g, out, delays); + delays_size = layers * sizeof(int); + } + } + memcpy( out + ((layers - 1) * stride), u, stride ); + if (layers >= 2) { + two_back = out - 2 * stride; + } + + if (delays) { + (*delays)[layers - 1U] = g.delay; + } + } + } while (u != 0); + + // free temp buffer; + STBI_FREE(g.out); + STBI_FREE(g.history); + STBI_FREE(g.background); + + // do the final conversion after loading everything; + if (req_comp && req_comp != 4) + out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); + + *z = layers; + return out; + } else { + return stbi__errpuc("not GIF", "Image was not as a gif type."); + } +} + +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *u = 0; + stbi__gif g; + memset(&g, 0, sizeof(g)); + STBI_NOTUSED(ri); + + u = stbi__gif_load_next(s, &g, comp, req_comp, 0); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + if (u) { + *x = g.w; + *y = g.h; + + // moved conversion to after successful load so that the same + // can be done for multiple frames. + if (req_comp && req_comp != 4) + u = stbi__convert_format(u, 4, req_comp, g.w, g.h); + } else if (g.out) { + // if there was an error and we allocated an image buffer, free it! + STBI_FREE(g.out); + } + + // free buffers needed for multiple frame loading; + STBI_FREE(g.history); + STBI_FREE(g.background); + + return u; +} + +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) +{ + return stbi__gif_info_raw(s,x,y,comp); +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR loader +// originally by Nicolas Schulz +#ifndef STBI_NO_HDR +static int stbi__hdr_test_core(stbi__context *s, const char *signature) +{ + int i; + for (i=0; signature[i]; ++i) + if (stbi__get8(s) != signature[i]) + return 0; + stbi__rewind(s); + return 1; +} + +static int stbi__hdr_test(stbi__context* s) +{ + int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); + stbi__rewind(s); + if(!r) { + r = stbi__hdr_test_core(s, "#?RGBE\n"); + stbi__rewind(s); + } + return r; +} + +#define STBI__HDR_BUFLEN 1024 +static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) +{ + int len=0; + char c = '\0'; + + c = (char) stbi__get8(z); + + while (!stbi__at_eof(z) && c != '\n') { + buffer[len++] = c; + if (len == STBI__HDR_BUFLEN-1) { + // flush to end of line + while (!stbi__at_eof(z) && stbi__get8(z) != '\n') + ; + break; + } + c = (char) stbi__get8(z); + } + + buffer[len] = 0; + return buffer; +} + +static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) +{ + if ( input[3] != 0 ) { + float f1; + // Exponent + f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); + if (req_comp <= 2) + output[0] = (input[0] + input[1] + input[2]) * f1 / 3; + else { + output[0] = input[0] * f1; + output[1] = input[1] * f1; + output[2] = input[2] * f1; + } + if (req_comp == 2) output[1] = 1; + if (req_comp == 4) output[3] = 1; + } else { + switch (req_comp) { + case 4: output[3] = 1; /* fallthrough */ + case 3: output[0] = output[1] = output[2] = 0; + break; + case 2: output[1] = 1; /* fallthrough */ + case 1: output[0] = 0; + break; + } + } +} + +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int width, height; + stbi_uc *scanline; + float *hdr_data; + int len; + unsigned char count, value; + int i, j, k, c1,c2, z; + const char *headerToken; + STBI_NOTUSED(ri); + + // Check identifier + headerToken = stbi__hdr_gettoken(s,buffer); + if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) + return stbi__errpf("not HDR", "Corrupt HDR image"); + + // Parse header + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); + + // Parse width and height + // can't use sscanf() if we're not using stdio! + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + height = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + width = (int) strtol(token, NULL, 10); + + if (height > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + if (width > STBI_MAX_DIMENSIONS) return stbi__errpf("too large","Very large image (corrupt?)"); + + *x = width; + *y = height; + + if (comp) *comp = 3; + if (req_comp == 0) req_comp = 3; + + if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) + return stbi__errpf("too large", "HDR image is too large"); + + // Read data + hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); + if (!hdr_data) + return stbi__errpf("outofmem", "Out of memory"); + + // Load image data + // image data is stored as some number of sca + if ( width < 8 || width >= 32768) { + // Read flat data + for (j=0; j < height; ++j) { + for (i=0; i < width; ++i) { + stbi_uc rgbe[4]; + main_decode_loop: + stbi__getn(s, rgbe, 4); + stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); + } + } + } else { + // Read RLE-encoded data + scanline = NULL; + + for (j = 0; j < height; ++j) { + c1 = stbi__get8(s); + c2 = stbi__get8(s); + len = stbi__get8(s); + if (c1 != 2 || c2 != 2 || (len & 0x80)) { + // not run-length encoded, so we have to actually use THIS data as a decoded + // pixel (note this can't be a valid pixel--one of RGB must be >= 128) + stbi_uc rgbe[4]; + rgbe[0] = (stbi_uc) c1; + rgbe[1] = (stbi_uc) c2; + rgbe[2] = (stbi_uc) len; + rgbe[3] = (stbi_uc) stbi__get8(s); + stbi__hdr_convert(hdr_data, rgbe, req_comp); + i = 1; + j = 0; + STBI_FREE(scanline); + goto main_decode_loop; // yes, this makes no sense + } + len <<= 8; + len |= stbi__get8(s); + if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } + if (scanline == NULL) { + scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); + if (!scanline) { + STBI_FREE(hdr_data); + return stbi__errpf("outofmem", "Out of memory"); + } + } + + for (k = 0; k < 4; ++k) { + int nleft; + i = 0; + while ((nleft = width - i) > 0) { + count = stbi__get8(s); + if (count > 128) { + // Run + value = stbi__get8(s); + count -= 128; + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = value; + } else { + // Dump + if ((count == 0) || (count > nleft)) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = stbi__get8(s); + } + } + } + for (i=0; i < width; ++i) + stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); + } + if (scanline) + STBI_FREE(scanline); + } + + return hdr_data; +} + +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int dummy; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (stbi__hdr_test(s) == 0) { + stbi__rewind( s ); + return 0; + } + + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) { + stbi__rewind( s ); + return 0; + } + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *y = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *x = (int) strtol(token, NULL, 10); + *comp = 3; + return 1; +} +#endif // STBI_NO_HDR + +#ifndef STBI_NO_BMP +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) +{ + void *p; + stbi__bmp_data info; + + info.all_a = 255; + p = stbi__bmp_parse_header(s, &info); + if (p == NULL) { + stbi__rewind( s ); + return 0; + } + if (x) *x = s->img_x; + if (y) *y = s->img_y; + if (comp) { + if (info.bpp == 24 && info.ma == 0xff000000) + *comp = 3; + else + *comp = info.ma ? 4 : 3; + } + return 1; +} +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) +{ + int channelCount, dummy, depth; + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + *y = stbi__get32be(s); + *x = stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 8 && depth != 16) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 3) { + stbi__rewind( s ); + return 0; + } + *comp = 4; + return 1; +} + +static int stbi__psd_is16(stbi__context *s) +{ + int channelCount, depth; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + STBI_NOTUSED(stbi__get32be(s)); + STBI_NOTUSED(stbi__get32be(s)); + depth = stbi__get16be(s); + if (depth != 16) { + stbi__rewind( s ); + return 0; + } + return 1; +} +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) +{ + int act_comp=0,num_packets=0,chained,dummy; + stbi__pic_packet packets[10]; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { + stbi__rewind(s); + return 0; + } + + stbi__skip(s, 88); + + *x = stbi__get16be(s); + *y = stbi__get16be(s); + if (stbi__at_eof(s)) { + stbi__rewind( s); + return 0; + } + if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { + stbi__rewind( s ); + return 0; + } + + stbi__skip(s, 8); + + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return 0; + + packet = &packets[num_packets++]; + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + act_comp |= packet->channel; + + if (stbi__at_eof(s)) { + stbi__rewind( s ); + return 0; + } + if (packet->size != 8) { + stbi__rewind( s ); + return 0; + } + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); + + return 1; +} +#endif + +// ************************************************************************************************* +// Portable Gray Map and Portable Pixel Map loader +// by Ken Miller +// +// PGM: http://netpbm.sourceforge.net/doc/pgm.html +// PPM: http://netpbm.sourceforge.net/doc/ppm.html +// +// Known limitations: +// Does not support comments in the header section +// Does not support ASCII image data (formats P2 and P3) + +#ifndef STBI_NO_PNM + +static int stbi__pnm_test(stbi__context *s) +{ + char p, t; + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind( s ); + return 0; + } + return 1; +} + +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + STBI_NOTUSED(ri); + + ri->bits_per_channel = stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n); + if (ri->bits_per_channel == 0) + return 0; + + if (s->img_y > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + if (s->img_x > STBI_MAX_DIMENSIONS) return stbi__errpuc("too large","Very large image (corrupt?)"); + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + + if (!stbi__mad4sizes_valid(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0)) + return stbi__errpuc("too large", "PNM too large"); + + out = (stbi_uc *) stbi__malloc_mad4(s->img_n, s->img_x, s->img_y, ri->bits_per_channel / 8, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (!stbi__getn(s, out, s->img_n * s->img_x * s->img_y * (ri->bits_per_channel / 8))) { + STBI_FREE(out); + return stbi__errpuc("bad PNM", "PNM file truncated"); + } + + if (req_comp && req_comp != s->img_n) { + if (ri->bits_per_channel == 16) { + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, s->img_n, req_comp, s->img_x, s->img_y); + } else { + out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); + } + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + return out; +} + +static int stbi__pnm_isspace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; +} + +static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) +{ + for (;;) { + while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) + *c = (char) stbi__get8(s); + + if (stbi__at_eof(s) || *c != '#') + break; + + while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) + *c = (char) stbi__get8(s); + } +} + +static int stbi__pnm_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +static int stbi__pnm_getinteger(stbi__context *s, char *c) +{ + int value = 0; + + while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { + value = value*10 + (*c - '0'); + *c = (char) stbi__get8(s); + if((value > 214748364) || (value == 214748364 && *c > '7')) + return stbi__err("integer parse overflow", "Parsing an integer in the PPM header overflowed a 32-bit int"); + } + + return value; +} + +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) +{ + int maxv, dummy; + char c, p, t; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + stbi__rewind(s); + + // Get identifier + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind(s); + return 0; + } + + *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm + + c = (char) stbi__get8(s); + stbi__pnm_skip_whitespace(s, &c); + + *x = stbi__pnm_getinteger(s, &c); // read width + if(*x == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + *y = stbi__pnm_getinteger(s, &c); // read height + if (*y == 0) + return stbi__err("invalid width", "PPM image header had zero or overflowing width"); + stbi__pnm_skip_whitespace(s, &c); + + maxv = stbi__pnm_getinteger(s, &c); // read max value + if (maxv > 65535) + return stbi__err("max value > 65535", "PPM image supports only 8-bit and 16-bit images"); + else if (maxv > 255) + return 16; + else + return 8; +} + +static int stbi__pnm_is16(stbi__context *s) +{ + if (stbi__pnm_info(s, NULL, NULL, NULL) == 16) + return 1; + return 0; +} +#endif + +static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) +{ + #ifndef STBI_NO_JPEG + if (stbi__jpeg_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNG + if (stbi__png_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_GIF + if (stbi__gif_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_BMP + if (stbi__bmp_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PIC + if (stbi__pic_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_info(s, x, y, comp)) return 1; + #endif + + // test tga last because it's a crappy test! + #ifndef STBI_NO_TGA + if (stbi__tga_info(s, x, y, comp)) + return 1; + #endif + return stbi__err("unknown image type", "Image not of any known type, or corrupt"); +} + +static int stbi__is_16_main(stbi__context *s) +{ + #ifndef STBI_NO_PNG + if (stbi__png_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_is16(s)) return 1; + #endif + return 0; +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_info_from_file(f, x, y, comp); + fclose(f); + return result; +} + +STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__info_main(&s,x,y,comp); + fseek(f,pos,SEEK_SET); + return r; +} + +STBIDEF int stbi_is_16_bit(char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_is_16_bit_from_file(f); + fclose(f); + return result; +} + +STBIDEF int stbi_is_16_bit_from_file(FILE *f) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__is_16_main(&s); + fseek(f,pos,SEEK_SET); + return r; +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__is_16_main(&s); +} + +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__is_16_main(&s); +} + +#endif // STB_IMAGE_IMPLEMENTATION + +/* + revision history: + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug + 1-bit BMP + *_is_16_bit api + avoid warnings + 2.16 (2017-07-23) all functions have 16-bit variants; + STBI_NO_STDIO works again; + compilation fixes; + fix rounding in unpremultiply; + optimize vertical flip; + disable raw_len validation; + documentation fixes + 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; + warning fixes; disable run-time SSE detection on gcc; + uniform handling of optional "return" values; + thread-safe initialization of zlib tables + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) allocate large structures on the stack + remove white matting for transparent PSD + fix reported channel count for PNG & BMP + re-enable SSE2 in non-gcc 64-bit + support RGB-formatted JPEG + read 16-bit PNGs (only as 8-bit) + 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED + 2.09 (2016-01-16) allow comments in PNM files + 16-bit-per-pixel TGA (not bit-per-component) + info() for TGA could break due to .hdr handling + info() for BMP to shares code instead of sloppy parse + can use STBI_REALLOC_SIZED if allocator doesn't support realloc + code cleanup + 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA + 2.07 (2015-09-13) fix compiler warnings + partial animated GIF support + limited 16-bpc PSD support + #ifdef unused functions + bug with < 92 byte PIC,PNM,HDR,TGA + 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value + 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning + 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit + 2.03 (2015-04-12) extra corruption checking (mmozeiko) + stbi_set_flip_vertically_on_load (nguillemot) + fix NEON support; fix mingw support + 2.02 (2015-01-19) fix incorrect assert, fix warning + 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 + 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG + 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) + progressive JPEG (stb) + PGM/PPM support (Ken Miller) + STBI_MALLOC,STBI_REALLOC,STBI_FREE + GIF bugfix -- seemingly never worked + STBI_NO_*, STBI_ONLY_* + 1.48 (2014-12-14) fix incorrectly-named assert() + 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) + optimize PNG (ryg) + fix bug in interlaced PNG with user-specified channel count (stb) + 1.46 (2014-08-26) + fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG + 1.45 (2014-08-16) + fix MSVC-ARM internal compiler error by wrapping malloc + 1.44 (2014-08-07) + various warning fixes from Ronny Chevalier + 1.43 (2014-07-15) + fix MSVC-only compiler problem in code changed in 1.42 + 1.42 (2014-07-09) + don't define _CRT_SECURE_NO_WARNINGS (affects user code) + fixes to stbi__cleanup_jpeg path + added STBI_ASSERT to avoid requiring assert.h + 1.41 (2014-06-25) + fix search&replace from 1.36 that messed up comments/error messages + 1.40 (2014-06-22) + fix gcc struct-initialization warning + 1.39 (2014-06-15) + fix to TGA optimization when req_comp != number of components in TGA; + fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) + add support for BMP version 5 (more ignored fields) + 1.38 (2014-06-06) + suppress MSVC warnings on integer casts truncating values + fix accidental rename of 'skip' field of I/O + 1.37 (2014-06-04) + remove duplicate typedef + 1.36 (2014-06-03) + convert to header file single-file library + if de-iphone isn't set, load iphone images color-swapped instead of returning NULL + 1.35 (2014-05-27) + various warnings + fix broken STBI_SIMD path + fix bug where stbi_load_from_file no longer left file pointer in correct place + fix broken non-easy path for 32-bit BMP (possibly never used) + TGA optimization by Arseny Kapoulkine + 1.34 (unknown) + use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case + 1.33 (2011-07-14) + make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements + 1.32 (2011-07-13) + support for "info" function for all supported filetypes (SpartanJ) + 1.31 (2011-06-20) + a few more leak fixes, bug in PNG handling (SpartanJ) + 1.30 (2011-06-11) + added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) + removed deprecated format-specific test/load functions + removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway + error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) + fix inefficiency in decoding 32-bit BMP (David Woo) + 1.29 (2010-08-16) + various warning fixes from Aurelien Pocheville + 1.28 (2010-08-01) + fix bug in GIF palette transparency (SpartanJ) + 1.27 (2010-08-01) + cast-to-stbi_uc to fix warnings + 1.26 (2010-07-24) + fix bug in file buffering for PNG reported by SpartanJ + 1.25 (2010-07-17) + refix trans_data warning (Won Chun) + 1.24 (2010-07-12) + perf improvements reading from files on platforms with lock-heavy fgetc() + minor perf improvements for jpeg + deprecated type-specific functions so we'll get feedback if they're needed + attempt to fix trans_data warning (Won Chun) + 1.23 fixed bug in iPhone support + 1.22 (2010-07-10) + removed image *writing* support + stbi_info support from Jetro Lauha + GIF support from Jean-Marc Lienher + iPhone PNG-extensions from James Brown + warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) + 1.21 fix use of 'stbi_uc' in header (reported by jon blow) + 1.20 added support for Softimage PIC, by Tom Seddon + 1.19 bug in interlaced PNG corruption check (found by ryg) + 1.18 (2008-08-02) + fix a threading bug (local mutable static) + 1.17 support interlaced PNG + 1.16 major bugfix - stbi__convert_format converted one too many pixels + 1.15 initialize some fields for thread safety + 1.14 fix threadsafe conversion bug + header-file-only version (#define STBI_HEADER_FILE_ONLY before including) + 1.13 threadsafe + 1.12 const qualifiers in the API + 1.11 Support installable IDCT, colorspace conversion routines + 1.10 Fixes for 64-bit (don't use "unsigned long") + optimized upsampling by Fabian "ryg" Giesen + 1.09 Fix format-conversion for PSD code (bad global variables!) + 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz + 1.07 attempt to fix C++ warning/errors again + 1.06 attempt to fix C++ warning/errors again + 1.05 fix TGA loading to return correct *comp and use good luminance calc + 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free + 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR + 1.02 support for (subset of) HDR files, float interface for preferred access to them + 1.01 fix bug: possible bug in handling right-side up bmps... not sure + fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all + 1.00 interface to zlib that skips zlib header + 0.99 correct handling of alpha in palette + 0.98 TGA loader by lonesock; dynamically add loaders (untested) + 0.97 jpeg errors on too large a file; also catch another malloc failure + 0.96 fix detection of invalid v value - particleman@mollyrocket forum + 0.95 during header scan, seek to markers in case of padding + 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same + 0.93 handle jpegtran output; verbose errors + 0.92 read 4,8,16,24,32-bit BMP files of several formats + 0.91 output 24-bit Windows 3.0 BMP files + 0.90 fix a few more warnings; bump version number to approach 1.0 + 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd + 0.60 fix compiling as c++ + 0.59 fix warnings: merge Dave Moore's -Wall fixes + 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian + 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available + 0.56 fix bug: zlib uncompressed mode len vs. nlen + 0.55 fix bug: restart_interval not initialized to 0 + 0.54 allow NULL for 'int *comp' + 0.53 fix bug in png 3->4; speedup png decoding + 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments + 0.51 obey req_comp requests, 1-component jpegs return as 1-component, + on 'test' only check type, not whether we support this variant + 0.50 (2006-11-19) + first released version +*/ + + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/ggml/examples/stb_image_write.h b/ggml/examples/stb_image_write.h new file mode 100644 index 0000000..e4b32ed --- /dev/null +++ b/ggml/examples/stb_image_write.h @@ -0,0 +1,1724 @@ +/* stb_image_write - v1.16 - public domain - http://nothings.org/stb + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function + for PNG compression (instead of the builtin one), it must have the following signature: + unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); + The returned data will be freed with STBIW_FREE() (free() by default), + so it must be heap allocated with STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); + int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically + + There are also five equivalent functions that use an arbitrary write function. You are + expected to open/close your file-equivalent before and after calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); + int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE + int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression + int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + Andrew Kensler + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +STBIWDEF int stbi_write_tga_with_rle; +STBIWDEF int stbi_write_png_compression_level; +STBIWDEF int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); + +#ifdef STBIW_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif//INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p,newsz) realloc(p,newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) +#endif + + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) +#endif + + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +static int stbi__flip_vertically_on_write = 0; + +STBIWDEF void stbi_flip_vertically_on_write(int flag) +{ + stbi__flip_vertically_on_write = flag; +} + +typedef struct +{ + stbi_write_func *func; + void *context; + unsigned char buffer[64]; + int buf_used; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) +{ + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) +{ + fwrite(data,1,size,(FILE*) context); +} + +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, (int) bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_WIN32) && defined(STBIW_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename)/sizeof(*wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode)/sizeof(*wMode))) + return 0; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, const char *filename) +{ + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) +{ + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) +{ + while (*fmt) { + switch (*fmt++) { + case ' ': break; + case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context,&x,1); + break; } + case '2': { int x = va_arg(v,int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x>>8); + s->func(s->context,b,2); + break; } + case '4': { stbiw_uint32 x = va_arg(v,int); + unsigned char b[4]; + b[0]=STBIW_UCHAR(x); + b[1]=STBIW_UCHAR(x>>8); + b[2]=STBIW_UCHAR(x>>16); + b[3]=STBIW_UCHAR(x>>24); + s->func(s->context,b,4); + break; } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) +{ + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__write_flush(stbi__write_context *s) +{ + if (s->buf_used) { + s->func(s->context, &s->buffer, s->buf_used); + s->buf_used = 0; + } +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) +{ + s->func(s->context, &c, 1); +} + +static void stbiw__write1(stbi__write_context *s, unsigned char a) +{ + if ((size_t)s->buf_used + 1 > sizeof(s->buffer)) + stbiw__write_flush(s); + s->buffer[s->buf_used++] = a; +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) +{ + int n; + if ((size_t)s->buf_used + 3 > sizeof(s->buffer)) + stbiw__write_flush(s); + n = s->buf_used; + s->buf_used = n+3; + s->buffer[n+0] = a; + s->buffer[n+1] = b; + s->buffer[n+2] = c; +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) +{ + unsigned char bg[3] = { 255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + stbiw__write1(s, d[comp - 1]); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + stbiw__write1(s, d[0]); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + stbiw__write1(s, d[comp - 1]); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) +{ + stbiw_uint32 zero = 0; + int i,j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) { + j_end = -1; j = y-1; + } else { + j_end = y; j = 0; + } + + for (; j != j_end; j += vdir) { + for (i=0; i < x; ++i) { + unsigned char *d = (unsigned char *) data + (j*x+i)*comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + stbiw__write_flush(s); + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) +{ + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) +{ + if (comp != 4) { + // write RGB bitmap + int pad = (-x*3) & 3; + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, + "11 4 22 4" "4 44 22 444444", + 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header + 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header + } else { + // RGBA bitmaps need a v4 header + // use BI_BITFIELDS mode with 32bpp and alpha mask + // (straight BI_RGB with alpha mask doesn't work in most readers) + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *)data,1,0, + "11 4 22 4" "4 44 22 444444 4444 4 444 444 444 444", + 'B', 'M', 14+108+x*y*4, 0, 0, 14+108, // file header + 108, x,y, 1,32, 3,0,0,0,0,0, 0xff0000,0xff00,0xff,0xff000000u, 0, 0,0,0, 0,0,0, 0,0,0, 0,0,0); // bitmap V4 header + } +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //!STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) +{ + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp-1 : comp; + int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i,j,k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y-1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *) data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + stbiw__write1(s, header); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + stbiw__write1(s, header); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + stbiw__write_flush(s); + } + return 1; +} + +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_tga_core(&s, x, y, comp, (void *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +#ifndef STBI_WRITE_NO_STDIO + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) +{ + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) +{ + unsigned char lengthbyte = STBIW_UCHAR(length+128); + STBIW_ASSERT(length+128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) +{ + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) +{ + unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width&0xff00)>>8; + scanlineheader[3] = (width&0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x=0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c,r; + /* encode into scratch buffer */ + for (x=0; x < width; x++) { + switch(ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width*0] = rgbe[0]; + scratch[x + width*1] = rgbe[1]; + scratch[x + width*2] = rgbe[2]; + scratch[x + width*3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c=0; c < 4; c++) { + unsigned char *comp = &scratch[width*c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r+2 < width) { + if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) + break; + ++r; + } + if (r+2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r-x; + if (len > 128) len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r-x; + if (len > 127) len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) +{ + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full output scanline. + unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); + int i, len; + char buffer[128]; + char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header)-1); + +#ifdef __STDC_LIB_EXT1__ + len = sprintf_s(buffer, sizeof(buffer), "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = sprintf(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#endif + s->func(s->context, buffer, len); + + for(i=0; i < y; i++) + stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *) data); +} + +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() +#define stbiw__sbraw(a) ((int *) (void *) (a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) +#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) +{ + int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; + void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) ((int *) p)[1] = 0; + *arr = (void *) ((int *) p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) +{ + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) +{ + int res=0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) +{ + int i; + for (i=0; i < limit && i < 258; ++i) + if (a[i] != b[i]) break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) +{ + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code,codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) +#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) +{ +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; + static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; + static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; + static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; + unsigned int bitbuf=0; + int i,j, bitcount=0; + unsigned char *out = NULL; + unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(unsigned char**)); + if (hash_table == NULL) + return NULL; + if (quality < 5) quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1,1); // BFINAL = 1 + stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman + + for (i=0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i=0; + while (i < data_len-3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); + if (d >= best) { best=d; bestloc=hlist[j]; } + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h],data+i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal + h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32767) { + int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int) (data+i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j=0; best > lengthc[j+1]-1; ++j); + stbiw__zlib_huff(j+257); + if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j=0; d > distc[j+1]-1; ++j); + stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); + if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (;i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0,1); + + for (i=0; i < stbiw__ZHASH; ++i) + (void) stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + // store uncompressed instead if compression was worse + if (stbiw__sbn(out) > data_len + 2 + ((data_len+32766)/32767)*5) { + stbiw__sbn(out) = 2; // truncate to DEFLATE 32K window and FLEVEL = 1 + for (j = 0; j < data_len;) { + int blocklen = data_len - j; + if (blocklen > 32767) blocklen = 32767; + stbiw__sbpush(out, data_len - j == blocklen); // BFINAL = ?, BTYPE = 0 -- no compression + stbiw__sbpush(out, STBIW_UCHAR(blocklen)); // LEN + stbiw__sbpush(out, STBIW_UCHAR(blocklen >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(~blocklen)); // NLEN + stbiw__sbpush(out, STBIW_UCHAR(~blocklen >> 8)); + memcpy(out+stbiw__sbn(out), data+j, blocklen); + stbiw__sbn(out) += blocklen; + j += blocklen; + } + } + + { + // compute adler32 on input + unsigned int s1=1, s2=0; + int blocklen = (int) (data_len % 5552); + j=0; + while (j < data_len) { + for (i=0; i < blocklen; ++i) { s1 += data[j+i]; s2 += s1; } + s1 %= 65521; s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *) stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) +{ +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + + unsigned int crc = ~0u; + int i; + for (i=0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) +#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); +#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) +{ + unsigned int crc = stbiw__crc32(*data - len - 4, len+4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) +{ + int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); + if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); + if (pb <= pc) return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) +{ + static int mapping[] = { 0,1,2,3,4 }; + static int firstmap[] = { 0,1,0,5,6 }; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); + int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type==0) { + memcpy(line_buffer, z, width*n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: line_buffer[i] = z[i]; break; + case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; + case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; + case 5: line_buffer[i] = z[i]; break; + case 6: line_buffer[i] = z[i]; break; + } + } + switch (type) { + case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; + case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; + case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; + case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; + case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) +{ + int force_filter = stbi_write_force_png_filter; + int ctype[5] = { -1, 0, 4, 2, 6 }; + unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; + unsigned char *out,*o, *filt, *zlib; + signed char *line_buffer; + int j,zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; + line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } + for (j=0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the better. + est = 0; + for (i = 0; i < x*n; ++i) { + est += abs((signed char) line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and line_buffer contains the data + filt[j*(x*n+1)] = (unsigned char) filter_type; + STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); + if (!out) return 0; + *out_len = 8 + 12+13 + 12+zlen + 12; + + o=out; + STBIW_MEMMOVE(o,sig,8); o+= 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o,13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o,0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o,0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) +{ + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { STBIW_FREE(png); return 0; } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) +{ + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, + 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while(bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if(c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val-1 : val; + bits[1] = 1; + while(tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if(end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for(i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i]==0 && i<=end0pos; ++i) { + } + nrzeroes = i-startpos; + if ( nrzeroes >= 16 ) { + int lng = nrzeroes>>4; + int nrmarker; + for (nrmarker=1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if(end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; + static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, + 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, + 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, + 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, + 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, + 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, + 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; + static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, + 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, + 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, + 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; + static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; + static const unsigned short YAC_HT[256][2] = { + {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const unsigned short UVAC_HT[256][2] = { + {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, + 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; + static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, + 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; + static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; + + int row, col, i, k, subsample; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if(!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + subsample = quality <= 90 ? 1 : 0; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for(i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i]*quality+50)/100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); + uvti = (UVQT[i]*quality+50)/100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); + } + + for(row = 0, k = 0; row < 8; ++row) { + for(col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; + static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; + const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), + 3,1,(unsigned char)(subsample?0x22:0x11),0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; + s->func(s->context, (void*)head0, sizeof(head0)); + s->func(s->context, (void*)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void*)head1, sizeof(head1)); + s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); + s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); + s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); + s->func(s->context, (void*)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + int DCY=0, DCU=0, DCV=0; + int bitBuf=0, bitCnt=0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + const unsigned char *dataR = (const unsigned char *)data; + const unsigned char *dataG = dataR + ofsG; + const unsigned char *dataB = dataR + ofsB; + int x, y, pos; + if(subsample) { + for(y = 0; y < height; y += 16) { + for(x = 0; x < width; x += 16) { + float Y[256], U[256], V[256]; + for(row = y, pos = 0; row < y+16; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+16; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+0, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+8, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+128, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y+136, 16, fdtbl_Y, DCY, YDC_HT, YAC_HT); + + // subsample U,V + { + float subU[64], subV[64]; + int yy, xx; + for(yy = 0, pos = 0; yy < 8; ++yy) { + for(xx = 0; xx < 8; ++xx, ++pos) { + int j = yy*32+xx*2; + subU[pos] = (U[j+0] + U[j+1] + U[j+16] + U[j+17]) * 0.25f; + subV[pos] = (V[j+0] + V[j+1] + V[j+16] + V[j+17]) * 0.25f; + } + } + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subU, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, subV, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + } else { + for(y = 0; y < height; y += 8) { + for(x = 0; x < width; x += 8) { + float Y[64], U[64], V[64]; + for(row = y, pos = 0; row < y+8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+8; ++col, ++pos) { + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + float r = dataR[p], g = dataG[p], b = dataB[p]; + Y[pos]= +0.29900f*r + 0.58700f*g + 0.11400f*b - 128; + U[pos]= -0.16874f*r - 0.33126f*g + 0.50000f*b; + V[pos]= +0.50000f*r - 0.41869f*g - 0.08131f*b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, Y, 8, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, U, 8, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, V, 8, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); +} + + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s = { 0 }; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.16 (2021-07-11) + make Deflate code emit uncompressed blocks when it would otherwise expand + support writing BMPs with alpha channel + 1.15 (2020-07-13) unknown + 1.14 (2020-02-02) updated JPEG writer to downsample chroma channels + 1.13 + 1.12 + 1.11 (2019-08-11) + + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter + 1.07 (2017-07-24) + doc fix + 1.06 (2017-07-23) + writing JPEG (using Jon Olick's code) + 1.05 ??? + 1.04 (2017-03-03) + monochrome BMP expansion + 1.03 ??? + 1.02 (2016-04-02) + avoid allocating large structures on the stack + 1.01 (2016-01-16) + STBIW_REALLOC_SIZED: support allocators with no realloc support + avoid race-condition in crc initialization + minor compile issues + 1.00 (2015-09-14) + installable file IO function + 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/ggml/examples/test-cmake/CMakeLists.txt b/ggml/examples/test-cmake/CMakeLists.txt new file mode 100644 index 0000000..d6bc0cc --- /dev/null +++ b/ggml/examples/test-cmake/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.14) +project(ggml-simple) + +set(CMAKE_CXX_STANDARD 17) + +find_package(ggml CONFIG REQUIRED) + +set(TEST_TARGET test-cmake) +add_executable(test-cmake test-cmake.cpp) +target_link_libraries(test-cmake PRIVATE ggml::ggml) diff --git a/ggml/examples/test-cmake/README.md b/ggml/examples/test-cmake/README.md new file mode 100644 index 0000000..8f580d1 --- /dev/null +++ b/ggml/examples/test-cmake/README.md @@ -0,0 +1,3 @@ +## cmake-test + +This directory can be built as a separate project with an installed ggml. diff --git a/ggml/examples/test-cmake/test-cmake.cpp b/ggml/examples/test-cmake/test-cmake.cpp new file mode 100644 index 0000000..029c889 --- /dev/null +++ b/ggml/examples/test-cmake/test-cmake.cpp @@ -0,0 +1,6 @@ +#include "ggml-backend.h" + +int main(void) { + ggml_backend_load_all(); + return 0; +} diff --git a/ggml/examples/yolo/CMakeLists.txt b/ggml/examples/yolo/CMakeLists.txt new file mode 100644 index 0000000..0c068ce --- /dev/null +++ b/ggml/examples/yolo/CMakeLists.txt @@ -0,0 +1,6 @@ +# +# yolov3-tiny + +set(TEST_TARGET yolov3-tiny) +add_executable(${TEST_TARGET} yolov3-tiny.cpp yolo-image.cpp) +target_link_libraries(${TEST_TARGET} PRIVATE ggml common) diff --git a/ggml/examples/yolo/README.md b/ggml/examples/yolo/README.md new file mode 100644 index 0000000..d2ced38 --- /dev/null +++ b/ggml/examples/yolo/README.md @@ -0,0 +1,59 @@ +This example shows how to implement YOLO object detection with ggml using pretrained model. + +# YOLOv3-tiny + +Download the model weights: + +```bash +$ wget https://pjreddie.com/media/files/yolov3-tiny.weights +$ sha1sum yolov3-tiny.weights +40f3c11883bef62fd850213bc14266632ed4414f yolov3-tiny.weights +``` + +Convert the weights to GGUF format: + +```bash +$ ./convert-yolov3-tiny.py yolov3-tiny.weights +yolov3-tiny.weights converted to yolov3-tiny.gguf +``` + +Alternatively, you can download the converted model from [HuggingFace](https://huggingface.co/rgerganov/yolo-gguf/resolve/main/yolov3-tiny.gguf) + +Object detection: + +```bash +$ wget https://raw.githubusercontent.com/pjreddie/darknet/master/data/dog.jpg +$ ./yolov3-tiny -m yolov3-tiny.gguf -i dog.jpg +load_model: using CUDA backend +ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no +ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no +ggml_cuda_init: found 1 CUDA devices: + Device 0: NVIDIA T1200 Laptop GPU, compute capability 7.5, VMM: yes +Layer 0 output shape: 416 x 416 x 16 x 1 +Layer 1 output shape: 208 x 208 x 16 x 1 +Layer 2 output shape: 208 x 208 x 32 x 1 +Layer 3 output shape: 104 x 104 x 32 x 1 +Layer 4 output shape: 104 x 104 x 64 x 1 +Layer 5 output shape: 52 x 52 x 64 x 1 +Layer 6 output shape: 52 x 52 x 128 x 1 +Layer 7 output shape: 26 x 26 x 128 x 1 +Layer 8 output shape: 26 x 26 x 256 x 1 +Layer 9 output shape: 13 x 13 x 256 x 1 +Layer 10 output shape: 13 x 13 x 512 x 1 +Layer 11 output shape: 13 x 13 x 512 x 1 +Layer 12 output shape: 13 x 13 x 1024 x 1 +Layer 13 output shape: 13 x 13 x 256 x 1 +Layer 14 output shape: 13 x 13 x 512 x 1 +Layer 15 output shape: 13 x 13 x 255 x 1 +Layer 18 output shape: 13 x 13 x 128 x 1 +Layer 19 output shape: 26 x 26 x 128 x 1 +Layer 20 output shape: 26 x 26 x 384 x 1 +Layer 21 output shape: 26 x 26 x 256 x 1 +Layer 22 output shape: 26 x 26 x 255 x 1 +dog: 57% +car: 52% +truck: 56% +car: 62% +bicycle: 59% +Detected objects saved in 'predictions.jpg' (time: 0.057000 sec.) +``` \ No newline at end of file diff --git a/ggml/examples/yolo/convert-yolov3-tiny.py b/ggml/examples/yolo/convert-yolov3-tiny.py new file mode 100755 index 0000000..4c47991 --- /dev/null +++ b/ggml/examples/yolo/convert-yolov3-tiny.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import sys +import gguf +import numpy as np + +def save_conv2d_layer(f, gguf_writer, prefix, inp_c, filters, size, batch_normalize=True): + biases = np.fromfile(f, dtype=np.float32, count=filters) + gguf_writer.add_tensor(prefix + "_biases", biases, raw_shape=(1, filters, 1, 1)) + + if batch_normalize: + scales = np.fromfile(f, dtype=np.float32, count=filters) + gguf_writer.add_tensor(prefix + "_scales", scales, raw_shape=(1, filters, 1, 1)) + rolling_mean = np.fromfile(f, dtype=np.float32, count=filters) + gguf_writer.add_tensor(prefix + "_rolling_mean", rolling_mean, raw_shape=(1, filters, 1, 1)) + rolling_variance = np.fromfile(f, dtype=np.float32, count=filters) + gguf_writer.add_tensor(prefix + "_rolling_variance", rolling_variance, raw_shape=(1, filters, 1, 1)) + + weights_count = filters * inp_c * size * size + l0_weights = np.fromfile(f, dtype=np.float32, count=weights_count) + ## ggml doesn't support f32 convolution yet, use f16 instead + l0_weights = l0_weights.astype(np.float16) + gguf_writer.add_tensor(prefix + "_weights", l0_weights, raw_shape=(filters, inp_c, size, size)) + + +if __name__ == '__main__': + if len(sys.argv) != 2: + print("Usage: %s " % sys.argv[0]) + sys.exit(1) + outfile = 'yolov3-tiny.gguf' + gguf_writer = gguf.GGUFWriter(outfile, 'yolov3-tiny') + + f = open(sys.argv[1], 'rb') + f.read(20) # skip header + save_conv2d_layer(f, gguf_writer, "l0", 3, 16, 3) + save_conv2d_layer(f, gguf_writer, "l1", 16, 32, 3) + save_conv2d_layer(f, gguf_writer, "l2", 32, 64, 3) + save_conv2d_layer(f, gguf_writer, "l3", 64, 128, 3) + save_conv2d_layer(f, gguf_writer, "l4", 128, 256, 3) + save_conv2d_layer(f, gguf_writer, "l5", 256, 512, 3) + save_conv2d_layer(f, gguf_writer, "l6", 512, 1024, 3) + save_conv2d_layer(f, gguf_writer, "l7", 1024, 256, 1) + save_conv2d_layer(f, gguf_writer, "l8", 256, 512, 3) + save_conv2d_layer(f, gguf_writer, "l9", 512, 255, 1, batch_normalize=False) + save_conv2d_layer(f, gguf_writer, "l10", 256, 128, 1) + save_conv2d_layer(f, gguf_writer, "l11", 384, 256, 3) + save_conv2d_layer(f, gguf_writer, "l12", 256, 255, 1, batch_normalize=False) + f.close() + + gguf_writer.write_header_to_file() + gguf_writer.write_kv_data_to_file() + gguf_writer.write_tensors_to_file() + gguf_writer.close() + print("{} converted to {}".format(sys.argv[1], outfile)) diff --git a/ggml/examples/yolo/data/coco.names b/ggml/examples/yolo/data/coco.names new file mode 100644 index 0000000..ca76c80 --- /dev/null +++ b/ggml/examples/yolo/data/coco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/ggml/examples/yolo/data/labels/100_0.png b/ggml/examples/yolo/data/labels/100_0.png new file mode 100644 index 0000000000000000000000000000000000000000..77878ecc6f6b416896bbf2b8986aa55ccd2f6342 GIT binary patch literal 320 zcmeAS@N?(olHy`uVBq!ia0vp^96-zsA{f#?O#TU^SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Q-UR@G-fAyZEm#}JO|$v^m6&UOF)pMKzw#>3K$42EXi_x7IUPcmtD zS~@NJZFVdQ&MBb@0C~My`v3p{ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/100_1.png b/ggml/examples/yolo/data/labels/100_1.png new file mode 100644 index 0000000000000000000000000000000000000000..86181c0b479e96a019a8ed316955e0d915b58a15 GIT binary patch literal 377 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4}5r*_XAID#R(foNHMqa%6MchbYM#*Z=;H zV=;ZYTZ_5bz`R-L%D3egoz~p?91z>~%D$_DVT0MV^41sT*MK&wmbgZgq$HN4S|t~y z0x1R~14A=iLqlBy(+~r5D`O)oV+&m%!@%I$Jx68~4Y~O#nQ4`{H7sxb3Gpt_hMdHt X)EtJC>o@a2fz06P>gTe~DWM4f=m&g{ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/100_2.png b/ggml/examples/yolo/data/labels/100_2.png new file mode 100644 index 0000000000000000000000000000000000000000..5d430467d64659407898d3373b718f548e6e0fb2 GIT binary patch literal 451 zcmeAS@N?(olHy`uVBq!ia0vp^!a%IR0VEi%UX=lHSkfJR9T^xl_H+M9WCckkdj$D1 zFjT2AFf_Ckr_Wm>bfjssS zPhVH|2i&ZD3i|6pT|NMXc6hothG?8m{?X5{y{WnRK656U%Ere$JUr*w7vESvcdqdF zzs`cus=QAOR>(LDWv4VST+=MrcvWFvaphKSWqqGF>$RAfKc`JtnJtl+w7jt=sbg=W z!N!*k$5cCRRLzjMz*e^6_qu~eUP($ES&(F&m}J}#R@2y#+t4|e#b}ztk(!5j6WO!& zF$%p(XxLpTHsNUW#=?ol%ua9S+h57&*~sz#-{P{a$CR+(-9vfTRyFgBb9ftc$ z&rPhC5ZZl4!oi_ZZ`G0eHf$0df17m#&o(gRdI$+0RH{jwTKdq2=gpadPX9j&A1{@gHqROC-tdHC=5d4hTeSyNgSYV_DA|6si| zyM4lirq>Cqn>Gmw1!{8Z z^gcyZuAUI1Bo&64N1h(_vDtbV=y}x=*NBpo#FA92t_x%s7(DxFk&B`sH$NpatrE9}=}Q_efbP0 Hl+XkKL0`32 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/100_4.png b/ggml/examples/yolo/data/labels/100_4.png new file mode 100644 index 0000000000000000000000000000000000000000..d22f891c21e795e4a96870f4b50e560fa2fd8b83 GIT binary patch literal 577 zcmeAS@N?(olHy`uVBq!ia0vp^azJd#0VEiT4OqH?6id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BRTQ^!u(i0|TR(r;B5V#p&c9{hZfInGZS{OlIufV$Lk^qTh=@@8xdO zvKpO7>n7VJOseUIA)~(5`Ol&zv#rJdWCmZog7m&Ph1i|Ad|er zq56E6%3cAT*jAYp(>yNKCcHd);?hZvOQi`fZBJZU>TyXl;pOQQmsmY6%}scjeB#nZ zk4xSMFKJI$S~_t_fAiAf2`Tdp6+QnyVJmdmuefA&0Mk|RGpk!q&r5foc#r3qp>ETu zxT#LPlj?5?)P2iV`2IU3;!?x$?}=_1MjJXm2sNp5xg=@1`|75BTXE_Cr3tB}_ZKX` zQg8h5Wbeuk*3)u6lug@iWIlUN>FL@lJk7eoyM+js%nXAL`h0w zNvc(HQ7VvPFfuSS(={~IH82e^Ft;)`wlXl&1u_f_p05dOM$wR)pOTqYiCe?8kMHck e(FU|3Cow5Chau(q%{)+IVeoYIb6Mw<&;$TaeB59F literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/100_5.png b/ggml/examples/yolo/data/labels/100_5.png new file mode 100644 index 0000000000000000000000000000000000000000..7f65183fda65ae392f0a4dc4b4f38e7d82f074c5 GIT binary patch literal 631 zcmeAS@N?(olHy`uVBq!ia0vp^%0TSE0VEiNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5`mMvaIb1_s6)PZ!4!kK@Td`ekF{dDz}Ni2 zKPS3P5z=m-^Zv^99am~}9wjy~t(&ADarMpgbO)xc@Qo`X4;^Y?YROApG4uG52DUHN z7p}x4q#t0owIZu7a$$GcRQ z&|1hgd%S2cWNw@Fzx$!nrGqmKwN9LrnyCNBaFWep+g-E1TfNBIW%&6)cD9IU zj`7B)PwiuyI8?uh7fz9m7jw(}WVEp>a{7G7?zlNm&x1mfA#9aalK1UND}YggTe~DWM4fhM4dX literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/100_6.png b/ggml/examples/yolo/data/labels/100_6.png new file mode 100644 index 0000000000000000000000000000000000000000..18eb1730652e1eea0541646d21a56d93d1c58354 GIT binary patch literal 697 zcmeAS@N?(olHy`uVBq!ia0vp^T0rc@0VEh+#LT=4q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|khkrLtVqjog=jq}Y;&D9rN5625B@g5Gg*9wF!JC8W_*ddU59ZkEwO@7o1>V&dKcmpkcf6U;|G@`s0vWhtD*yX%%X$O*z04 zyirT{@S;hLxu#-Ey(Km|vf$v8?;95Umg)6< zY0Wnyg1IOrH8IdiR86EyXsvyT!PH%jt~=rvsWqBPc-0p>xprL$IvrnGvUGaltn&+& zuW>3_x}ibrkVi}O#i@P`TsIOmYMa(=5z`WAU-(5|dDbPN6_sgn~R$NNsnHqobu9d9&hJXL-rDI=~t!R*MyX5-9YyH7PU*&gQN-0^ozhZ%s`-PBw z&yRWN)dfssFj?9=Lo-_{q@_^3>y+@C8D8y3#YVP|9T;Wt^PpdzJG6~y9BM+|IvOQ)4%W2Y_vJc*z_cquS)*EkZD9Y@RMf;rV8o(Yn?Kydt#A_{2FK3tro|=CtXWMtd*OmWm_dl%{h$R_?j_GQR#7t4IEI zMZ=o+OXWCrPf&U`v0PWdr`TieEemOtJscf>8(D?&SeEJXGB0E|6lypqlPxFu)cJu` zxLJ_B^v?fBPEHnG?xf`~{mSjKCB--OQyREemL1!mbCFAe$2&J>r$@S_zzL80H4EbQ z8gjTCx#axzRk9p|m}`MY{&s$bx=T}2GB&U`xLx{s@$-+XF3mwqX^aiBCIZ*1*qPpn zcx2fcAG<43A$(Al<(I~XJhcacOme>@k8IxA=wQryiqoRaL6bL%^9IYa#Rm?4ot!7Y z5X~?{J=SuAlrDpX`2Xv%S^q?$@)x#D;my2Svx>9kaodqHA6B=o_cAZHCDq9sWGMIe z+wx^~TulcnU#k2Lr$hepIi@M9P3U`My{&QORmprtC#eTF+aw-V3Oo=v&XB$4w<5!N z|4sP|6#V;(Pe+yC@!VRq?-56KT-Qp6sWo@@eQDfy&sKKcyER)1(z*}Z*0HS7!C8<`)MX5lF!N|bSOxMs**T6Kyz}(8%*vi;U z7sxO$$W#slrY9r~x%nxXX_dG&NYuT&15Rc@8*&npQgawmuHVc9C2j^!S3j3^P6ildaLnr9a(~x#43epn-o3MW z-OzJ1>V3S34YSd|W!zj-{~cZ@%f%q)>=_}I=Y0Zbf@+CtL`h0wNvc(HQ7VvPFfuSS z(={~IH82e^Ft;)`vNE#J1u_f_q)Hs;qG-s?PsvQH#H}G)^QR)%*+3g|5|dJM7*ZG9 Si<<)!V(@hJb6Mw<&;$V2eOf~R literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/101_1.png b/ggml/examples/yolo/data/labels/101_1.png new file mode 100644 index 0000000000000000000000000000000000000000..5a39331d337f3744517717df327e139be37875d0 GIT binary patch literal 388 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4Uo}jfzrd zUMD_?`u+TSky+bbg?UEK?`jn|(oQgax@UBtfqBCBj>d)6wi0icKh=FSC~5AwZT6hs zBVLO6{;~b_;zfo&Ods#9G&J~7WMts-hxPV9HTSm8^EGmzwJj_R8@@f>&}HM)2XusL ziEBhjN@7W>RdP`(kYX@0Ff`LOG}JXP4KXmcGB&a@w$KGK3=FQ_b7V%*kei>9nO2Eg j!}8{z5T65W$Vp5}&0$Dga4&8S&kr_Wm>bfjssS zPhVH|2i&ZD3i|6pT|NMX_ItWGhG?8m{?X4V(bjUK+4b$+?D>k_v7Zwd7}eGn7i_%x zh9}^_CKj!-DKi@z^%t+V?wsA&_?Iu@&3r!Q=4ajlpC`&NJ4;5q5ePVM*kO5DQbOcl zk}r?O16dCpsn9oC9y$$;j=!y#YS=rM8X6S*ZLt!2mDRS>ftOqHO{~sJ{R2twD9XUGXT+j=jB32D|`ACK=Z*s)YbI^o~n+vnH?i}tjLgx%lfmyr=& zkzQ~_T8n{UOXyvt!d znCk)=1_oNe7N1cxoLMfGlY@EB~ zZze3aX>SSTxzXvSD{Xe=baRBy>;;WKJcW-6_gsE(@ZT*S&pG*`K`wvi>97l*t7sB? zE-qeTzU8dNpX7q2M;IqEFhonqt1v8l`xzJnswJ)wB`Jv|saDBFsX&Us$iUD{*U(Vc zz%<0b+{)O<%G6vJ$S^Q?_R}I4MMG|WN@iLmZVl6yG+YFSJJ5!l#H7?5hSUZ3;^qK_ O7(8A5T-G@yGywpvHnj8r literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/101_4.png b/ggml/examples/yolo/data/labels/101_4.png new file mode 100644 index 0000000000000000000000000000000000000000..7d7960bf326477a118aef185d4d3a2ec91526456 GIT binary patch literal 581 zcmeAS@N?(olHy`uVBq!ia0vp^azJd#0VEiT4OqH?6id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BRTQ^!u(i0|TR#r;B5V#p&c9{o)2aUi_gi!%rLSIh)J;dfUGPO0~+3 zy^LHe+Vd8A{?BapZeloor2fPu`@fPG9KP7Me2Ld*;|XY+kk{NXL%;IFc)?SyN}~Qe{75a;uEk zB!!7G9VNyQxAf#VJsMnc6fOnw+-S6Lxy96OBJ2{y-EML?ye02m{r`>3Z1Ec|&EB!Z z`{1SG2}kSRJ9^}^drK>LPdBod+w55{%5Xa0@q}Ii%P#AGoF->hAH1~adE%zno`goe z6<@Lw4W@5?Bi2!WOZbuQYx@m9;`l$W?CI_{nWFihY2u7khD|R2%!-0?Tqkd>?wU1$ zUsBQ2%kOXZymSBMG>oPurY8uSdf)#d9IbayaK++FSEjQuwAmio{k<=v5E!tkC9V-A zDTyViR>?)FK#IZ0z|c(B&`{UFG{nH%%GlV-z+4x|Ffeeu9czxFAvZrIGp!Q027|Vz gphUn9vLPoiDK&>7b-}&3IY1!>Pgg&ebxsLQ07SahNB{r; literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/101_5.png b/ggml/examples/yolo/data/labels/101_5.png new file mode 100644 index 0000000000000000000000000000000000000000..08d80033269264ec6cdbc2ed49f9690347004f53 GIT binary patch literal 654 zcmeAS@N?(olHy`uVBq!ia0vp^%0TSE0VEiNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|+GW`rG1_s7vPZ!4!kK@Td{3Q%}R!N3B+nef(tH1Ex_)&AmwEW+# zv1fZz%GfLf)>XBwd=@Ut&CHds`d{m^XEOzv*$nEM7CoED#VoequbAgq_ZA_|SYJVl zMH}X}aF{t=4RPMVyW-(`ez7g>t2Q!zmXKexN|U8~Q#13Vhw4Eh2~D#TOs{{? zOuWGqwpnoftjYw_;*YClwyaUhu?#Day}7QW=H&+s6V(U7pJxfKY5S=<>&QKEk4zq+ zMLe}Dyw(+bEc?^2TC;7%#Gj38o@_P?@F?Wz>2oyHEK|avg-ZosY)P zn6x{JXT`^V|CzKyT(TK0Sodc3IbYAe~&-K#RDkkh` znp19fJ0WmUjkGEMKaZmULZ4sSuaZjG5T&S{P-4|3BVKUrsT#klo*SE4;M@~+S6Zw) zPVWjdWnRO^V3D)(=dOhCXkZMfmbgZgq$HN4S|t~y0x1R~14A=iLqlBy(+~r5D`R6T zLvvjq!@!`oFm@%1hTQy=%(P0}8vf`!6$PgOpba^RNvSyusSEDK%>fEAc)I$ztaD0e F0syG$@)7_5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/101_6.png b/ggml/examples/yolo/data/labels/101_6.png new file mode 100644 index 0000000000000000000000000000000000000000..9d554db3384849b8b926c211fb53a814f92738f3 GIT binary patch literal 726 zcmeAS@N?(olHy`uVBq!ia0vp^nn3Ku0VEihvz6`vDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWFERwW+UBU`5+19@P~_JgY7>Eb!7^;U(ClRk~1XC&#Lg zmQ_z$KiwBPdK>hF%WMMYdXx%&huaxIg*%{)} z7e&`GaB+S4=d#2oh_l_B_)LTydKj;yt{$PT{`$OFixbOYxU-GG0>zu`N zyASe_Pyg?>;F4x*O#UAj`fvZ@#?XcTzW#rq^}pWdSjh3XUytpVo%;X(JjLv zX-}Kz^?rJ>u+LG8hD6mxuYxx}Hg}6snsUP}v~QJ+kQSrPMpm9HOB6l>V_LPuHKHUX zu_VKd4a7?@ib8(SHf=>i!B21`2w|DkBe%}>cptHiD0%9WQ+ f;Iss^Atx~@HHRT}!M(USKp_TCS3j3^P6Hn8iK)W z+@4`NZBGpn6&HE99C1ofUVmcBR28M5e|asJZqG<-Jn6Ex)uY4YpUr#Oqqg<$c0RW& z7xUVA|LOahUG?XFe%&p8;Xf-+e#K$wf2DtJM);P+Oi1}TBhGNg)5?R@Y1}FaNo#+< zJ*%(MthF;c>faw}j^Hw_FMSplJbIiTJZE+=J}9lV{|R@T(7)-QtsUPKl|$?<7XHWnCw6;HK6Q56WbU7sQpA~*f*2?jv-d}$?|Mb4l@USn4qLPCf7&AO z_pTTaN$Zq*zxVt#=&k5a`)gWx zLo2BINypU(H~myTz1S2bl=0fY{PT?Wck4=52+iK~CZ@dT{{7oWryV-K(2ilB;K%rE z_qiT(&Hw#Rs5rb|`#*=0+m6SDZ+s^L(~WA0YeY#(Vo9o1a#1RfVlXl=G}ARS)HN^- zF)+6>HnuW0(*-gN3^J7ifk_ZaLvDUbW?Cg~4H9)P?|@S#(1x7Eq|_XS)CKqA<^Y8l NJYD@<);T3K0RVeeU+n+@ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/102_0.png b/ggml/examples/yolo/data/labels/102_0.png new file mode 100644 index 0000000000000000000000000000000000000000..7b8b0fb1eb10f7d8d1de1adb7812c2545e3a1d76 GIT binary patch literal 305 zcmeAS@N?(olHy`uVBq!ia0vp^Y(UJ-0VEhIEf;$MDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWWMyQb z3uG7=NR>FwMbVI(pOTqYiCaUq=1)biGl4ebBqpWiFr-;t=LhFdwo0;^{pWL&{=WL5eHx8%9$49)ldORnoEHr~+P{r}*)J|i)a3Fnv?+$K$z-8$*v zcA%lEC9V-ADTyViR>?)FK#IZ0z|c(B&`{UFG{nH%%Gk)t*g_Y`Ffh1w&yg8LLvDUb nW?Cg~4a=KbP0l+XkKL*#78 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/102_2.png b/ggml/examples/yolo/data/labels/102_2.png new file mode 100644 index 0000000000000000000000000000000000000000..2068f0be835ffe74770247fd8622fe2647369def GIT binary patch literal 354 zcmeAS@N?(olHy`uVBq!ia0vp^JV30#0VEih3yzfmDVB6cUq=Rpjs4tz5?Mi#$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4R(Z#}J9j$v^m6`X{MGx+wqupU$vJzRQ@Q(W*uGV8eqy zj}j6L>i>K;nNcjNu}5@?=n+e%CpL|R-r5+P2b|NS=e7?i7izZJM z7!dnCk)= z1_oNe7N1cx z#@79Pj8o+qlhXJ8tNm!wvYzXS)_>;NEYFIWf8PB6@XvmQr@{h%?koH&(fat@SfTNL z%TMXAU%xu~NGh!T-~Q{JBvVa_!Hu5`e>G0-nyljc8)&C$iEBhjN@7W>RdP`(kYX@0 zFf`LOG}JXP4KXmcGB&a@HP;0)3=E$Aw8%x#kei>9nO2Eg!}KK$7r`C{+K`i&l$yhk UW_g_-7*Y%jp00i_>zopr0I*JakN^Mx literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/102_4.png b/ggml/examples/yolo/data/labels/102_4.png new file mode 100644 index 0000000000000000000000000000000000000000..37637d36a4fafce5d333c7ac7be771680531575f GIT binary patch literal 398 zcmeAS@N?(olHy`uVBq!ia0vp^!a!`w0VEjO&h=XZDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWGxf4pir}?i(`nz>Es{$%-^Sf+2VJ%=Joym|9Ju?I#1&f_#mfe zHg_gtY>vG!bK49-=C&J=2L0LpuG@eAwcqKeTEo#>33?3Ostrfw7I*w?YEk*OxZ`H? zku!lu*El9!`yyJrb=$UQ%$*-2n-AzrkT7TFTCpr7XTcvcEjGc~(flmMfk!VKC=g?2 zSQ~RzMCvA!J?)FK#IZ0z|c(B&`{UFG{nH%%GlV-z+4x|Ffeeu w9czxFAvZrIGp!Q027|Vz-QX|)+K`i&l$yhkW_g_-7>EoEp00i_>zopr02@?+S^xk5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/102_5.png b/ggml/examples/yolo/data/labels/102_5.png new file mode 100644 index 0000000000000000000000000000000000000000..a86b69bf24f8fd1a5ec83496dde41d43c093876f GIT binary patch literal 411 zcmeAS@N?(olHy`uVBq!ia0vp^;y~=c0VEi%oZ4&*q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC0W_mv&aOZkpafHrx4R3&|Mvbf`++?6 z5>H=O_6OXoe2Ri)`WaF{p$VQYjv*GOlYjIx?B_fG_RWlqf6v7$%x2X4r8B#ok;!S| z(ee+52OJD0{fJ9r%X##EslND!N96gFKHhxiuz*FdynwwqUg2XB8_$Qia*vvI zYut{qZ?sU85MADH!LaVC;Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|khkrLt0t(Icba4!^IGy~XpQSGUl-ZP|$CI{szN-H?pV2d^SBhCC z$9>Yz=o1GTV%7Q?+s}lt#TZ5HNZ~VjFzI9y=gqhOzkWY^uBW*`Qg}tnRs9(s17^m* zDG0uhBIl&qQos0gOxvj)f8D2BGELPyFySWi)Xc*ZZt_mel>cO;f5`Cwqt;?^KE~4x zM%yJfG=6eaH2(0=Ov_lyc_xpTftGPdD#M!4zcV~#rvm-OpjzS@QIe8al4_M)lnSI6 zj0_CTbPWx44NOA}%&m-#t&GfcfeZtKrJaHQP&DM`r(~v8;?{8G%1bA3*Z^(FNlZ%3 WVMw#Q&JPS+1_n=8KbLh*2~7YpYK@Qp literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/102_7.png b/ggml/examples/yolo/data/labels/102_7.png new file mode 100644 index 0000000000000000000000000000000000000000..a9a73815d31c0db4e7273c332db8561938399029 GIT binary patch literal 442 zcmeAS@N?(olHy`uVBq!ia0vp^azGry0VEhU%(zhkq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5`)g*j_1fkNv&T^vI^jwk=^oeKj1|1&xYJS^ zm@`XWO6u_N>{xnQQsRTh`J@Btf9LS8FEM1cdwE}}pfIg})-I-UJ&~8S1(8#;6B_kT zPh9%?VU@meJB!aPd9}Cy0HTzhWE}fRRZoOS9}3Nlfp|o4uI&ZOBf%jeQ4FtnXwn> zbk!2qh?11Vl2ohYqEsNoU}RuurfX=ZYhW5;U~Xk>Y-Mbw3uG7=WGV+HplHa=PsvQH n#H~T1?&Tez26m7QIf+TBISgr**ZF|~&cNX5>gTe~DWM4fx!0I% literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_0.png b/ggml/examples/yolo/data/labels/103_0.png new file mode 100644 index 0000000000000000000000000000000000000000..17cc959e05c2b88dbe2f3e5732a53ca5e013358f GIT binary patch literal 333 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3w8!B*90Kp`hj7sn8e>&ZX(8F?g1)-Rao|Fyo&`_1yWm5eW+y8ZvZ zobe_1p6&BA7+(Ir`loH1>3_+6ru%;++i;)IpCH2jgYP;&Lrk=#@QD(CMxbe`C9V-A zDTyViR>?)FK#IZ0z|c(B&`{UFG{nH%%Gk)t$U+y$FffoRah!{yAvZrIGp!Q0hHTBB gieN7QZOBPXO3h(N_q$-f0Vu@a>FVdQ&MBb@0EpOPhyVZp literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_1.png b/ggml/examples/yolo/data/labels/103_1.png new file mode 100644 index 0000000000000000000000000000000000000000..175794c43d115cd51a9b892c5dae68365b796832 GIT binary patch literal 415 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4hvpsO#~7H_3ET^0 zYkN>_Z~n=-Vkgff`vxmDhl+VTg$jSg6>ZAfbOheBw1l5Z_`-i;b?b*dV*`=e-Ic^SyR+>4p#$%$=S}#}5Rh}WY*&GnFwlLfC9V-ADTyViR>?)FK#IZ0z|c(B&`{UF zG{nH%%Gk)t*g_Y`Ffh1w&yg8LLvDUbW?Cg~4a=K(Krkr_Wm>bfjssS zPhVH|2i&ZD3i|6pT|NMX{&~7MhG?8m{?X4V(bjU~>&v&N?=9@G{gT4KbgpgxEgqhy z0vo%NcmyUei~nB2@#Iw66;)1yqQggaH8?feH=LOs#baQY%*OWdd&1`OgGcH%8W^ZJ zzxl1sD_YGtWe#U)i*m>t&43))kTasorgt_b8in|9rM1g3Hw(LDFyFbSrm&h{r6h<& z?^~F}iITa7>eE^o^}f{$b6?39^D5Z*KRVS>spq16!@so+tykCI6^?pt_v3&3yvBv1 zx<2)V1_o!?yK{@n%Jwmd-DWp=$P@8~LnlT~%D~{nf5z^=%xpZ(DJ7G5Py9-~u%SMM zvy_|pvxUO_QmvAUQh^kMk%6I^ zuA!l>foX_=xs|bzm5I47kYQk;6>RYtMMG|WN@iLmZVl0V4gTQZ2ilO6n3S5sknVTE Regja5!PC{xWt~$(699l+v6TP- literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_3.png b/ggml/examples/yolo/data/labels/103_3.png new file mode 100644 index 0000000000000000000000000000000000000000..1101880ff08ec20766ba4cebb6bce1dab71622ac GIT binary patch literal 586 zcmeAS@N?(olHy`uVBq!ia0vp^l0dA@0VEh!7}S6`Ea{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4d}NTrA%goQM%gX<(kHr84t;R)XQ%HM^9A+1T_{X6gyd&X&A!EjQtJ zf>DveH?0-TVag}(uo>n)%rkX3p~G&demFHsz-Y3=ndX9-z8o{t1cV>{uH~2+7cz5e z#y6SP(;>aD?>zoUO_QmvAUQh^kMk%6I^uA!l>foX_=xs|bzm8rQdkYQl(?59O8iiX_$l+3hB m+#04YX}AcEKcEdciAkwB4C#Is>^A^~7(8A5T-G@yGywpnSk;LD literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_4.png b/ggml/examples/yolo/data/labels/103_4.png new file mode 100644 index 0000000000000000000000000000000000000000..b8bf32e00fe4f4e5254849baa1f29a6b872144f2 GIT binary patch literal 687 zcmeAS@N?(olHy`uVBq!ia0vp^azJd#0VEiT4OqH?6id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BRTQ^!u(i0|Vm{PZ!4!i_^(J`o#@;y!d@z*53}^dcz_l$?QM#ruoeW z)Ky+Cw&Dp`Q2pqCy1`-z#k)t2FtD%DzPR+W#019Qjv+5Ic~&^cyZ%yRce#>$`1Jn$ z=fgfYi5^@Me9&f@WWf5E>5dcYj&nY{dU$Wu48h1M<{67W2n&krJ8>yB!B05IW3BVF zZ%oB~ik|gJE|az~c<*=T%dk|*6h|yZbF~E)3?PUo4%A6$QZ0^J^7E*rFW8(K|i>xjgKCI7!KRAmnfdAYw%W0Q{e#Y?yR<5eQprAlaiy&tn%pPZ*`dBdCE2`}AZX1-8~s1no5^f+{naq7Hq9;WSrmnJk+vQ9n6@LB4? zrt3EyYR>)-c=W1@A@5I z{#(jqY-)=Buk87M|Lg;ODkoo5|F`clF_@L!y1FT;EB4mHe^Nmb0bxdr4CT$%?{-iA zeGV9fswJ)wB`Jv|saDBFsX&Us$iUD{*U(Vcz%<0b+{)P4%D`L~$S^Q)yd7(fq9Hdw pB{QuOw+4f@r=aA<4zeL9F)1~NA>HqS{RW^AgQu&X%Q~loCIC)44h;YR literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_5.png b/ggml/examples/yolo/data/labels/103_5.png new file mode 100644 index 0000000000000000000000000000000000000000..6c81a57ed60ca197a47a7ac840ac31fe3580f752 GIT binary patch literal 781 zcmeAS@N?(olHy`uVBq!ia0vp^%0TSE0VEiNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|+GW`rGpgfbOi(`n#@#G)=5(Yi1BtxCeP5G^ARA*FPQJP+VcE|1`zU)g_;hE~d{_#atTCzrB z!WE~`;>p}v2WRbi{B!<){%H}4R}NiKar~KeJ#pULvu(?siCVX4e-{!|GQM!dr>%%z znSHkHs#ygb=cQFM$``Df*!&@E@rs|yU2C2hha8W-V!`Dc_F3T2M3?U4VkxHL3Nqd% zrp6Pl$}|-SUA%I}HLRFz_pH1W)9;FN7)(rCPh{0OUz(;iOUii4hrospaSJsn4>c`P z<2&@)AuzZ{>fGrCD}R_ynzVS$iY&WNPCNSUKlIY`Sg>4v#!j*R1zuvG6PIjUzQN{T zXvw3)_oIu?PQ0XJW9y%Mfb&7tzn#e^GJh{x`Sbrl0pHG7$vuC*{tp)D68e69#!nFr zvssF7BDtP82Y)C}G5!Dl{)}m&Pb4Q?DwjJ`%%Yw6w|^CzsKcj~Hg=}>Ee~epBxK#& z%hr7VMAkOPIg?p}lNlcBJBCf>6P(Q+EX?%K`j4}QC4298YXj5Z1$BOP{GlI<^-MHp z{BK`bd7d-$$ZV_kH%|HrFFu-}xRrU`vwHhJj(Kp_TCS3j3^P6(cF0X)y|^i5tD{xV z)65;k@5|p?mT#W^BK3LZ^Ecl&=0BhF{uAeq+XAkUQ(S-6Ke_VvoA17${DkXUcvNw$z;sYuoZTF=_mx_zgO{u;rA zPk(K`K7GXZ{IkA?t>vq$>k6Khg<8*;V(ZdSAFtSjba;HpR@FB;9Yt6 zkeSgpg~{TnPrDTRKmYa7JLGlKHrGye+j8f!kcwTGINX~LsXd;$YKM~1oR*rjn zTUIYh$O!sdYy3*rNN~dezt@~%@&0q4Px-s1x$~J2^Dp@)mZF#m0r0i*VCu!YRrvXYSWhFWhQ^_sJ8Gi0X=5b@FdL~ zubmc(;ivX~ti80b`TT|##i`qK9?ud#Rco*N@K1~O{;4acUH`_@+q=J7ig)VjONIri zb2K&-ncA1mTzQuJ@8SjCGww`hU)S5H$;g%~ttAz7KqZB(>Dj%`NxKvm@4sd8*j&o| zS6aZ%`ojyO<4+X)KU#f4{#D|C6}?W;$-va6TH+c}l9E`GYL#4+3Zxi}3=GY54Gnb- zOhXLJt&EMWjLdX_3VnqM| literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/103_7.png b/ggml/examples/yolo/data/labels/103_7.png new file mode 100644 index 0000000000000000000000000000000000000000..1919f01f87731704aafc2adc208444a662cb8989 GIT binary patch literal 971 zcmeAS@N?(olHy`uVBq!ia0vp^20$Fb0VEh6^1K!YQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fIej<{=xYAttpqL{7Et#;zYtaOy zMg6lCG_@A-e_FOc|4sAK3&&fXxOQ@=e_<4KVR4jnX$lOVdT_?XGzn2Y)tuC6L8}u_ z@4UA>)%v;ZbKCQguNSTRcIR#VJo)tM`_F67bNor2XyVZfME^ON?yW7}toC$gS&WPy z>lQubHD{vd`KByiB`iAe!0D$h8o{9!^TaOq&+mOOsaIlki36{RxnovZ0PB{!ENfMh zxe9tTPBk~rUR5{g@qq&!&Dok(_x`2v9J4;JC!)#E_p&gpcuMSZYd=h6|5`k1sgh*2JolW5Vah%yw^M&b4x7j#T9bQm1D8C~4zzAWG`2 z^RoZPMI_HGzi7sCAV~5?W=MNutkH%`FO1k$sV>+!|LI|exhzRMd;D_Q6O_NnFk7bv zI4yRUSTW&XmekgS1Ded+UJ5R;Z@BYvvWb?vgEEuc1r23|ukPsz3a5^xu4kEbkv&89 zteU;ym-5KKg63A%N7lQ|=PytQJ8|h!-MahhtFk}bzf$Mwty5#0rKV*Nbwqc{+?SjO zZ{KF$#2Lu6BVVarhw*@2X!OYvnI64Ezl-Bn@=Z|`zn1?#Og?h{wG)3+YfqlDc-s`5 zv;FEnD+@^uNk)escar{QiC*XPYsvYw$f)DGM|AMnf41*_O|zTjv-me7*AlMFylKlh z<^^p~axIwGhF%pzwy{`H#+g+78TS>9axckz2hCWe~TAO86^H#Gm~-l%o{M(*8z zH!Ad7A9nwgHrr9>->hj|zw~!lgxvA^|M!2Z`@LE&$~9@d7%+LOmbgZgq$HN4S|t~y z0x1R~14A=iLqlBy(+~r5D`R6TV>4YK!@wX@IS`mrkTm4xr(~v8;?^Kh_wo)n0|9Nw aNlZ%3VMzD8V7~z<#Ng@b=d#Wzp$Pzs$&N_? literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/104_0.png b/ggml/examples/yolo/data/labels/104_0.png new file mode 100644 index 0000000000000000000000000000000000000000..4ce801bc1b8576d29c1aee7d2efecbff9bbb0e79 GIT binary patch literal 315 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3w8!B*90Kp_K97sn8e>&Y+t{_`{ONR%v4k3JBup!@fk)!|r$x1ayp z-MKF~r2)e_f;l9a@fRIB8oR3OD*WMF8fYiOuz zU>agzZe?s_Wn`fXWEdDol{n5t(U6;;l9^VCTSK!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4lbDp6!;o<=Ur7;Y27{-opUXO@geCy+4{OE% literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/104_2.png b/ggml/examples/yolo/data/labels/104_2.png new file mode 100644 index 0000000000000000000000000000000000000000..aa88c847292c72c88c37c1a55165ea2bb325ede2 GIT binary patch literal 382 zcmeAS@N?(olHy`uVBq!ia0vp^LO`s*0VEi##LIPo6id3JuOkD)#(wTUiL4;WWRD45bDP46hP^x@Isih!@P+6=(yLU`q0KcVYP7-hXC4kjGx) z>Fdh=fSZ+1L4RGS%LkxPv8Rh;h{pNk8}c$SKjm2s&ZH+8$RBZ_c+dLRpNcHL0IgRoag8WRNi0dV zN-jzTQVd20hGx2khPno(AqM7F#zs~q=DI+Jfq_=A#b*=^x%nxXX_dG&MDsQHgZ&J& bAtx~@HHRVNUcQndP>8|P)z4*}Q$iB}4kvD^ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/104_3.png b/ggml/examples/yolo/data/labels/104_3.png new file mode 100644 index 0000000000000000000000000000000000000000..d574267590c40926cbc763b373b0ee3654c63c1f GIT binary patch literal 412 zcmeAS@N?(olHy`uVBq!ia0vp^;y|p;0VEi<{q$J{q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC0W_mv&aOZkpafHrx4R3&|Mvbf`++?6 z5>H=O_6OXodnu zz$1^_H~gr7ldznC}Q!>*kach{qq~RhsM1VHrBqpWiFl5}zS5gEDF?hQA KxvXNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}crr&qHfkJCMT^vI!PABiU&C9a)|3;>^vpfz5Y_u#RRR6#Ax&Aoi zfsx*x4$YmelNs6m$SwA&JD=phlw>|pb^mRn1q{I{CldrtDk>l3nQYrQfraU3q<+Jl z{lDKT+{sJvJi9$W{GSo)XXS~i=7(&zbDg>FJ}Kod+w>Ynu?en|Oq{vIRVOg$PfF37 zv_#pXfz@*pucr}*+=76UksgyyzL@lSg39HKs@wa$CaM3F=a-neWUdNB!e(<}*2={T zfbLc;ag8WRNi0dVN-jzTQVd20hGx2khPno(AqM7F#>Q3#=DI+Jfq~=gSaTE&x%nxX oX_dG&7_>d@25Mjj*^rZ%l$yhkaW7v<5h%pq>FVdQ&MBb@0FB^|P5=M^ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/104_5.png b/ggml/examples/yolo/data/labels/104_5.png new file mode 100644 index 0000000000000000000000000000000000000000..3159b0e9d837d74e5253b876c21f9885dd7fe291 GIT binary patch literal 476 zcmeAS@N?(olHy`uVBq!ia0vp^NkD0I=&#WBR=c=C;%5-fgykMl~bk#ta~4mr6(XqJX$P@ej+ z0}Agh6-e)%wBOrA^1*@sLSm}dbtNV+{*K{Mon0Xr!1y@LMD=x~#D&H$`68;{ViYDt zI(bf$U6^LvB{bn_rFvbXOWyTj2Uo3&Q#j_fS3k745>#%Y%01CZiCIadZB@&2^+}Ut z19IyFru-hy#oMVvV02i&Xf# zh19O71ogUd>L;rHS?txeZPI_1gDWNlsunOXT)u3v?DX?9|A3xTEpd$~Nl7e8wMs5Z z1yT$~28L$3hK9NZrXdFAR>sCwhUU6JhJitEVeCp24Y~O#nQ4`{HT=zopr0M>Gwa{vGU literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/104_6.png b/ggml/examples/yolo/data/labels/104_6.png new file mode 100644 index 0000000000000000000000000000000000000000..018e0e1bf16e5db0c67c7b73ede31e6557e00a27 GIT binary patch literal 511 zcmeAS@N?(olHy`uVBq!ia0vp^>Okzp0VEh?7YXG8DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tW-nOU%#5>-`X6kXKRxQ;(ICSshUXnv4!-)~_pndvy0yET=q^UifdzQ z*Ls)GwR^)j77e=48PtV8!I*O)g||< zy{i^4+V%fPBgk8AO;!v^uDYf{e-gZa{#Pw=jVMV;EJ?LWE=mPb3`PcqX1a!kx(22p z2If}A##Tn=xAXWu&35HBsVlm_`LeWQOd$wokLD}VQ=gfA?c81%Ms zbbq?};?(>@OLuM5*xx@fMl~^L>x@&a1|k2yH=LZBq*Z<>WFF&W?E@k6+E(t7RGZ!p zB!hpZ*qnNvq&2_U{b!cX)9#NN_uXcF`U~=7zt(b}pY@XtK>W<^Vaf3PJ@>{FvwpY$ zLrk^AHKHUXu_VKd4a7?@ib8(SHh=>i!B2ARr%z`#e+kei>9 onO2EggGAlSJK#70+K`i&l$yhkaW7v<5h%pq>FVdQ&MBb@0Ojh?SO5S3 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_0.png b/ggml/examples/yolo/data/labels/105_0.png new file mode 100644 index 0000000000000000000000000000000000000000..388be1ae33f3bc907215d853b4b3d8b7d071c645 GIT binary patch literal 296 zcmeAS@N?(olHy`uVBq!ia0vp^%s|Y|0VEja3uN#CDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tW?)FK#IZ0z|c(B&`{UFG{nH%%Gk)t$U+y$FffoRah!{y sAvZrIGp!Q0hHTBBieT3PZOBPXO3h)&T$v%X9Vo=$>FVdQ&MBb@05r@_5C8xG literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_1.png b/ggml/examples/yolo/data/labels/105_1.png new file mode 100644 index 0000000000000000000000000000000000000000..284a1305b4c46bead31b21700a535f532597b351 GIT binary patch literal 306 zcmeAS@N?(olHy`uVBq!ia0vp^Y(Ol=0VEiX6>j1OQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fI&ZX(*?4sCY<8St+05T@OT1pp#qsHaDfZ5` zq8cH_9|IWP$7i_5znZ%js71BJHKHUXu_VKd4a7?@ib8(A4! z=mHrA2G{O6GNWk7%}>cptHiBgdGk+*JApRjBqpWiFl4UG5ZVqjgTd3)&t;ucLK6TK C@l=2S literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_2.png b/ggml/examples/yolo/data/labels/105_2.png new file mode 100644 index 0000000000000000000000000000000000000000..d378969f08e8df857014b10176e119a2ecd5df83 GIT binary patch literal 318 zcmeAS@N?(olHy`uVBq!ia0vp^>_Du*0VEiHPq1wSQY`6?zK#qG8~eHcB(j1elRbib z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Q5iSWl2)#-1*YAsp9}fABN%6zuA{{z`PhzS9Q|1TZhjXMCA8p+ax@ z#o&}uix>ADzW#pcAzZ}ID8Glaov7|ftIx;Y9?C1WI$O_~uBzpw; zGB8xBF)%c=FfjZA3N^f7U???UV0e|lz+g3lfkC`r&aOZkpafHrx4R3&|Mvbf`++?6 z5>H=O_6OXodBxz%wm2HnwHm329stPF~UO_QmvAUQh^kMk%6I^uA!l>foX_=xs|bzm8rQdkYQl(?59O8iiX_$l+3hB+#04Y hX}AdX1<;0^#H7?5hRl^2Lfe5t44$rjF6*2UngHi-V7mYS literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_4.png b/ggml/examples/yolo/data/labels/105_4.png new file mode 100644 index 0000000000000000000000000000000000000000..afb260bdeee31b909bb9f647223093447d597efd GIT binary patch literal 352 zcmeAS@N?(olHy`uVBq!ia0vp^+(2x~0VEhUZTC|LQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fIG^xNf|k&wMbN?)FK#IZ0z|c(B&`{UFG{nH%%GlV-z+4x| zFfeeu9czxFAvZrIGp!Q027|Vz-C$nFVdQ&MBb@08)W$ Ang9R* literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_5.png b/ggml/examples/yolo/data/labels/105_5.png new file mode 100644 index 0000000000000000000000000000000000000000..53274dd7cee7ef2eb424c174b5439ee3e85f8ad8 GIT binary patch literal 360 zcmeAS@N?(olHy`uVBq!ia0vp@K!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4bHKpBwcxLiX z_GiB74Gr1}_jEMgv&|@HJS6y&*-%yB%xpf>&&jIM~dxIJ0Rx*6?@mEau&JzJzq*~${QIe8al4_M)lnSI6j0_CTbPWx44NOA}%&m-# ztqjd|feZtK-on_GC>nC}Q!>*kaclUa^Hdb6id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BLtj?N=iUP^iGu#WBRy6d!%trxfd}kQqn&=vu~(?6)R_N}wI!eb>8F2ZpUn!jd_{H)t*w0d){j=$0_|2Uag8WRNi0dV zN-jzTQVd20hGx2khPno(AqM7F#>Q4g=DI+JfdTWsC@&NZx%nxXX_dG&6fJ)A4eVi{ c4LONPsW}XpD>H<)1BDnoUHx3vIVCg!03)`1%>V!Z literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/105_7.png b/ggml/examples/yolo/data/labels/105_7.png new file mode 100644 index 0000000000000000000000000000000000000000..02b1b7d7a182c551a6a9d5d83508003bf2ecc633 GIT binary patch literal 391 zcmeAS@N?(olHy`uVBq!ia0vp^fzm0Xkxq!^4049#>64RsAnLk!HVjE${~&2@nc1B3G+QCCnjnqFMwYrjhJnEzW-mj1OQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fI&ZX(*?9gt|NPFla=s3asKUKPPmU{aX|XTu zkxaWh!)Y7P9r^JkkB`MW0rjYsxJHzuB$lLFB^RXvDF!10Lo;1NLtO*Y5Cd~7VNSwWJ?9znhg z3{`3j3=J&|48MRv4KElNN(~qoUL`OvSj}Ky5HFasE6@fg!Ib3f?!xfDz5mR9AdkJo z)7O>#0XHk3f`M|ZC&(}pPZ!4!iOb19_*r=hj;Z_4yHT&;#?8!pSWaNE;E5Z&NqaUX z^Z)K%B*FjKB;fk%g{K;RGBL=mTgSJ)a7QvwziNqVL`h0wNvc(HQ7VvPFfuSS(={~I zH82e^Ft;)`vNEyI1u_f_?nrw`p=ij>PsvQH#I0ei+lOUfM+0rhNlZ%3VaPH_bmj#L OF?hQAxvXNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3yGsK$>AfkG~xE{-7*my>_+GxKdAqwX{BQ3+vmeM~ zFY)wsWq-iU%BRTQ^!u(iP$!C8<`)MX5lF!N|bSOxMs**T6Kyz}(8%*vi0M7sxO$aJ(IB uj-nwqKP5A*61N6}wx``-9|3L1NlZ%3VaPH_bmj#LF?hQAxvX!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4<3Nmd82Jg+Lh%-L>JGuRXn&X$X=`4j)_BrR?NIG{=ykNe>_JG#%WIKtPh7Ze} zW6El$n4ela#XU!gg<)0q*Vl=PF(6;5mbgZgq$HN4S|t~y0x1R~14A=iLqlBy(+~r5 zD`R6TLvvjq!@!`oFm@%1hTQy=%(P0}8vf`!6$N_`XhTk7QfdxEmO-L3$WjJRS3j3^ HP66id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BLtj?N=iUP$=8e#WBRbP0l+XkKH6eL7 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/106_7.png b/ggml/examples/yolo/data/labels/106_7.png new file mode 100644 index 0000000000000000000000000000000000000000..727f3f6b677086464dfdce20517744ebdf039ae1 GIT binary patch literal 390 zcmeAS@N?(olHy`uVBq!ia0vp^LO>kC0VEh!EwT6oq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N8WN*Z1;Fq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PK1J2=*>lzdg{nPW978NlC;#YYl+X~o_2A6;br*l|dameU0q2PLu-v$rP zO|R#dxv`4NF8A}`VOse7Z9oBoUWt^L#nZ)35O|~2fI&==A%}bJE#DhEUILw=TH+c} zl9E`GYL#4+3Zxi}3=GY54Gnb-OhXLJt&EMWjLmg{3zopr0Gb$c)&Kwi literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_0.png b/ggml/examples/yolo/data/labels/107_0.png new file mode 100644 index 0000000000000000000000000000000000000000..9c8d8361f67f4460815d754572fe982b3d3454ea GIT binary patch literal 314 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3yUv?kF4ppd?&i(?4K_2e7t{>rmR7`)sc*tox8OPJCGrb~Wo%?+WT^{e7#Qqf_R>Vrkei>9nO2Eg!`Zk`Q^D>9+K`i&l$yhk?VeUV8z{u! M>FVdQ&MBb@09jO84FCWD literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_1.png b/ggml/examples/yolo/data/labels/107_1.png new file mode 100644 index 0000000000000000000000000000000000000000..7dc3d2bc0c6069337c1abb8129bf1c0b93023296 GIT binary patch literal 358 zcmeAS@N?(olHy`uVBq!ia0vp^JU}eQ0VEhM1}hx{QY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fI@TP`Gd2`aj=6C7 zfPwG{$B+C=Wf;0PpIYp;O7$_&8r2flh?11Vl2ohYqEsNoU}RuurfX=ZYhW5;U~Xk> zWMyoj3uG7=T)XGUjG`eoKP5A*61Rrs%|9W21KN<2n3S5sknNsUJR4{RgQu&X%Q~lo FCIEy8aRUGV literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_2.png b/ggml/examples/yolo/data/labels/107_2.png new file mode 100644 index 0000000000000000000000000000000000000000..d27cf783ba22a79e6cb7d43a916eee01bf7fa41e GIT binary patch literal 410 zcmeAS@N?(olHy`uVBq!ia0vp^f+QY`6?zK#qG8~eHcB(j1elRbib z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Q5iSWl2){hlt4AsXkCPjK_a{g;rvW>}b!J`KhD*2C!&z{+sSJ(My{Y?k%Ub}@q_6lk| z`N6F5NnVy`4F^M4T~>MwFx^mZVxG7o`Fz1|tJQGhIVNT?5k)19K~5 zBP$aNT_D52;EuG16pDu2{FKbJO57UOx_wv%4iBIWIf+TBISkqEX~naFLJXd+elF{r G5}E)=E`hiJ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_3.png b/ggml/examples/yolo/data/labels/107_3.png new file mode 100644 index 0000000000000000000000000000000000000000..04b6c22b68d72ae077f269c2899519f1a581ba4e GIT binary patch literal 446 zcmeAS@N?(olHy`uVBq!ia0vp^VnD3T0VEhAt<|RhDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWla7voF(d^dC#@@q>-4y%6}|p z8z-J|aQd&hYxltf=C<-0hn54WJn_>SJzl!MIP_o1Vahe@F9tso`4_F-zmWZEwXVS7 z|NK=Pb`w}OUNU6*CZiD{DP?e|+WN$5p3Me}68SpL?h#bJ^vXfz++v4^|052b5MX0) zI=%bkOy&)VKNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}crr&qHfkO8^T^vI!PABiUz{@26Kbeu|vZTXik>Hc==HJ#vgmRd^+Qg3F%!-2zte~&7ir#H{x(Glg}x;qiWmzd3;{hA5kj*uOdF9 z>|d*H-!gL$v3(lx*oyA%T+b7X{bI$P%OQ)w7O>o%T zsjD0Nko~hre1doS?#P2NkM(a{x@psqY+Ik~J%2()iU-rlzsGsqp03blX9(sM{=A@F zn+@ny)e_f;l9a@fRIB8oR3OD*WMF8fYiOuzU>agzZe?t2WniugWEdDY-i|d#(U6;; pl9^VCTZ2K{({6D10&U1iOiImR$aYUFo(&XY@O1TaS?83{1OOLXtup`s literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_5.png b/ggml/examples/yolo/data/labels/107_5.png new file mode 100644 index 0000000000000000000000000000000000000000..bc9730c7cff4114b2963b4551d5d493db690c441 GIT binary patch literal 526 zcmeAS@N?(olHy`uVBq!ia0vp^3P9|@0VEjWo@x~VDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWg1__q1f5%xR{zy7B)Q22g!8A+rg-~0= z5BnRUB?b--7xdq#ZE<|6T%Wvs5v$hxAL6^EXD#?7`G14;CPuCJhQs1t1AcnR+*{Ln z%KQ;Wd)A5?@xF#tEgr^xhXf}*4fq*p(X>lM>HHo6_m~3*=KN<7+41Ml!Ulb&RZ3#v zCI2@Z6!Os6SFxBY)KE6OX{R#N*Ar?nny)0-U0APk#(3t5D!a6v$c@R|B>al!X}E>R zAOH4M27kMp56b(m;F=F~;h+CG451U-oDcq2Hwq9v3skZH1b660hW4ia))5L^5#^5p zKGcg{;kfr1D7C&})r1;`pbzsIBti`qi7+t8ow#&$yJn{cFoaY~Tq8hVYcP@ zKacMe-}}Dv*}rqXky~a6K2E!)wR(+|$K12uH%C2xb>xe2*rRmOhB;jS9~oXaV!Z#l z;)N&L`~TWqxFTJjET{47occu8St~zQrZyzauUVD3V5i#rzj6VOy7M!{Gd3RmzJqzo zpVQm-@=nS6?DwPbnw3}ao+n%CTH6ljZ4sO2{Ik1jlJ15F&5L$vCp^}VIq5H>6ZF{2 z?oh(V{B=(>S+-qy=#|HuAl)@dTmHtxMNb$`m9Nx@WS8C-7*frcv*pNClSK)BQ?*xG zC0s6D_r&nUUsc_8Pj0Oa+;n?)FK#IZ0z|c(B&`{UFG{nH%%GlV-$Xpl5Ffd^L7v+VbAvZrIGp!Q0 khN8ufzJa3-XhTk7QfdxEwtHIfY@iT>r>mdKI;Vst0MDV}^#A|> literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/107_7.png b/ggml/examples/yolo/data/labels/107_7.png new file mode 100644 index 0000000000000000000000000000000000000000..95f81a590bb8202c4b617de8d26a4e663dff374c GIT binary patch literal 609 zcmeAS@N?(olHy`uVBq!ia0vp^+CUt_0VEjgoF;;3mUKs7M+SzC{oH>NS%G|oWRD45bDP46hOx7_4S6Fo+k-*%fF5lweBoc6VX;-`;;_Kaj^> z;_2(k{(zg6Pf;~|_MG($42)r(E{-7{$KPJ{*EpOg!TzD#{@2!wnPnS}CZ2M9^61f{ zBWDY@-z~fCduWo1kWiFUpP%&hskaJUm&n{mWtZ6!uw=#5PwSrr&U*j&$#3oJ->d(9 z`L^f$=RJF$t&KUoc*6|kVDC#&*=x66TYR?XZ_1yh8>Y=V6N)c;U0%`_>TzFeWx3>) zn`-6JOl$8e)<&?D7B5Nvyx^bB%l9%F&W&f1#7qi2o|!RA*(e&%W8)P)Y3I|pZOVyv zK26)=CfUwr=#85=*Qc@U&WZbG9JflAeT$`XBMy$G7AE z)QFo`W+jIvrG|Vo*nD-yBhjR%OZl&D_g-VOg?abyBd02j3xYc%EDIWM=XFl)xwn3~ z)0_3vbJ*^6PHlPgLw%!sb64;Yucgr+*)|r*wv`qgIki%6#s3)*H;Wsb-|U}Is%u&0 zzotjyUG2YZk8H0+WxxIZMtt*6yDtWjOlN?Bu3F+6QIe8al4_M)lnSI6j0_CTbPWx4 z4NOA}%&m-#t&GiefeZtK^C3}JP&DM`r(~v8;?}T_^HBykT7fp?BqpWiFl4)@70(6= OF?hQAxvX1C?w?R;uyklJvk!p|9nRIfB)tC7;i6?Q1EBC>JOAvEpd$~ zNl7e8wMs5Z1yT$~28L$3hK9NZrXdFAR>nqFMwYrjhJnEzW-mj1OQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fI&ZX(*?9i(eEqU;!8gHW49?z3d-xZ>v;-

nqF#umCjhJnGgdydQ~8glbfGSez? kYgpd=6XGhM4LONPsW}WeH<_MU0L@_VboFyt=akR{0QW3Tg#Z8m literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/108_2.png b/ggml/examples/yolo/data/labels/108_2.png new file mode 100644 index 0000000000000000000000000000000000000000..7a95038efe867765d91161814436102c2886b22f GIT binary patch literal 296 zcmeAS@N?(olHy`uVBq!ia0vp^>_Du*0VEiHPq1wSQY`6?zK#qG8~eHcB(j1elRbib z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Q5iSWl2)vYsxEAsp9}e@M&7)I9xP-*%sehv#>@!e4o&6H6KUno=I&2$Y7bq!2I49u;JjjT*8bb$;5gFDh5QYad7 r^HVa@DsgLA>-J$8*mXb~auSnLa~N`NGCi{Z3Nd)P`njxgN@xNA6lqVt literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/108_3.png b/ggml/examples/yolo/data/labels/108_3.png new file mode 100644 index 0000000000000000000000000000000000000000..81637b38cc6fe44d415a784af6c79e6195db704e GIT binary patch literal 298 zcmeAS@N?(olHy`uVBq!ia0vp^oItG20VEju=l%T%q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3x*!yOBxfkN`0E{-7*my<0f^Rv|d|NsB|h8c_;b^V(>lK-|dR&8K- zCG`ENG@s`>pip$lXf7zk|i v06GjwLvDUbW?Cg~4T*0)Sb*IJv>_)kDK&>7=O)uLkPw5XtDnm{r-UW|5zbS6 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/108_4.png b/ggml/examples/yolo/data/labels/108_4.png new file mode 100644 index 0000000000000000000000000000000000000000..0ea83f24b0dd470c0021086eb6d7c2c0de69234e GIT binary patch literal 298 zcmeAS@N?(olHy`uVBq!ia0vp^+(2x~0VEhUZTC|LQY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fIxi-BT7;dOH!?pi&B9UgOP!unXaLsu7PQYfw`5jv6X?jE|6hh;CMUM u97RKJeoAIqC2kD{ZBM(w?gQG8lbDp6!;o{6>6rykh{4m<&t;ucLK6TzBu_yA literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/108_5.png b/ggml/examples/yolo/data/labels/108_5.png new file mode 100644 index 0000000000000000000000000000000000000000..98d79d89675860ddef3832afdb705c63cc67c0fc GIT binary patch literal 300 zcmeAS@N?(olHy`uVBq!ia0vp@K!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4Q5L=DI+JfkAI! v>`D|3x%nxXX_dG&{Ly(T3U(pThMdHt)EtJKn@rCvfIkC0VEh!EwT6oq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N8WN*Z1;Fq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PK1J2=*>lzdh15J<978NlCs!28bNxHa&}_{t@<&co3IdNf{WmsCw#^$;}hJnHP xkf5SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_XsdCeZ?*keR28V+hCf3_EmbinXA{go@wTz{of5@{^=Gl zo+$bCf5yZ2^-m8wEHj?)Q+}@Ji5LdSjHvL-jM_j07*tDKBT7;dOH!?pi&B9UgOP!u znXaLsu7PQYfw`5jk(H69E|6hhu!q@86GcOAeoAIqC2kF8<33FVYG4Q1kdv5{n!}L0 Sj`i9)kgcAselF{r5}E)#nOvLz literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/109_1.png b/ggml/examples/yolo/data/labels/109_1.png new file mode 100644 index 0000000000000000000000000000000000000000..16bc989df9e86a0e54ac0c316e13f803073c7f05 GIT binary patch literal 371 zcmeAS@N?(olHy`uVBq!ia0vp^!ayv>0VEjMzx`MYq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3x_>z#i>fI=CbE{-7*my>_=b4t8WOgq1!X0_sKkC*fHU4wi+ZC;SH z(8o!Khv%o;njjvYo^pkxHM{@Uo24ZbRDP|uS3X?RF@t}?tkVYyWISFPl{g;BYPgcf z*5>bUYVJZ=rE9!P`iu?!ESr0Sp;ew?8Bd_F#2s-lpq;8Et`Q|Ei6yC4$wjF^iowXh z&`j6RP}jgT#K7Fj*vQJ*LKny|Ft~Qlkr_opZhlH;S|x4`%bR~fJPNcSCow5Chaq$P=2GZ;Kw{an^LB{Ts5R2_56 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/109_2.png b/ggml/examples/yolo/data/labels/109_2.png new file mode 100644 index 0000000000000000000000000000000000000000..cf8cf63d65b863cce47d7b918048f0484fd7efe6 GIT binary patch literal 426 zcmeAS@N?(olHy`uVBq!ia0vp^(m<@h0VEiXF~vLwQY`6?zK#qG8~eHcB(j1elRbib z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Q5iSWl2)^F3W0Lp07O|LB)C=;7HJKfC(&^m?|9`|{u9&iOB+R6f74 z@!|itOofvQ8~*ALq|Au1y77udAzN@p>hS{)_$NoRiOu9lxy!iulBC3* z#G>N|l}Wo%?+VxbFU7#Q4<_K-r+kei>9nO2Eg!&=!FkI(=B literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/109_3.png b/ggml/examples/yolo/data/labels/109_3.png new file mode 100644 index 0000000000000000000000000000000000000000..5f6757b902ecc9aeae160c4fe47effb3114aeb15 GIT binary patch literal 475 zcmeAS@N?(olHy`uVBq!ia0vp^%0R5m0VEi%Kg@dpq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990ficX>Pih57+xhXFj&oCU=S~uvn$XBD8ZEE?e4^zc?9gDoo@>w2 zo~RtdE6$vKYm=FOG0;?GrK0ND{#6TCN-<1#T_Mwx)YJ;}n`((`L`h0wNvc(HQ7VvP zFfuSS(={~IH82e^Ft;)`vNE;M1u_f_1U7mALlQ|tZhlH;S|x4`iElnwfCCh0Lr!8+ XY7Rs0I@W9JfINn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}crr&qH85kH@JzX3_JdP*-@V7~j2(?}{O)e|mmc8k{!?nX%#*S)X zp6id+D;WLXc4Sqd;f7O3^O{#rO=oC&`v1RwQhEc^;ol0Km)Q*LnlD?6Y^_h05ODbS z^Z$S2$cZd(0WwG9zhE?s7M*2g2T@ZnPxrl;bHD^n9D-1;Pb zqIy23xz<<46ZO%m2W4tgw&qMWbmD}=(-*L&rt8NDlrK-BD{GvaDLB+yOY2{sx z$-h_%i@0o<6xQwP4mkaXSudSqS~S=5^Gu)DD__x4P%@IxDd#(Pzhm7s&V^?DI^Si^ z-S1gv#m6FTDnak!p!+L`h0w zNvc(HQ7VvPFfuSS(={~IH82e^Ft;)`wlXl+1u_f_9B;>(qiD#@PsvQH#I3=g?P)hS e7JxS7BqpWiFyyXdy|xZ0#Ng@b=d#Wzp$PyVX1?+O literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/109_5.png b/ggml/examples/yolo/data/labels/109_5.png new file mode 100644 index 0000000000000000000000000000000000000000..be6b733ebdb6aa5afb2e1d2a8b1a1cad2a38112c GIT binary patch literal 587 zcmeAS@N?(olHy`uVBq!ia0vp^#z5@A0VEhE)d80|TRjr;B5V$MNJJ{#FSRSB+Pthnzp~a%Who^08wlRtxU_ zDj)O4XXaK7ZRX`cS3boT*t)kId~p8%|NO@ik_iXCh`Ib)BplP>n91#PzOLiI`~TBL zB_=c+u$+*?uf0j)bC8Mc^Hzy~h9|TC*G~>;mH2Ug!Hb>hooeR=KHKxfb}Zy}?dKHO zJmZI(Md!gMl8YTwx?NpbSh;=9h`I7JGY3D-o^W9P(?1^jURCo8{l0LJ=V9?5r>n-Y zVpFOG>UOaTXSa97xL!=|>`-H$G0(JM;rvAc6W?l7i79{aX^3p@xcT2uU}5Tsst9|f zC-M$oU6fQO#&ukXNwIx)BC~^E$xPTYL*FICY}34u6x&|{8=F0x!Z*Cgbh-AJU2u1& zN{MXuJk1kTO^QkKoG)g!yPhv+s(RM{O7g1loG*XWWqUxZRqRX*r@nvO|7)xJX<+cG zmbgZgq$HN4S|t~y0x1R~14A=iLqlBy(+~r5D`R6TLvvjq!@!`oFm@%1hTQy=%(P0} l8vf`!6$M8i(1x7Eq|_XS+;yzi)&YeWJYD@<);T3K0RXzz+0+04 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/109_6.png b/ggml/examples/yolo/data/labels/109_6.png new file mode 100644 index 0000000000000000000000000000000000000000..a02a117596acca0803549d9b5e752fb76a4549ec GIT binary patch literal 628 zcmeAS@N?(olHy`uVBq!ia0vp^HbCsf0VEg>y~!v7QY`6?zK#qG8~eHcB(ehe3dtTp zz6=aiY77hwEes65fIH=O_6OXoe2UW3el@Z%Ffe9%x;TbJ98Uh>&zK-_)n-*?$mbQ$9xnbFvvsnFIP>fk zp%*<<9v&BF3jHA=SrO7CzQ^_2ou@o(y$a_-)7e-Gb}|MR8#i$(p1J*nccG=6k{C~l ze|owD+n(D8wf>bZ;xfKbb7axZ8H~IIa*SF(p0w+odCxH|qu945F0sX#X*FGIRtYW(f1tmqfIq~9*>|&{my|uv^&TYyw z_8X@4>Rz8e2cBI#%iXj;=j?w?1~4m2?kW!(!-D1QTo*rFHU-9mYKdz^NlIc#s#S7P zDv)9@GB7mLH8j*UFby#RgQu&X%Q~loCIIAiNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}5@Y!?LGcYi&_H=O!i8!46!=LwnLDs1inv0k9%rXKoLd>3q^tLO> z3oFlGl>gf{Waj$Sm(HqMNJ_An)TrCJnm)}p6q&>RdApRMLgNE#j;ZHA?U;}<x|pKIaK=C0F0xe*hcpUJjqDkvB{wsrZDskm!z%-({BE;`Fp7A|@9*Q4#n;uC+G-Ip1Q z1^X>t<@)JA;}xDzHevH)Ca#jY(Q1cMTmCILmD6@ydzt3WjAc4U9x5jIHF~Z{n6-(= z_pRj)AMIIZ?8{7^E6*(A^?e(@<4l#w%>O|3V^X!mHKHUXu_VKd4a7?@ib8(SHh>jD`D2IoVfuApeh%}>cptHiBgALpYCa9RP{ bkdv5{n!}L0j`i9)pb&$ntDnm{r-UW|{3H)W literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/110_0.png b/ggml/examples/yolo/data/labels/110_0.png new file mode 100644 index 0000000000000000000000000000000000000000..49a258238ad79c3cbdfede642ee5cec684b435eb GIT binary patch literal 312 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3yUv?kF4ppdSoi(?4K_2eJ?j65&C6wMAOd%fOa*6WAa^A0=Qdhow~ zudc(bU;pbLpXHEhuVc8r%>F`F(Bc@NM%5D6h?11Vl2ohYqEsNoU}RuurfX=ZYhW5; zU~Xk>WMyQj3uG7=>|yrOMA49&pOTqYiCe?jxKC5Tt_9kVlbDp6!;ojr(=GxOV(@hJ Kb6Mw<&;$TcYFl&w literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/110_1.png b/ggml/examples/yolo/data/labels/110_1.png new file mode 100644 index 0000000000000000000000000000000000000000..10630ccea620f79d056be9aea5d943006ca39c27 GIT binary patch literal 341 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4>!h+|KMR%bA0ck4|8Ic(k!%Bafy)9B0S$a}#(TPMv&oapQl6eXCz4nB}zm0Xkxq!^4049#>64RsAnLk!HVjE$^}Ep>qm1A~H45bDP46hP^x@Isih!@P+6=(yLU`q0KcVYP7-hXC4kjGx) z>Fdh=fSZ+1!9Y3I6J%Jvr;B5V#`)wQ{frW9J1;DD)qQJS@7PjjY+!JwPWUX(5{V^Y zhg-`|8aOa~_M9<+!#(77i;$;+HoMx98kr>zIn9}yd3{cZCAEq(H*5K<;8F@Xd_aLy zcowVRtOExW_=KbQ1!wiCY?-;vsadma0s}*5%z2L54la;uR7+eVN>UO_QmvAUQh^kM zk%6I^uA!l>foX_=xs|bzm5GHekO6dww1*UmhTQy=%(P0}8rHgfSO)el(1x7Eq|_XS UJae9Q5ugx*r>mdKI;Vst09bu#uK)l5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/110_3.png b/ggml/examples/yolo/data/labels/110_3.png new file mode 100644 index 0000000000000000000000000000000000000000..b0d6ae1bde6d28e6e4afa7c13e5643e2349263b6 GIT binary patch literal 414 zcmeAS@N?(olHy`uVBq!ia0vp^;y|p;0VEi<{q$J{q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC0W_mv&aOZkpafHrx4R3&|Mvbf`++?6 z5>H=O_6OXodiQp`*_a9+}=;q5kj@d*UN&$32e=KF&7yIC;gR z{KQAg1CQS4kNwE#AKvUN!?2i5@OsTrEft{iR7+eVN>UO_QmvAUQh^kMk%6I^uA!l> zfoX_=xs|bzm8pd;kYQjTu+ak;7)Tm&^HVa@DsgK_eDlEq940^;auSnLa~Sf>dD=yQ OLJXd+elF{r5}E+JSdc{k literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/110_4.png b/ggml/examples/yolo/data/labels/110_4.png new file mode 100644 index 0000000000000000000000000000000000000000..8f57528a36347b334bc9f39cf3c0e6a15f90a3c5 GIT binary patch literal 444 zcmeAS@N?(olHy`uVBq!ia0vp^vOsLg0VEh^u;;Xy){e0sC;f~7X=e#C#Ivz>B zRo$}2oLL~{|NsAnoiiD^epvrocv3Dr#ewPJicRho5&;KnQX<6_H*|jbV-P8v5~;v( z@2~KQTJyR5XOjQ*DMj-aKPz-I_;0dl%^m$6`g0C2c~3|U7ErvvAU2~(XSaajZf>0m zT{_jmiq*n80nIwUc@=+i=>+tuobDF+wBIl#G*F2xHmp8mTG7q_KMt(WsZC+z__NP@ zH@jy&(DAAzt`Q|Ei6yC4$wjF^iowXh&`j6RP}jgT#K7Fj*x1UzLKny|FxWm>_a2Id r-29Zxv`X9>j-8vD1P(Ev4LONPsW}XJ<~;2pKp_TCS3j3^P6kD0Ics#WBR=c=8W_0fQbb$xtV6)$?jUP81p$oMbs$*T3XR zw}HePmQC}QZ8AE*kp5A6$E5QD3(x-lum2$3fvxEI#i@TzCp$1d(!Vez#VDacBt$hX zVUlxHWRv8&mgm-8LJAXJ?flBG@#(vVh|d#|SwgG%3-Y%)wWcl9N~~_-=)A8buu&#N zb>9ic2nIoJw@@S5$dg$eGq_hx;y&Ru&*_Ly;FJx2-8yx|HZ0)l5;DDVLzIy_Yq3V5 zYv`gjS5KBzLdiN_mpHXg%7q#&nfgba!)&}n(G1?1_r%_u`5wDOkzp0VEh?7YXG8DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWUp{*>BkKmg1Iym^3Ny2591QvHGL@10 zM#H*yQ>DyOdDvdOicy@hVAD1u^G%1kGhR6;Bpi<3@L+Z3tAdR;C0_j~Q)+rR+1kg1 z-T7|RlY&~d&3D*h=gMUrS>n`s^y1a4i&jN*cZ-&kx(8gfZSDN;5PJL%udZu{lWN?e zRhJ$8J(mS^vafi>bm8hxXVrI$S7|$hi#Nx{vhUgz^5#|Ag4N!~*X3zWd^N3qm2{}P zYw7<^=EbkvuJW>$GL$f%`%`9iniuGM)e_f;l9a@fRIB8oR3OD*WMF8fYiOuzU>agz zZe?t2Wn``kWEdDQ|BLcM(U6;;l9^VCTSL*}N8i9f4zwXBF)1~NA@P7xOxBp literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/110_7.png b/ggml/examples/yolo/data/labels/110_7.png new file mode 100644 index 0000000000000000000000000000000000000000..c7b4754ac10b30067ab80db341a76f08f5fbaccc GIT binary patch literal 544 zcmeAS@N?(olHy`uVBq!ia0vp^dO#e)0VEi1WJmb`DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWs!=UXOHTvEC6@niG zv2WCfZ(R3BEhbA;>}mdmssA50d!9WQvaVz1uH1y8V8f~G$y)u5Cw&iu#C5LRBcpb` ze`Sr@vQK3;r?w|+&2Mo3$?5a-@<)yRezQIuUwq1aYQNTUpP%)U4y^bT_;m3pb`MJi zH=cV7gP+*21H(+U#5JNMC9x#cD!C{XNHG{07@FxC8tNLDh8UPz85>&}o9hA@1_tLt wqOPE5$jwj5OsmALVISwC3~;OfZOBPXO3h)&Gv{d+0SYmAy85}Sb4q9e0Ev{#nE(I) literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/111_0.png b/ggml/examples/yolo/data/labels/111_0.png new file mode 100644 index 0000000000000000000000000000000000000000..a0e58834cb5c5b5e88811f2cf56061448fbe0360 GIT binary patch literal 320 zcmeAS@N?(olHy`uVBq!ia0vp^96-zsA{f#?O#TU^SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_XsdCeZ?*kg2DOV+hCf!wGi*j&m;U|N z?`Jf+reRmV#8P(2vS=@nHEi208A7gkC5S#|y$&=$wZt`|BqgyV)hf9t6-Y4{85o-B z8XD>vn1&dbTNxW!8CmKA83qP>n7uSnH00)|WTsW()^IlN(^Rmlfi~nMCZ*;u!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4%kB}>~IpB}AmIX0BA!_Vp3`jL%sv|_9mbZgQu&X%Q~loCII*=f_?x1 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/111_2.png b/ggml/examples/yolo/data/labels/111_2.png new file mode 100644 index 0000000000000000000000000000000000000000..477108699c03709393d95e0ad4d912ea9930ea23 GIT binary patch literal 471 zcmeAS@N?(olHy`uVBq!ia0vp^B0#Lb0VEjAIoY#-6id3JuOkD)#(wTUiL4;WWRD45bDP46hOx7_4S6Fo+k-*%fF5lweBoc6VX;-`;;_Kaj^> z;_2(k{(zg6Pr*Pr))Qpd8BZ6-5RLQ6Kl)h>cr*|4OaGevEXGx^X4aWg3`}aj1)Fxe zN-Stt$*xw>!F-}LXGI|+r+y1ZU2sA|!hMdUcj74t532dZ=5SBRZ+vl!GbOB%wOLZ4 zVxOb#hSY=$Y}``D1`&tn1f4u^fJ@9IF`=N5^O})?!9O<6HY0-%{~Ne|85n3ZhU&4* z+Q*?(Hf?63BD2m9XUP{6f)$=|dM=n~uFz+Hit~$C#Qh@-Y76Gc?J#eBa4*N5<;3EM zv<)-<7EW1Ey?fdA!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4PJA|Jl;kd3)c2(|_hHJjlQ{ zC(t}F<=hH`1B~A6lg&_(_reM#oE?v{hX(0BuG=IZ&%2M;27dAI96C3pR?^LDMP|6o&nZp=Sb5`hN$8lxm4< zL`h0wNvc(HQ7VvPFfuSS(={~IH82e^Ft;)`vNE;M1u_f_1U7mABLPW6ZhlH;S|x4` jiElnwfa3yaLr!8+Y7Rrb1NZhOpb&$ntDnm{r-UW|5Q4Yp literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/111_4.png b/ggml/examples/yolo/data/labels/111_4.png new file mode 100644 index 0000000000000000000000000000000000000000..7a4aeda25fe1cd8e961362b47eeeaa97d0a64e96 GIT binary patch literal 615 zcmeAS@N?(olHy`uVBq!ia0vp^@<43L0VEjae9GSnq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5`zA(sQU7#J91JY5_^EKVo?=$AL>ImIvhv;6d99Z}YlHJiWvJF4{G zW_m9pTgvnQuG8nAKH9*eGk?*E{~jq0%t6y9KB=!XOkkLKoa1Ln$ApUCSL@@~eLLY; zv8+DfQ?w$}x_kwzeQJ(B?gsqaKPzF~P4gYIjSn!{$SZuZK49~0Hi!Lx6DICb$0xxD z=9D~fxXmkO)N&%b!BRcYDO6JB$QM49|IDMq)!KjANp2Ij?QEa7Z#}DG%GGBHOY#+V zSRZumogaH^O@;WjKaJcyr!I%dF|;vp>K=3B_zMhr)e_f;l9a@fRIB8oR3OD*WMF8f zYiOuzU>agzZe?t2WniHTWEdE1pR9WiMMG|WN@iLmZVkuIO-%wvF3^UY#H7?5hI|L^ R?M*--22WQ%mvv4FO#o1)>K*_9 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/111_5.png b/ggml/examples/yolo/data/labels/111_5.png new file mode 100644 index 0000000000000000000000000000000000000000..180a9c58957adfc3b2bcdbf9e873013e9dba72ea GIT binary patch literal 686 zcmeAS@N?(olHy`uVBq!ia0vp^szB_(0VEi9oOx&lq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|c_iC047#J8Ad%8G=cpOju;m?yG5o)?BD`2J7-B;^%H5JmN3|~s> zh2CGj!sr0wgM0innmEIfVY5-1=c|fOEB^NyU4|kh#rLtx`As|9|P!qV5#p+xTJ1Mdb>nO;dSx z3dBk6T9qwOXxXvKfMY#->{b5`3w4*N@?!E$jPfpzv^Pagz99NHwaKG~ZS|?c57(t@ zm^nRm2dd-h*kS5&l{KTn)-70+OHi+|&DLb%wFO$cSW2()W!RWG@A#w9R(q|vW7WUr zRaN>7f*XFf1$?>4`l|2bhcBkGT1m|3`O_0P-QTM#t^MV6lts#M+do^W<1F4k#UHK| z$k@kdv-0%J86OmiRBfCNRKD0Mxb4irJwNQbR{fvPY#`d1|F?hHtN;IJHf3}*#r-`V z^7X&(Q~@W>+;TRo|ce4*{EY?zg(>I!Xq-FrV^oy3~HC0;-6btg5p!P z#5JNMC9x#cD!C{XNHG{07@FxC8tNLDh8UPz85>&}TId291_srZx(+BBa`RI%(<*Um k;7XbL2dIG^WJ6A3QfdxEz61C6CZG_5r>mdKI;Vst07lIIJOBUy literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/111_6.png b/ggml/examples/yolo/data/labels/111_6.png new file mode 100644 index 0000000000000000000000000000000000000000..e1482cd3de68a6854afdd02a5a3a2d641cebc228 GIT binary patch literal 781 zcmeAS@N?(olHy`uVBq!ia0vp^T0rc@0VEh+#LT=4q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}PX}=m-fbvY9E{-7{$CH2fndB)ARFdaz zduDC7boZVJ34sRlcSd?km#>kyz!3b&ILE6#)W9%->EYfPUhz@J39LGkFHE%yNIJl} zY>`&2TEEJ1tuvjE9(+CFwSV?O)rwDi%*-$UUQ8<7A#gocW$Ds_#Q)|7%66iM_h$a* zU-w1H*k=QeWlq@UgDvR?rv5tIk`&NfqpTbf^UXZuW$lg=>RV>$w9nkgvuX>c+6FG$ zOZA!#scRN`B{r`ABCf1^w$aP5o<;jFZ*eVi+NR%!_@>@w39I5x`!A^Dc|}qyoI9+F zVc8d9v86u)c9v|ow048X!9}0yr9+H3Ts8U4mc|Bnif_2&Y`|V~WRZy2(|-v|zbAU} zCj?6IP4#Bg{(GABsaf)>Kk95#XESs%MKnYtYuPew{;1-T$v@?Eg{<>R?GUdg`Ytbv zBcys9!*{Z!W$>Bs{^@V)%yA2?WP8TFY_-AC?FlOz{!45+CHu!fS?S;5DgW{tIt$M? zY5$30JTJY+U%;#W<{{(hM`!6PTx36csl#UX7kj6d|EHhg;574%w)SLGVK`qfLFw^A zC1$+|f8|48{uf_%L)2*NKlkuo|EtZ+rUvc)pRD!wfBk;FT~6C-rd(+Y5&0jRe7<8z zz>XJ+SyL8>edtd+HnU@s@kfo_;qU+U%AIdsVa{^$-aZVgImsx96syh?&e+uQ^Mf)ba}G zN)T}2-|3j~%sEhDFNa=JXA-Mxpo){AZK%W*XBNS)Ni1O<3-&!}b`o6bCO@UIHYZhk z&FeW17SHePEKXary~(owzw!RWXZNhX^F4?cXHijL%)0Xky&somdsg~!eW>VDuSatEUHD|ALEN0s=`?BBns$Uzrrk@K}i&qj^ zH%IhZ0{{IFT$6Xtkrih7_ek^h;U#lVdUqxmZ{^Nh={RZ842=(V_a~__oYKggaMEOQ zoziQ~%&DGwKH_(j4y`YbQrpp_x5F)kS@o@)%6Xa82Tz0ayYv6(m3Mq#e`A>BKa)q` z<)asCJtyEyz23Y^Zv$G&UeoXTxLx5ddH{rspIh4&w7&{ zJyVMo{JP~rVprZ*)$fbk+MMRhFKBopsOq)J>Y&)wl~=d@Y7Cm%&UVq!C^|&{mesZ& z7dCG%ZwmYMeEW{*eUXc|cdyQx+__<%gHgIr&BOib?@uh~ez1B+|BCs}jStS~=?hps zw$Z+#nHJyCrDT}nH?1sepS?sOAr7xyjX4vjyr@Jske>%@1pqOLz5Hr@F4 zO!^XM-~WsDJ`PMm0`GtEZf|(to%f>I7?{*lOI#yLQW8s2t&)pUffR$0fuWhMp`osU zX^4Tjm9epvvAHggVPJ4RBzopr0ByWz+5i9m literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_0.png b/ggml/examples/yolo/data/labels/112_0.png new file mode 100644 index 0000000000000000000000000000000000000000..d732efebfeffc29b6530cc348bc36189b61bb0ad GIT binary patch literal 322 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3yUv?kF4ppdzzi(?4K_2eJ?j65&C6n%BrS|gzQ=ilyAZVI_y|JRoY zPk6THS5TVdgli>_y8rMlS-bGn+Q2W&3_Z#=N@Ck&+JHu=mbgZgq$HN4S|t~y0x1R~ z14A=iLqlBy(+~r5D`O)oBTHQ%!@ytzopr04Ky>tN;K2 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_1.png b/ggml/examples/yolo/data/labels/112_1.png new file mode 100644 index 0000000000000000000000000000000000000000..11243af759f13ef0a6e490dd37dac29ef7bd25d3 GIT binary patch literal 382 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4BPW!97L-`RvSx)eZc=%TEYDq`+pX`u_6# z(m6ag1?H(S3#&6XAK_l{Hks3o!BT7;d zOH!?pi&B9UgOP!unXaLsu7PQYfw`5jk(IHfE|6hhP_WW#JBo(f{FKbJO57S4zkK2b e`x$6MPGVAO4nx6N=3V7LAqG!ZKbLh*2~7a4G45bDP46hP^x@Isih!@P+6=(yLU`q0KcVYP7-hXC4kjGx) z>Fdh=fSZ+1!9Y3I6J*$SPZ!4!jq}Mr`WYqIwqMxKJbBjj`ev@bX$c88YQmU3j5yA_ zbCebvIyhLybj&`uX`l8y-V;%5!butIC*__eBn1577B%|8Zni|`gbkbMEWVAe3@2D} zYLzg&Qp#j&%UW?kZ^Z-|W@oXG74k}-S|>Ix6w_MrQC!DF^28mEVi#_YClgtin-7@J z5NQ6qr1hzj!!EgJS2WaCwEOHkxOcCkjE>MJHbJ5P^?5IIXP5I^76dd;IOa5CA{#^1 z`z6O%T~Cw%y`Wm+8c~vxSdwa$T$Bo=7>o=I&2$Y7bq!2I49u;JjjT*8bb$;5gFDh5 vQYad7^HVa@DsgLA>-J$8INX3XbP0l+XkKOqGrO literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_3.png b/ggml/examples/yolo/data/labels/112_3.png new file mode 100644 index 0000000000000000000000000000000000000000..f830185e693ed124642b9b69a46d4b30a17c116b GIT binary patch literal 507 zcmeAS@N?(olHy`uVBq!ia0vp^;y|p;0VEi<{q$J{q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC0W_mv&aOZkpafHrx4R3&|Mvbf`++?6 z5>H=O_6OXodx1gH;$j@M&q~iU}-JfmS zqMoxp`g-u_>x6T=k8wPyJbZNh;o3yTC!L3n+at_-@uqf&@Rp6q2?b6#eX;0^v+Z>S;p0wb2qw1GRZGG_W1vN>t&|?QNP7(>kKa) z4iNm`FZ)1(h_=BH$)RpQo=_~wHJIKY86bP0 Hl+XkK2#2?5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_4.png b/ggml/examples/yolo/data/labels/112_4.png new file mode 100644 index 0000000000000000000000000000000000000000..a4b184a479a06f33fbaed17922ad4178037e2e6c GIT binary patch literal 568 zcmeAS@N?(olHy`uVBq!ia0vp^vOsLg0VEh^uvngEo|G$2p0kg;s+anWnQydufPJrI8k8U!+4R%rON{j&r?)FK#IZ0 zz|c(B&`{UFG{nH%%GlV-z(N>zsJYD@<);T3K0RXAK(vScE literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_5.png b/ggml/examples/yolo/data/labels/112_5.png new file mode 100644 index 0000000000000000000000000000000000000000..02f97a201620537e7a64f78c6e6508d8e342e970 GIT binary patch literal 641 zcmeAS@N?(olHy`uVBq!ia0vp^NMu<*RsXM&6B;gP`KabkkqB_qZxhmIE4k3Q%!Sp7 zQ-kxsR0COuP$s5p+IEbtalV4>pHl4>Xutf!Zu)boX2=q5Ur*jcv-OuMI(@1Sn3D0= zjce`HfP=9bUSW$^cFDPHkc;fR;4h+Tthgxegx8;d#qT7QBSK^%J#C}DoN9E9oH*6T zG58Ui;J>CNDPprGX>@$ZRZ*4A@Yg;f2QBJc^~hQ~V#-JDq6@KZ zftwhWRx}Fjda2>HXTq9x?ZlmJOCAY?6j)~Yysm9qU9p_4cdC5G%cIW}LyjtYukcWG zTlM(t&HrDXpI&@)!=6LmWt=&!UO4*het1ekOQ&vO>ZEF}46Q5k5}zCs|E8fO6Q7W> z_5c6>XWddf=DPm>mmK)%|Nr~RvYd7E9A{3}Q*1Zs5_*#ma?(Njr@ugeJ10YAGxzxj z>G)<~q^Op-MwFx^mZVxG7o`Fz1|tJQGhIVNT?5k)19K~5V=F@oT_D52pxRQ`0YyV@ reoAIqC2kE|DO3LdHL!zh$Vp5}&0#1w%e<=`D8%6D>gTe~DWM4fC2Qza literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/112_6.png b/ggml/examples/yolo/data/labels/112_6.png new file mode 100644 index 0000000000000000000000000000000000000000..2001b39e50b2eba5d5ff44f65464de467c4ea892 GIT binary patch literal 710 zcmeAS@N?(olHy`uVBq!ia0vp^8bIvD0VEjStn_>Uq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}PX}=m-7#J9Ld%8G=cpQIw#aH8Sq73_o^v#u-nO)7zr@amz^j;*Z zA?~I3SfRJM`O&g19z_cZ42ow?`abcjJ^PQL^bAY%%+H_U-=>rK4ga`c9)o zJLkdDFPkQ5B=7WOs=di_?7>8*{f!+L^*kQTn)x8(nZX*RO)vNa|9EcwtRzwW$zM!e z>T)-W)byXX?b(H%O0-2D@6xwCbNfy>%k`=D(Q6v7%L!gPG?n$sntZW`jT`RpU3>1L zXmXUL;*h-}L*n|1n53UMn-~{L&vp2h!`||oqi=Jk?4f8D^P5Yg4^^|&<|H*==zqXB z`-;Pk{s&sKBNXR2Opx!<R<(R5F;(>N}O=Vj2Y7oS9~ul!VJquI3XU*jpozXg9|Gc$X4ConP`D>lu4pR={J zKi{5vwRqwsV0cPg5}9oDhwWaNaa~>$j{`7fRZCnWN>UO_QmvAUQh^kMk%6I^uA!l> zfoX_=xs|c8m65qFkYQlJ{4dH2MMG|WN@iLmZVg3?AAJL-9iR<4iAkwB3~`}vx>*zU^6ENZ?Y zWu@UIurX40mZP>8tK$i!9UOd*f)pQJXyWkFY2>idSfP9Os$zgkWke?Did@45)23|s z<)^#XyJ-J=+o*%C_sXlk|DCX>{ChR$~OOQSIQ^B!A{00*ito*FVb~!ugznyrs*pYpG zFNNCv`4`nPHMT@TU(4m^oL`m(Z)+EQ3CL&I zuhHN0!{)$FMR|plzl@&f#~hMhwU4bbPW4aYtGn0!&3(naKwqe?^UeA41HqqK|2h2O z`Y^kwp65|}<9wmibN-K3`@D*FiS#}{Jy+nL>%BVu4{?v!Q^nclxr*-7uTPBOd~&*P zv-xFhX?yp1mHSf9`6hsKe>M&Mo`SW{YZk^locsIEsn7D4zrOsxXOj5rlqDkd za~D7DOsKzq;YAG(kHv>;8~46_KlQ(QX>~no*(xsUXOBOu1*Qnq64!{5l*E!$tK_0o zAjM#0U}&amXsByo8e(8>Wo&F^Y_1Dr7#N%niMoQKAvZrIGp!Q0hJBomGQepLXhTk7 YQfdxE!CB^AgTe~DWM4fZU9|N literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/113_1.png b/ggml/examples/yolo/data/labels/113_1.png new file mode 100644 index 0000000000000000000000000000000000000000..4c81c210b49b704d7453e17289b5f40c39be627b GIT binary patch literal 386 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4x^d2SCV}7> z$t8Y&>K`&0KK-BS$HtbK=2yw_{=cWXo?iRpfZGcjV&ugBGcv4cyAkx!C8<`)MX5lF!N|bSOxMs**T6Kyz}(8%$jaDK7sxO$C|K#W9YsTKeoAIqC2kFj iUp{ez{SCAsCow5ChoNv6OV>l75QC?ypUXO@geCwchI#}5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/113_2.png b/ggml/examples/yolo/data/labels/113_2.png new file mode 100644 index 0000000000000000000000000000000000000000..b29b8d012959e44ec9924a5f3735dbf0bca544cc GIT binary patch literal 450 zcmeAS@N?(olHy`uVBq!ia0vp^LO`s*0VEi##LIPo6id3JuOkD)#(wTUiL4;WWRD45bDP46hP^x@Isih!@P+6=(yLU`q0KcVYP7-hXC4kjGx) z>Fdh=fSZ+1!9Y3I6J*$SPZ!4!jq}Mr`WYqIHecvg)h*6f=B_=PoScx5!1H&)*#il@ zVpaLa7#Q7N8YtE)^s6sZi=Wvju2sULvFIDiiC?1U7(`F88}&8!)}Ce*-ISIP&=@7k zBN~*N(9y~j>exC(Qle{t)7Ah*X|^_n6#*O}8saZn%o8R)pA)dK*+Dwio*_~`C!z00 zK6}c^?Ky9!BsOs06>P2YI3ug}<^L?ph$D~G?%Z}LNXzUO_QmvAUQh^kMk%6I^uA!l>foX_=xs|bzm5GHekYQkON7_RQ uMMG|WN@iLmZVhYQJ}d)jU{;D-R4hMt^ z-q+iAe3aEH44lWv#qwRjd(QLp2Ih%cEl1Q-H%urii(P2Q%q;G-i1)~oD7I~d!u-rm zPEto=cuP;$v>YiFI4$?=#QswSokta}ne6!N{ZOb@;hN!&BT745Y&*V6>22iGyU3Yj z+wOg)*;|KoBi}*QqK87y9oHz$_~D*kacf9?^T7ff-as325|dJM7z%f>bUg$LF?hQAxvXYFzo z&~ACdBgm$~QSqzZc*aIqwcj_p85vd9FFLi)Jl%oqly_jtU&(+*yG?I&IwYdMUHtxk z|C{cVmQ{@>{)Ms>AMtaJ-9Jk*vW$7g*&3F2C$my+`WRRX3bM%moS34RXVEF*#ObkV zje^>JdzG66phR(}Xfbry}d3h`CKIPvT}g5#~&}`F~mh-*>i-C#tMdEPm~0 zKE-?U``7O`Ys}a}ZvOw6vGIJn((j8Bu0hFmS*xnk8&xuYap?N~`)@4yDuC_dwfiqL z#W?=Sz0~PA{r=8h_7lgZxUwzYbmX)WLtP}p<$vBD55LcF0*0q*iEBhjN@7W>RdP`( zkYX@0Ff`LOG}JXP4KXmcGB&m{u+RlE3=FnU*1dNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|c_iC047#J9HJY5_^JdP*-@Ru;?StS|jtZ%yidYV+h#;#jeV^}}h zondb_K9Tj$Z&GRlLHD=f87H84qtC#|qNd%S_o!hFHX zp9@YjFq&Mv;&M=>WZB&xfByfzXjHc&XvVDBJpt}p{OQII$06Z%L+EiS2VI*11GaToVs9@W!oY(wnMY~S3MK%lFH|J zZxB_L;2f6B9r&5YmzRB$lz+#p>J?u55{u7ghL~;-xN^mzVspUNGwy+#84htfhYEKr zdZyTN{1?YjEww8aSDOl$CrtUX&8ho)nZ|ypbx-_ls%}-B=N9$&!4dfBimY<)-}TL% z9$Vvn{{HNvsU)d%<g716KJE}ODpX5c zBT7;dOH!?pi&B9UgOP!unXaLsu7PQYfw`5jv6Z2PE|6hhP;IH}fTAHcKP5A*61N7f jl&OD!8rVTLbP0l+XkK+T!yY literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/113_6.png b/ggml/examples/yolo/data/labels/113_6.png new file mode 100644 index 0000000000000000000000000000000000000000..c4e0d989ddbc48df1e5bc73ca672d18495e40824 GIT binary patch literal 702 zcmeAS@N?(olHy`uVBq!ia0vp^8bIvD0VEjStn_>Uq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5}PX}=m-7#J8gd%8G=cpOju;jfY)5o)ojAVl#dSHJ0Q?+pdoKfc)< zSpMODQ8JrE0?#fMp_x34+JPc=MM(~9YrfY1|KGN$ccN(C^K=Jh!TlRFV@@AyVEnW} zBj)&-2G*Q|0TolFXIruPmbBLEJ}5eKakYHS@Bfc0W{XT>+i~bA$HMFl2Yy9%xh}Nx zIr5M@h+%uOh7G&$gvwqS;s0|kh+aF$61M2rP0y0nYX<)RoV#PaLt5*1F8HJNCFMf* zst0OY12|nD@~v>P_i6p3!L{B|R4Qr3?rtfY_(aV&#=8!wW&~<5Kh1Xtf6=?lK+gTg zog|mw9Xw*~y92KLEq3*{eQ&Xl>CyQKqT3RdeJGjHam%^I%7w{YW6wzzVWIZQ7r7GK z&V02ORm|A^bUg$LF?hQAxvXaF1tC~@GU@v#|7=LJ0lm7h*M;$s zpZD+c>}U6?&+VOl;Xd!hz3)ob|F-|A&KxSa@ZhvJyFTtdaBqKv#}AfpYx~C;_pfU# zDqr>GxJ1<5_og#zYlQx6nAtJ;<)o^7VMn!>L62&;vdmRt{jTd`^moPlX@x?$j(1HC`NwWA z*b#k8u>M=^s;(F7&tCBVV8XOLnfFW0+8*-@`VRTYs`5AA^|L*=9J%uNyMB=e$`LP* z)@Xik3wve|uUo}$zgYRqZP)*&&Yx)g=~ZA;b6hTYA4`R{Hs6`&C)WQLIW)iLfcb_b z_djYe_v6BX{$4P5-hcYC)PwJd7pfOWXtC8_XuPX0y5}o*jW);U>^(EDnck6fH+X;k zkzq>Q(eh-DS1*@nncLO8obIPQvBH1es`l+Ub^+TL2Y>!>v8pOViP8P@p0^VO*!eFWGI^SSua6@yX)GKfhyg z&h+)3TQjdJlHo=I z&2$Y7bq!2I49u;JjjfE$b%6{6gYzL#S5P$M=BH$)RpQpLkMmImIJE(7$Vp5}&0#3q T#nSZgTe~DWM4fyERD| literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/114_0.png b/ggml/examples/yolo/data/labels/114_0.png new file mode 100644 index 0000000000000000000000000000000000000000..9d42671db0a8f219f7f3497109ca87c338cf02c6 GIT binary patch literal 299 zcmeAS@N?(olHy`uVBq!ia0vp^tU%1o0VEhE@kw|BDVB6cUq=Rpjs4tz5!lV|}@NWs&^F@)oK@(+F{o)b@sVhkYKpu8`KD-SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Yp!eP<26vet~dYx zzkq9xV3xMv87&pzjf4STeZYBq9i4;B-JXpC>2OC7#SFv=^7g9 z8kmL{m|GbeSs7dE0vQGd1uMO_qiD#@PsvQH#I1qx%O`HIlYutmBqpWiFcc~C-i`(e OF?hQAxvXkr_Wm>bfjssS zPhVH|2i&ZD3I@uto*=_KJzX3_BrYfa=;x7OYb#&0`8P+$_t}k&iN(4T5@zr`4K-xq zy!SmoZgs}@u1{h!yw1Hg2zJ_MOnIf+TBISfV0ytku)LJXd+ KelF{r5}E*P0&D#M literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/114_4.png b/ggml/examples/yolo/data/labels/114_4.png new file mode 100644 index 0000000000000000000000000000000000000000..65912f0814169a0d5f2f73d2369537970f8911b1 GIT binary patch literal 391 zcmeAS@N?(olHy`uVBq!ia0vp^B0y})0VEhwgr+zFDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWEs{%tOh+?Ldq4tO$1{8{{NpjPfdM#-ao?) z3(T1XUi{y$b%=pcT(IfB$-*Q)4=wG5Nj$R!nkLQR;d%3_%ZDfHzd#cY&zhvUlYfyKgX3UL}x#+)oD8PeNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|c_iC04fI^czT^vI!PAC89XEx~Z;t}oqp74S9#67V^Cl0Qa+xkg5 z;*l}4M9TmFv(GTF{#hTWB9-unXP$RdP`(kYX@0Ff`LOG}JXP z4KXmcGB&m{w9o}I3=FC*bsbPN<*DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWez-zyECZ%rY#0+$a7{ZJCf{(W4dKu9Eso-gR9&w`ZQ^ zsyP-N56re)PD@Of@Vi%woyEzxzR70O!IjVLcg?xOJj4F~qSJ=+4snR@h5qSp=sy1;zdc4%Zl`3$ldy*qZ@0PUiOcOQt#~r6{}gZ3&i^|OuDJ7IDlNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5|wbCb@B0)-BGx;Tb-98dn?&tTA_B@t@yNc7Iy;?RWWfuHhrGlmCb z@w2Y~_y2$FsUr-`zy1cWzB8D3{QrMf1BFHbAJ*CvpLmWNh{UA1Puu>df8hbHqYb7K zTxttWX#YJLpunRhDigzVqv6Yu{}Nmd3S4jgu9s$JwNDN{uth*FVg0}4;0ukWD+IZ& z)CmV%_$vn1&db zTNxW$8Jp_@83qRDL!z#rXvob^$xN%ntzjSMqYQ8e0&U1iOiImRC{pIV9Ssy>@O1Ta JS?83{1OQZFnTh}a literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_0.png b/ggml/examples/yolo/data/labels/115_0.png new file mode 100644 index 0000000000000000000000000000000000000000..4767c453c05cb3cb0de36b7dc16362eb6565f82d GIT binary patch literal 315 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3yUv?kF4ppb#5i(?4K_2eJ?j65&CM6>rs2<&zLcvyHt#af?l_6mPr z957!Q?C@*Nylv<8b$%@{VQ67+7XKP7y&kAnwZt`|BqgyV)hf9t6-Y4{85o-B8XD>v zn1&dbTNxW!8CmKA83qP>n7uSnH00)|WTsW()^IlN(^Rm7fi~nMCZ*;u6o+uVJ_{6L N@O1TaS?83{1OR-rT2%l5 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_1.png b/ggml/examples/yolo/data/labels/115_1.png new file mode 100644 index 0000000000000000000000000000000000000000..79f5d39cd69c95b7e602f5de47613efc03d475a6 GIT binary patch literal 373 zcmeAS@N?(olHy`uVBq!ia0vp^Ahs9>kYKpu8`KD-SkfJR9T^xl_H+M9WCijSl0AZa z85pY67#JE_7#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PJ_Yp!eP<2|g*0BWzEUUh4mS%1ad!63FDQy!WVN&nFyLX<|Nmc}{Nl&? z5634u>fCK@{HzynuwRWg%x`+~6bm6XhLT-d!t~Ayy#v~+TH+c}l9E`GYL#4+3Zxi} z3=GY54Gnb-OhXLJt&EMVj4gG63DwrAm)wSK?bJcwcbhK)T zYeY#(Vo9o1a#1RfVlXl=G}ARS)HN^-F)+6>HnK9Y&;>FK4DLvKNTF!R%}>cptHiBg jt=orX;E)2^kdv5{n!`{W!u9$rP>8|P)z4*}Q$iB}7krcx literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_3.png b/ggml/examples/yolo/data/labels/115_3.png new file mode 100644 index 0000000000000000000000000000000000000000..bc0d24f56d239aaa7a6738acb543678091088ca4 GIT binary patch literal 497 zcmeAS@N?(olHy`uVBq!ia0vp^!a%Ie0VEjq3%$7jq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3x*!yOBxfkMwbT^vI+&L{uq=agvMdt<-x=BVf}TSuYawxNxTEHiBM z4j;L9%isXRZK)MA_&bg#FJ@YlQr~biui?hJ{Qv*$R~~-wm1R+$uE!hAj=wc_XZm6l zHgd1X`>Dks5^UG~YMf-Blt=J48kJ_B$r|Pnv1Ezv*aNvkrUb-6t%Y*5om_=ACHB zVSC-nE_wFNl%(?vn1&dbTNxW!nOf)q83qOd y8$E#Gi=-hpKP5A*61RrLHy-AZn5QC?ypUXO@geCweq_e63 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_4.png b/ggml/examples/yolo/data/labels/115_4.png new file mode 100644 index 0000000000000000000000000000000000000000..a39b0f0c8094a685dddb1479c0471dd00549a66f GIT binary patch literal 552 zcmeAS@N?(olHy`uVBq!ia0vp^5bC!ZDuaZ1e6#3TPw!k$?^9{&q31! z-RB8H%exbW)*lqQttix=Bq90DQB@(ZfJNux8O{_g=fliz6#MiOI>R<7&FeG&|9^fb z<7Q!nb-&&+7OE`X!qIZ@H^-|x`i&Za=k~F_`&`fcMDWos^$CYxoxkRz%&5i|w|Nn> z$nv`rH5eIGgTHORQK4E33_H~l*NBpo#FA92=)}hMdHt)EtK55U$r}fkF(Pu6{1-oD!M< Dy*$V2 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_5.png b/ggml/examples/yolo/data/labels/115_5.png new file mode 100644 index 0000000000000000000000000000000000000000..63e00783a5de96c3bf0b031b4b83c2bbda9fe727 GIT binary patch literal 618 zcmeAS@N?(olHy`uVBq!ia0vp^GC=IW0VEi7v^PZ!4!i_^(J`q>S7y!vcT#%Evt;yOd}$)4Ho7kmGE zA(Yy{==S`v#-!aP3qfAek0 zBUg4ZTwlpNExDP?CnZJaSL24s%sP@>#YsQyRM!7#nK7&1RzT$7q@wv7yp}m{sj4}q zwp0F9-OSAU8(P-#o_SxBdE7wdy&}^z$JH+tqYm7^;>u+;?F9RZjh{A8;`N-BE*Di} z=ixFbEv9Fl&^2OC z7#SFv=^7g98kmL{m|GbeTNzsD0vQGd)t0&rC>nC}Q!>*kackg8nfeE)fgNN+PGVAO W4nuJW*Xy%DAqG!ZKbLh*2~7Zc0qQ6K literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_6.png b/ggml/examples/yolo/data/labels/115_6.png new file mode 100644 index 0000000000000000000000000000000000000000..344ef8493076438732bedaed260fd9b57fd77521 GIT binary patch literal 687 zcmeAS@N?(olHy`uVBq!ia0vp^3P9||0VEjy*KvvfDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tW0?{I7;U0XG%oB;*4qJ z^{G0%|NsC0vSp0LhwdNptzeldzKgr`x=0$^!BmfZH@HIsn7>D@d@Y=kuh*z`gGv0) zM}E}>+nhv89&o+d?jyi@XnVoYR?{mSON}i=&NNicwS1-&#%eDj_%Pk*sqVBFb_e@J zj^r%N-kpc!u`>HU-)gRh4 z3|cjRY>#Ple9u3LiC-n0NmRy+`Qt2sPwy=hxsncW)v-G6wPjqV!29Y1tAOyM#C@Ok z^G@5xJNfy4j=&FYVsd4Crjw=SNG9HKi*Sg2c6{1ri7oX9oIfqH5v%O1_L%t9s!=fZ z-WSEJDhv7ZjXG=1zw5g-{oJB=+pnv;u`yUP<&bEPNhdofMjCh+5=wnjKYe>w4U9t7 z64!{5l*E!$tK_0oAjM#0U}&amXsByo8e(8>Wo&F^WT6XW7#MurrnMMFLvDUbW?Cg~ k4K1%H$%7LK(1x7Eq|_XS;t;OaXMsWtp00i_>zopr0Ck-D>i_@% literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/115_7.png b/ggml/examples/yolo/data/labels/115_7.png new file mode 100644 index 0000000000000000000000000000000000000000..c6e743c0af5468aa3be94cf6fc06be42086cc12e GIT binary patch literal 766 zcmeAS@N?(olHy`uVBq!ia0vp^YCs&q0VEhs?~&R8q*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3fC?L07#My5g&JNkFq9fFFuY1&V6d9Oz#v{QXIG#NP=YDR+ueoXe|!I#{Xiaj ziKnkC`vY!PK1H>2lg^4VFfe}hba4#vIG+5&UnD^y)NEBwh~(O-e=YZyg@ymV@>c(Y z#_wGRT4ytIT{*QbA^h9cdHsy6JLVl;l(W`^*(zbxkLG1>;yIbk1ir9`N|uH4@VIK6 zGJ70w^gxNU=goN29Xp#Qzr~nrj&B+-t_;uMd0J0?O)~0QvdUFFLJMXAgFWd zpMBX;54nB)hExCkHxK2Ao%_*M%=76$GY$x731d6eT9{t6=?2V&}vPZ*bl}W4mH!NAeU3^=oY()o0 z(5!~vTY$c{u`#%rwcqK;yypj6B-S6QjcS}yP<(x^?QRPeLGFVcCw{#?-N609#;NP_ z`aFve!*$M{9`&C#u1cEQJWr=zDtXly{8|VTV1_nzopr0GcCHo=I&2$Y7bq!2I z49u;JjjW6B&Mb9V|=_PEg@ltpPl*)My5Z8 z7L~8G9F{GrG7EjP(5!tnqFCYHKDhJismdKI;Vst0FWbC?*IS* literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/116_3.png b/ggml/examples/yolo/data/labels/116_3.png new file mode 100644 index 0000000000000000000000000000000000000000..0314e85e75e0c6c22d2c0149042199407ba792e9 GIT binary patch literal 324 zcmeAS@N?(olHy`uVBq!ia0vp^{6MVD0VEij8hW<@DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tW2OC z7#SFv=^7g98kmL{m|GbeS(#es0vQGd0vkPmPDav@o1c=IR*73f;+qc^V3z}J$Vp5} X&0#1xl~bP0l+XkKR$5*F literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/116_4.png b/ggml/examples/yolo/data/labels/116_4.png new file mode 100644 index 0000000000000000000000000000000000000000..cf919a6ba13a9c69e5b0d93848f5a330b14c3334 GIT binary patch literal 324 zcmeAS@N?(olHy`uVBq!ia0vp^LO^WF0VEhQd7iKWDVB6cUq=Rpjs4tz545bDP46hOx7_4S6Fo+k-*%fF5lweBoc6VX;-`;;_Kaj^> z;_2(k{(zg6Pmx2&<-jeVkfo=KV~EE2^oO zG9@iHcP+ysLuScq{%pPS?p*>}5}Zy9Gy85`PG`H54m3oy#5JNMC9x#cD!C{XNHG{0 z7@FxC8tNLDh8UPz85>&}Sm**71_s+F>)u1rkei>9nO2Eg!?ANylfW(q+K`i&l$yg( Uaw@Mb4=BXo>FVdQ&MBb@03X>{t^fc4 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/116_5.png b/ggml/examples/yolo/data/labels/116_5.png new file mode 100644 index 0000000000000000000000000000000000000000..c9e04ed2ab4c48ed245a8954a1c460aa7be53a89 GIT binary patch literal 328 zcmeAS@N?(olHy`uVBq!ia0vp^qCo7x0VEhM*eCM=DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWEs{%yaqf)7p2c6Y!D0mV#O@6MlNm9X~_!* zYNER23fP&XI~o`}_GWKrIlSUUL+7IAz`(3dCI)4>fA&224ab27sg}4#l%ynNn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5`;j(H->fkKX+E{-778T_s%9mPK^p0s$9a*&X=#8jE7Do#QSk#%1laP2t212j>! z#5JNMC9x#cD!C{XNHG{07@FxC8tNLDh8UPz85>&}Tj&BA1_qWgTDB+}a`RI%(<*Um j@Nk^e2=)fhhMdHt)EtJAQ+aiHKp_TCS3j3^P6_E)T0VEiv{XS6wq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz3x}|J^R9fI|A7E{-7_*OPznGxEInlJxxl|E=cAFP8fK|DXN#&-sF& zPgj-i{dOqb*`)L};)`P@gQm8_agE4rUx8XxOI#yLQW8s2t&)pUffR$0fuWhMp`osU zX^4Tjm9dePv4Jj-VPNpokUs@QLvDUbW?Cg~4O4Qq-vGN8XhTk7QfdxE>E2AgA3z}n MPgg&ebxsLQ0H|(X#Q*>R literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_1.png b/ggml/examples/yolo/data/labels/117_1.png new file mode 100644 index 0000000000000000000000000000000000000000..59cae8a9830e32de09632c6bc97548b6d4e54e3a GIT binary patch literal 350 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4J<=E3ywjKs3W3T??9X8-@#&2D^X*75yzPh(@}#70HIN7MLKSG|$3TDyP63uO^G zhF{8W?i^XN+ZkxQYKdz^NlIc#s#S7PDv)9@GB7mLH8j*UFby#45bDP46hP^x@Isih!@P+6=(yLU`q0KcVYP7-hXC4kjGx) z>Fdh=fSZ+1!7wDMaxze;+|$J|MB{w&kA6mpwh0%C+1T1X3zS(iH{0B86>)I#4{R}U zaEfnm+O6Ppl!s?p!gX(BgA&IU8%c>t2c0(Z@C2=QR9Y?}DKV{~wU4Q^>)DsmqlJtw z^#$^u8y#SH%$4Es=e0;vhsIiGmUI2imv398&;M)Pk;K7}@Z-vL{S#Af0iB>);u=ws zl30>zm0Xkxq!^4049#>64RsAnLk!HVjE$^JEOmhl1B0p@=XEF=a`RI%(<*Um;5PgX g3Ql&A4LONPsW}X#do%rh0EHMlUHx3vIVCg!05n&0GXMYp literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_3.png b/ggml/examples/yolo/data/labels/117_3.png new file mode 100644 index 0000000000000000000000000000000000000000..18679c3975b37da13835d3dd31fe167846bab46c GIT binary patch literal 421 zcmeAS@N?(olHy`uVBq!ia0vp^5Y!S$?}@oZBWtXmGQD+`L>XVszlQB*AOACE=?rwyCErE<5qvg_OTmQn$M zYR8*13jd|N{j%*`2OC7#SFv z=^7g98kmL{m|GbeS(#es0vQGd0vkPmVS=O~H$NpatrE9}#5W%-z(E7FAtx~@HHV>e TZ>HZ5pb&$ntDnm{r-UW|`ZR*S literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_4.png b/ggml/examples/yolo/data/labels/117_4.png new file mode 100644 index 0000000000000000000000000000000000000000..bceedf94d925066c0afe60090bb9c4c7e049d0f5 GIT binary patch literal 470 zcmeAS@N?(olHy`uVBq!ia0vp^vOsLg0VEh^u;wz1;WmD*I6WYlyl<-1s>e5L$ z{4QalyWfa(tdZ-GZB?n}*V!$gl+B&=Tk&KZ+v#=fC)%2Bw0UgWeQ?usg-y8%X0P2_ za@ai@-io`3Eh-U;`6Pcsc-e(5AH7+Q2=8H={FZt3yQZb9YYf5{yfvHf{d0uOjN_j; z)PDEOXJp#3#3gdU8JUPxzyH_sF~)DmVZCUNiB@4CVR%+$aA1 z|375+^fn9W?uIar|ISM$NG_=?xN(w+q2b*5`Cm9EI|IF?TH+c}l9E`GYL#4+3Zxi} z3=GY54Gnb-OhXLJt&EMW3@mhk3M_YsY&3l1lo|3n3S5s VP`WqM?*~wb!PC{xWt~$(69C*xrxySK literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_5.png b/ggml/examples/yolo/data/labels/117_5.png new file mode 100644 index 0000000000000000000000000000000000000000..c0bddab6e857f4a905e5e20eab8d64276e08ba59 GIT binary patch literal 515 zcmeAS@N?(olHy`uVBq!ia0vp^N|XzF65*^ zP!U(ZPD{r9 zQ$Y29=>Z#Q5!Ky_Uf&$29%-)h@t-jDiQ|+#jaoCgA1+h8t;`{2^pijI(?sqw>2tWk zKc19aw3>a{lX@AKhJ&S#ttX_f^PkZEA#(nKsdt{XtV&FHGJSeNd!acptHiB=D`n~*payo34LONPsW}X#do%rh Q0EHMlUHx3vIVCg!0LY`c0RR91 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_6.png b/ggml/examples/yolo/data/labels/117_6.png new file mode 100644 index 0000000000000000000000000000000000000000..731fd1448d568383400162b42f43ab2e0b24dbcb GIT binary patch literal 549 zcmeAS@N?(olHy`uVBq!ia0vp^8bIvD0VEjStn_>Uq*&4&eH|GXHuiJ>Nn{1`6_P!I zd>I(3)EF2VS{N990fib~Fff!FFfhDIU|_JC!N4G1FlSew4N!t9$=lt9;eUJonf*W> zdx@v7EBga(Rz5`;j(H->85kI)JY5_^JdP*-@K;HY2(?&M5Te*CB4+GxptwXbd&QCG zfvt|zqF zLyo*L-?U5D;Nt&+R{xm`lj7zJy!!URnf12-*Z*qAqY^Ez{TqC1cO7dN)H?Ey$Ma#O z7@I|cVCb}GZUvHB|CBZV#2;$l_NZ|R{1M9EZnez!f1t>}`o37z(|7)7aQ&}8n{X&( zdC>#=wnP8_>#G(m*1A0J-(P#P*8l(i--vs@UjNw1Bd4_U=URMatCErwx_W%Yp}agzZe?t2Wn`fXWEdEH x-KMn|MMG|WN@iLmZVfH3C&`1O253W0Vp3`jL+Rd3zaKy$22WQ%mvv4FO#sVF!<7I4 literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/117_7.png b/ggml/examples/yolo/data/labels/117_7.png new file mode 100644 index 0000000000000000000000000000000000000000..c04ed0b1256b450aaa7948e740bf31a3c67a6b76 GIT binary patch literal 584 zcmeAS@N?(olHy`uVBq!ia0vp^dO#e)0VEi1WJmb`DVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tWRs&C1U zYCh&=>m8o%){B{Qu=;0k$1P>vM+J`G(lt(*8z?X{y4IQ&ilt1v^IzlC|J}w9n3dLF zoMJD0ba~CydXuOB&wqFncK3t4^Uwd?UKS$8-~S(9^ymL^r?8w=zUTfNU-jqze|6JS z8CjNp$`+_|PpSI)>y1vhZ=#-BSX}1i2gh6JK43pfMKgz z;u=wsl30>zm0Xkxq!^4049#>64RsAnLk!HVjE${~Ep&kl0|QGLEn5@~x%nxXX_dG& jcsNdK1jioGhMdHt)EtJ=y_tSLfIy{8|VTV1_n@~ literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/118_1.png b/ggml/examples/yolo/data/labels/118_1.png new file mode 100644 index 0000000000000000000000000000000000000000..fa886ac2f4f6a4dc392f332c93f865703e936f0a GIT binary patch literal 369 zcmeAS@N?(olHy`uVBq!ia0vp@KrF@qBpB=_dj0@XEa{HEjtmSN`?>!lvI6-E$sR$z z3=CCj3=9n|3=F@3LJcn%7)lKo7+xhXFj&oCU=S~uvn$XBD8ZEE?e4kr_Wm>bfjssS zPhVH|2i&ZD3Wgz3m6L%&>pfi@Lp07O|LEtGXj>E^#oYYEpJR7RW8-^Wk2iMQ%!ipa zTnKk$Ys>1m=fcM3yy1op8{56@0voN*@bIuZ_{6fcdCidB$ivgQscr8|CQ6&-d*R|xY~#3&wVXPiBl8q%A7i|!D&Y0YtcSv5w&I)(RSl!mS;OB{vv64!@7eUs~0?zm0Xkxq!^4049#>64RsAnLk!HVjE$^JEOmhl1B0p@=XEF=a`RI% n(<*Um;5Pg%1P(8t4LONPsW}W~DhcvVKp_TCS3j3^P6_(xD4tILwcKt7rl$aqY!D64p z^4n{|cga}>j~}lW3yAr^Jgp;&Yf8ouvF7?K5-fHJlYi`QSSVLL=f7E1zp~*TMWd-a z!ki2$A1cn+tDHUu^tNh=YeY#(Vo9o1a#1RfVlXl=G}ARS)HN^-F)+6>HnK9c)CDpO z47MF}1_msWhTQy=%(P0}8jjEY-2)D5pba^RNvSyuWhx2sPCy|BPgg&ebxsLQ0F!T| AzyJUM literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/118_4.png b/ggml/examples/yolo/data/labels/118_4.png new file mode 100644 index 0000000000000000000000000000000000000000..93991521924b8c3224489ca21a621365ae3a0586 GIT binary patch literal 576 zcmeAS@N?(olHy`uVBq!ia0vp^azJd#0VEiT4OqH?6id3JuOkD)#(wTUiL5}rLb6AY zF9SoB8UsT^3j@P1pisjL28L1t28LG&3=CE?7#PG0=Ijcz0ZK3>dAqwX{BQ3+vmeM~ zFY)wsWq-iU%BRR7#cYk4*cTre8b7@;qlWv z;(L+6?1gNeyQFlCES&pPx- zz6ksjE2OgL>wiuG)!)L($9_3Ha{Ry8;fl!_j=ySbWiryM(p+lF8^3IA=vaAU(qVDI z1w9M`-v1A%{oOy6kx4Dt-)BC{!A%;oCSHzGj;Oj+FLO!$N8_fT`-Ps($;prE=UjSU zA*r(e@w9*OoPDBmF0BvU_i*yR|1a$0UjAQv>BM_YsY&2i d1KN<2n3S5sP^OX~?*tTL@O1TaS?83{1ORcg*}?z- literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/118_5.png b/ggml/examples/yolo/data/labels/118_5.png new file mode 100644 index 0000000000000000000000000000000000000000..279c954365ddf40c62623842b47dbfad798bbb41 GIT binary patch literal 653 zcmeAS@N?(olHy`uVBq!ia0vp^DnRVO0VEj0`MN+fOS+@4BLl<6e(pbstU$g(vPY0F z14ES>14Ba#1H&(%P{RubhEf9thF1v;3|2E37{m+a>}0Wrb5XX~q(w3v z7uZ@B@iQd|%C^k%-;*HNT<_r+z0fd0pp)$0mQ0bi60ULOynLx? zZ6WgR&wbXg#XrSoFZzEo;pDFWaS2s7>(_M2ZEjRqd%yYb`j6tf7yW;!5V4FD5?)FK#IZ0z|c(B&`{UFG{nH%%GlV- z&_Wl;FfgdL)OA47kei>9nO2Eg16Rt_KR^xaARBTLlTvdS%2X2Moq$3Np00i_>zopr E00B7pod5s; literal 0 HcmV?d00001 diff --git a/ggml/examples/yolo/data/labels/118_6.png b/ggml/examples/yolo/data/labels/118_6.png new file mode 100644 index 0000000000000000000000000000000000000000..9379cb994f2e170c2dac1bbb6882882b59855c33 GIT binary patch literal 682 zcmeAS@N?(olHy`uVBq!ia0vp^nn3Ku0VEihvz6`vDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~LMH3o);76yi2K%s^g3=E|P3=FRl7#OT(FffQ0%-I!a1C(G&@^*J&_}|`tW?Eavh-$l6k-3+sW$7-xkF9@YlL~+OkA48 z9a(i*beXg|B9@#G->`Ax#*f?)+7&KMfi@g$ohuG1DMfTNDR>14Y!q_2n||l}w|&dJ zwg#s>|9iiG=9@D{4`k0wU00H7ntJnmfe7zjmeRJ2vm#eHZFPRHD0cj`k}v!HW