initial release
This commit is contained in:
civ
+200
@@ -0,0 +1,200 @@
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
project(omnivoice-ggml LANGUAGES C CXX)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# version.h: embed git commit hash into all binaries.
|
||||
# runs on every build, only rewrites if the hash changed.
|
||||
set(VERSION_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/version.h")
|
||||
add_custom_target(version ALL
|
||||
COMMAND "${CMAKE_COMMAND}" "-DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DOUTPUT=${VERSION_OUTPUT}"
|
||||
-P "${CMAKE_CURRENT_SOURCE_DIR}/tools/version.cmake"
|
||||
BYPRODUCTS "${VERSION_OUTPUT}"
|
||||
COMMENT "Checking git version"
|
||||
)
|
||||
|
||||
# pthread: required explicitly on older glibc (< 2.34) where libpthread
|
||||
# is not merged into libc. Modern distros link it implicitly but aarch64
|
||||
# and older x86_64 toolchains need the explicit dependency.
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# Suppress MSVC fopen/sprintf deprecation warnings, force UTF-8 source and
|
||||
# execution charsets so non-ASCII string literals (CJK in voice-design.h,
|
||||
# lang-map.h, text-chunker.h) survive the compile without a BOM. /utf-8 is
|
||||
# restricted to C and C++ since nvcc treats a bare /utf-8 as an input
|
||||
# filename and aborts with "A single input file is required". CUDA sources
|
||||
# do not carry CJK literals so they do not need this flag.
|
||||
if(MSVC)
|
||||
add_compile_definitions(_CRT_SECURE_NO_WARNINGS)
|
||||
add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/utf-8>)
|
||||
endif()
|
||||
|
||||
# Put executables and backend .so in the same directory (build root).
|
||||
# Without this, ggml defaults to bin/ for .so but executables stay in root,
|
||||
# and ggml_backend_load_all() can't find the backends at runtime.
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
|
||||
# Audio tokenizer tensor names can exceed default GGML_MAX_NAME of 64
|
||||
add_compile_definitions(GGML_MAX_NAME=128)
|
||||
|
||||
# Harden: mark fread/fwrite/etc with warn_unused_result on all platforms
|
||||
# SYCL excluded: _FORTIFY_SOURCE swaps memcpy for __memcpy_chk, unresolved in device code
|
||||
if(NOT MSVC AND NOT GGML_SYCL)
|
||||
add_compile_definitions(_FORTIFY_SOURCE=2)
|
||||
endif()
|
||||
|
||||
# CUDA architectures: cover Turing to Blackwell for distributed binaries.
|
||||
# Users can override with -DCMAKE_CUDA_ARCHITECTURES=native for local builds.
|
||||
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
|
||||
find_package(CUDAToolkit QUIET)
|
||||
if(CUDAToolkit_FOUND AND CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8")
|
||||
set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real;120a-real;121a-real")
|
||||
else()
|
||||
set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ggml as subdirectory, inherits GGML_CUDA, GGML_METAL, etc. from cmake flags
|
||||
# CUDA graphs default on: standalone ggml ships them off. Overridable with
|
||||
# -DGGML_CUDA_GRAPHS=OFF or at runtime with GGML_CUDA_DISABLE_GRAPHS=1.
|
||||
if(NOT DEFINED GGML_CUDA_GRAPHS)
|
||||
set(GGML_CUDA_GRAPHS_DEFAULT ON)
|
||||
endif()
|
||||
add_subdirectory(ggml)
|
||||
|
||||
# cpp-httplib (HTTP server library, no SSL, behind reverse proxy in prod).
|
||||
# Used by tts-server.
|
||||
add_subdirectory(vendor/cpp-httplib)
|
||||
|
||||
# yyjson (MIT, fast JSON parser/writer). Used by tts-server.
|
||||
add_library(yyjson STATIC vendor/yyjson/yyjson.c)
|
||||
target_include_directories(yyjson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/yyjson)
|
||||
if(MSVC)
|
||||
target_compile_options(yyjson PRIVATE /W0)
|
||||
else()
|
||||
target_compile_options(yyjson PRIVATE -w)
|
||||
endif()
|
||||
|
||||
# Shared compile options and ggml linkage
|
||||
macro(link_ggml_backends target)
|
||||
target_include_directories(${target} PRIVATE
|
||||
${CMAKE_SOURCE_DIR}/src
|
||||
${CMAKE_SOURCE_DIR}
|
||||
${CMAKE_BINARY_DIR}
|
||||
)
|
||||
target_include_directories(${target} SYSTEM PRIVATE
|
||||
${CMAKE_SOURCE_DIR}/ggml/include
|
||||
)
|
||||
if(MSVC)
|
||||
target_compile_options(${target} PRIVATE /W4 /wd4100 /wd4505)
|
||||
else()
|
||||
target_compile_options(${target} PRIVATE -Wall -Wextra -Wshadow -Wconversion
|
||||
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
|
||||
endif()
|
||||
target_link_libraries(${target} PRIVATE ggml Threads::Threads)
|
||||
if(TARGET ggml-base)
|
||||
target_link_libraries(${target} PRIVATE ggml-base)
|
||||
endif()
|
||||
foreach(backend cpu blas cuda metal vulkan sycl)
|
||||
if(TARGET ggml-${backend})
|
||||
get_target_property(CURRENT_BACKEND_TYPE ggml-${backend} TYPE)
|
||||
if (CURRENT_BACKEND_TYPE STREQUAL "MODULE_LIBRARY")
|
||||
# DL mode: backend is loaded at runtime via dlopen,
|
||||
# skip all link-time deps.
|
||||
continue()
|
||||
endif()
|
||||
target_link_libraries(${target} PRIVATE ggml-${backend})
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
# SYCL links its runtime PRIVATE, consumers need -fsycl to resolve libsycl.so
|
||||
if(TARGET ggml-sycl AND GGML_SYCL)
|
||||
target_link_options(${target} PRIVATE -fsycl)
|
||||
endif()
|
||||
add_dependencies(${target} version)
|
||||
endmacro()
|
||||
|
||||
# Core library always STATIC : the bundled CLI tools include
|
||||
# pipeline-tts.h / pipeline-codec.h / backend.h directly for the debug
|
||||
# paths (--llm-test, --maskgit-test) and need every pipeline_* / backend_*
|
||||
# symbol resolved without going through the public ABI. OMNIVOICE_STATIC
|
||||
# is propagated PUBLIC : the lib's own .cpp files see it (so OV_API
|
||||
# resolves to empty when compiling omnivoice.cpp on Windows), and every
|
||||
# consumer that links omnivoice-core inherits it too (same effect on
|
||||
# their side, no spurious dllimport on a static archive). The shared
|
||||
# library for ABI consumers is a separate, opt-in target below.
|
||||
add_library(omnivoice-core STATIC
|
||||
src/omnivoice.cpp
|
||||
src/pipeline-codec.cpp
|
||||
src/pipeline-tts.cpp
|
||||
)
|
||||
target_compile_definitions(omnivoice-core PUBLIC OMNIVOICE_STATIC)
|
||||
link_ggml_backends(omnivoice-core)
|
||||
|
||||
# Public shared library for ABI consumers (Python ctypes, Rust bindgen,
|
||||
# Go cgo). Opt-in : -DOMNIVOICE_SHARED=ON at configure time. Exports
|
||||
# only the OV_API-marked symbols ; every internal pipeline_* / backend_*
|
||||
# stays hidden inside the .so. Intentionally a different target name
|
||||
# from omnivoice-core so the static path used by the tools is never
|
||||
# affected.
|
||||
option(OMNIVOICE_SHARED "Build the shared omnivoice library for ABI consumers" OFF)
|
||||
if(OMNIVOICE_SHARED)
|
||||
add_library(omnivoice SHARED
|
||||
src/omnivoice.cpp
|
||||
src/pipeline-codec.cpp
|
||||
src/pipeline-tts.cpp
|
||||
)
|
||||
target_compile_definitions(omnivoice PRIVATE OMNIVOICE_BUILD)
|
||||
set_target_properties(omnivoice PROPERTIES
|
||||
C_VISIBILITY_PRESET hidden
|
||||
CXX_VISIBILITY_PRESET hidden
|
||||
VISIBILITY_INLINES_HIDDEN ON
|
||||
)
|
||||
link_ggml_backends(omnivoice)
|
||||
endif()
|
||||
|
||||
# quantize: GGUF requantizer (BF16 -> K-quants), shared with acestep.cpp policy
|
||||
add_executable(quantize tools/quantize.cpp)
|
||||
link_ggml_backends(quantize)
|
||||
|
||||
# omnivoice-codec : standalone codec CLI (codes <-> WAV)
|
||||
add_executable(omnivoice-codec tools/omnivoice-codec.cpp)
|
||||
target_link_libraries(omnivoice-codec PRIVATE omnivoice-core)
|
||||
link_ggml_backends(omnivoice-codec)
|
||||
|
||||
# omnivoice-tts : full TTS pipeline. Grows phase by phase, currently load-only.
|
||||
add_executable(omnivoice-tts tools/omnivoice-tts.cpp)
|
||||
target_link_libraries(omnivoice-tts PRIVATE omnivoice-core)
|
||||
link_ggml_backends(omnivoice-tts)
|
||||
|
||||
# tts-server : OpenAI-compatible HTTP server over the TTS pipeline.
|
||||
# Always built : it is part of the project, not an optional add-on.
|
||||
add_executable(tts-server tools/tts-server.cpp)
|
||||
target_link_libraries(tts-server PRIVATE omnivoice-core httplib yyjson)
|
||||
link_ggml_backends(tts-server)
|
||||
|
||||
# test-abi-c : pure C99 smoke test that locks in the public ABI contract.
|
||||
# Compiles omnivoice.h with a C compiler under -Wall -Werror -pedantic and
|
||||
# links against the static lib. The test never loads a model ; failure
|
||||
# means the public API regressed. Built by default so a regression breaks
|
||||
# the main build, not just an opt-in target.
|
||||
add_executable(test-abi-c tests/abi-c.c)
|
||||
set_target_properties(test-abi-c PROPERTIES
|
||||
C_STANDARD 99
|
||||
C_STANDARD_REQUIRED ON
|
||||
C_EXTENSIONS OFF
|
||||
)
|
||||
if(MSVC)
|
||||
target_compile_options(test-abi-c PRIVATE /W4 /WX)
|
||||
else()
|
||||
target_compile_options(test-abi-c PRIVATE -Wall -Werror -pedantic)
|
||||
endif()
|
||||
target_include_directories(test-abi-c PRIVATE
|
||||
${CMAKE_SOURCE_DIR}/src
|
||||
${CMAKE_BINARY_DIR}
|
||||
)
|
||||
target_link_libraries(test-abi-c PRIVATE omnivoice-core)
|
||||
link_ggml_backends(test-abi-c)
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
# Используем базовый образ Debian Trixie (13)
|
||||
FROM debian:trixie-slim
|
||||
|
||||
# Отключаем интерактивные запросы при установке пакетов
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Обновляем список пакетов и устанавливаем необходимые зависимости
|
||||
RUN apt-get update && apt-get install -y \
|
||||
# Базовые утилиты
|
||||
wget \
|
||||
curl \
|
||||
# вообще нахуй не надо, но пусть будет
|
||||
gnupg \
|
||||
lsb-release \
|
||||
# Vulkan Loader и инструменты
|
||||
libvulkan1 \
|
||||
vulkan-tools \
|
||||
# Драйверы Mesa (LLVMpipe для программного рендеринга)
|
||||
mesa-vulkan-drivers \
|
||||
# Заголовочные файлы для разработки (опционально)
|
||||
libvulkan-dev \
|
||||
# Дополнительные утилиты
|
||||
mesa-utils \
|
||||
# Библиотека без которой ничего не работает
|
||||
libgomp1 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Устанавливаем переменную окружения для Vulkan Loader
|
||||
ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/radeon_icd.json
|
||||
#ENV VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/lvp_icd.x86_64.json
|
||||
|
||||
# Опционально: копируем ваше приложение в контейнер
|
||||
COPY ./app/tts-server /app/
|
||||
COPY ./app/*so* /usr/local/lib/
|
||||
|
||||
RUN ldconfig && \
|
||||
echo "/usr/local/lib" > /etc/ld.so.conf.d/custom-libs.conf && \
|
||||
ldconfig
|
||||
ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
|
||||
|
||||
WORKDIR /app
|
||||
# Команда по умолчанию (замените на свою)
|
||||
CMD ["/app/tts-server"]
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023-2026 The omnivoice.cpp authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,128 @@
|
||||
# omnivoice.cpp
|
||||
|
||||
Local AI text-to-speech with voice cloning and voice design, powered
|
||||
by GGML. C++17 port of OmniVoice (k2-fsa/OmniVoice). 646 languages,
|
||||
24 kHz mono output, runs on CPU, CUDA, ROCm, Metal, Vulkan.
|
||||
|
||||
## Features
|
||||
|
||||
- Voice cloning from a reference WAV plus its transcript
|
||||
- Voice design via attribute keywords (gender, age, pitch, style,
|
||||
volume, emotion)
|
||||
- Auto voice with consistent speaker identity across long inputs
|
||||
- Long-form synthesis with punctuation-aware text chunking, voice
|
||||
prompt promotion, cross-fade and pydub-strict silence removal
|
||||
- Bit deterministic generation in greedy mode, seedable Philox PRNG
|
||||
for stochastic sampling
|
||||
- Q8_0 quantisation of the 612 M parameter Qwen3 backbone
|
||||
- Two CLI tools : `omnivoice-tts` (text -> WAV) and `omnivoice-codec`
|
||||
(WAV <-> RVQ codes)
|
||||
|
||||
## Build
|
||||
|
||||
```
|
||||
git clone --recurse-submodules https://github.com/ServeurpersoCom/omnivoice.cpp.git
|
||||
cd omnivoice.cpp
|
||||
./buildcuda.sh # NVIDIA GPU
|
||||
./buildvulkan.sh # AMD/Intel GPU (Vulkan)
|
||||
./buildcpu.sh # CPU only
|
||||
./buildall.sh # all backends, runtime DL loading
|
||||
NVCC_CCBIN=g++-13 ./buildcuda.sh # rolling release distros (Arch w/ GCC 16, etc.)
|
||||
```
|
||||
|
||||
## Model conversion
|
||||
|
||||
Pre-converted GGUFs are available on Hugging Face :
|
||||
|
||||
https://huggingface.co/Serveurperso/OmniVoice-GGUF
|
||||
|
||||
Drop them in `models/` and skip to the quick start. To convert from
|
||||
the original checkpoint :
|
||||
|
||||
```
|
||||
./checkpoints.sh # hf download k2-fsa/OmniVoice -> checkpoints/
|
||||
./convert.py # 2 GGUFs in BF16 -> models/
|
||||
./quantize.sh # base LM Q8_0 (tokenizer stays at native dtype)
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```
|
||||
echo "Hello world." | ./build/omnivoice-tts \
|
||||
--model models/omnivoice-base-Q8_0.gguf \
|
||||
--codec models/omnivoice-tokenizer-F32.gguf \
|
||||
--lang English -o hello.wav
|
||||
```
|
||||
|
||||
Voice cloning :
|
||||
|
||||
```
|
||||
./build/omnivoice-tts \
|
||||
--model models/omnivoice-base-Q8_0.gguf \
|
||||
--codec models/omnivoice-tokenizer-F32.gguf \
|
||||
--ref-wav ref.wav --ref-text ref.txt \
|
||||
--lang English -o out.wav < prompt.txt
|
||||
```
|
||||
|
||||
Pre-encoded reference (`clone.sh`): `omnivoice-codec` encodes a reference WAV
|
||||
into a compact `.rvq` latent (it applies the exact TTS reference
|
||||
preprocessing, so the codes are bit-identical to the `--ref-wav` path).
|
||||
Passing it via `--ref-rvq` skips the codec encode on every synthesis:
|
||||
|
||||
```
|
||||
build/omnivoice-codec --model models/omnivoice-tokenizer-F32.gguf -i ref.wav
|
||||
build/omnivoice-tts \
|
||||
--model models/omnivoice-base-Q8_0.gguf \
|
||||
--codec models/omnivoice-tokenizer-F32.gguf \
|
||||
--ref-rvq ref.rvq --ref-text ref.txt \
|
||||
--lang English -o out.wav < prompt.txt
|
||||
```
|
||||
|
||||
## Embedding the library
|
||||
|
||||
The CLI tools are thin wrappers over a public ABI. Single-header,
|
||||
single-name-prefix, plain C linkage so that C, C++, Python ctypes,
|
||||
Rust bindgen and Go cgo all consume it the same way.
|
||||
|
||||
```c
|
||||
#include "omnivoice.h"
|
||||
|
||||
struct ov_init_params iparams;
|
||||
ov_init_default_params(&iparams);
|
||||
iparams.model_path = "models/omnivoice-base-Q8_0.gguf";
|
||||
iparams.codec_path = "models/omnivoice-tokenizer-F32.gguf";
|
||||
|
||||
struct ov_context * ov = ov_init(&iparams);
|
||||
|
||||
struct ov_tts_params params;
|
||||
ov_tts_default_params(¶ms);
|
||||
params.text = "Hello world.";
|
||||
params.lang = "English";
|
||||
|
||||
struct ov_audio audio = { 0 };
|
||||
ov_synthesize(ov, ¶ms, &audio);
|
||||
/* audio.samples, audio.n_samples, audio.sample_rate, audio.channels */
|
||||
ov_audio_free(&audio);
|
||||
ov_free(ov);
|
||||
```
|
||||
|
||||
`tests/abi-c.c` is built with `-std=c99 -Wall -Werror -pedantic` on
|
||||
every build, so any regression that breaks plain C consumability fails
|
||||
the build, not just an opt-in target.
|
||||
|
||||
For a binding-friendly shared library (libomnivoice.so / .dll / .dylib),
|
||||
configure with `cmake -DOMNIVOICE_SHARED=ON ...`. The shared target
|
||||
exports only the `ov_*` symbols ; every internal `pipeline_*` and
|
||||
`backend_*` stays hidden inside the .so.
|
||||
|
||||
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the model, the
|
||||
GGUF layout, the inference pipeline, every CLI flag, the public API
|
||||
reference and the validation results.
|
||||
|
||||
## License
|
||||
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
|
||||
Upstream model : OmniVoice by Xiaomi / k2-fsa, Apache 2.0.
|
||||
Audio codec : Higgs Audio v2 (`bosonai/higgs-audio-v2-tokenizer`),
|
||||
Apache 2.0.
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
|
||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
|
||||
|
||||
rem rd /s /q build 2>nul
|
||||
mkdir build 2>nul
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON
|
||||
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
|
||||
|
||||
cd ..
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
|
||||
export PATH=/usr/local/cuda/bin:$PATH
|
||||
|
||||
cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_BLAS=ON
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
|
||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
|
||||
|
||||
rem rd /s /q build 2>nul
|
||||
mkdir build 2>nul
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_CUDA=ON
|
||||
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
|
||||
|
||||
cd ..
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf build
|
||||
mkdir build
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_BLAS=ON -DBLAS_INCLUDE_DIRS=$PREFIX/include/openblas
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
|
||||
call "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"
|
||||
|
||||
rem rd /s /q build 2>nul
|
||||
mkdir build 2>nul
|
||||
cd build
|
||||
|
||||
cmake .. -DGGML_VULKAN=ON
|
||||
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
|
||||
|
||||
cd ..
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
rm -rf app
|
||||
mkdir app
|
||||
cd app
|
||||
|
||||
cmake .. -DGGML_VULKAN=ON
|
||||
cmake --build . --config Release -j "$(nproc)"
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
# Download OmniVoice checkpoint from HuggingFace.
|
||||
# Usage: ./checkpoints.sh
|
||||
|
||||
set -eu
|
||||
|
||||
DIR="checkpoints"
|
||||
mkdir -p "$DIR"
|
||||
|
||||
HF="hf download --quiet"
|
||||
|
||||
dl_repo() {
|
||||
local name="$1" repo="$2"
|
||||
local target="$DIR/$name"
|
||||
if [ -d "$target" ] && [ "$(ls "$target"/*.safetensors 2>/dev/null | wc -l)" -gt 0 ]; then
|
||||
echo "[OK] $name"
|
||||
return
|
||||
fi
|
||||
echo "[Download] $name <- $repo"
|
||||
$HF "$repo" --local-dir "$target"
|
||||
}
|
||||
|
||||
# k2-fsa/OmniVoice : LLM core (Qwen3 0.6B + audio_emb + audio_heads)
|
||||
# audio_tokenizer/ subdir (HuBERT + DAC + RVQ + fc/fc2)
|
||||
# tokenizer.json + chat_template.jinja
|
||||
dl_repo "OmniVoice" "k2-fsa/OmniVoice"
|
||||
Executable
+329
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
# convert.py: safetensors to GGUF for OmniVoice (LM + audio tokenizer).
|
||||
# Reads from checkpoints/OmniVoice/, writes 2 byte-perfect F32 GGUFs to models/.
|
||||
# The source checkpoint is 100% F32 (k2-fsa/OmniVoice on Hugging Face),
|
||||
# so this converter never downcasts : every tensor is written in its native
|
||||
# source dtype and quantize.sh derives BF16 / Q8_0 from these F32 GGUFs.
|
||||
#
|
||||
# omnivoice-base-F32.gguf Qwen3 0.6B + audio_embeddings + audio_heads + tokenizer
|
||||
# omnivoice-tokenizer-F32.gguf HuBERT + DAC encoder/decoder + RVQ + fc/fc2
|
||||
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import gguf
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
CHECKPOINT_DIR = os.path.join(SCRIPT_DIR, "checkpoints", "OmniVoice")
|
||||
OUTPUT_DIR = os.path.join(SCRIPT_DIR, "models")
|
||||
|
||||
def log(tag, msg):
|
||||
print("[%s] %s" % (tag, msg), file=sys.stderr, flush=True)
|
||||
|
||||
# safetensors header reader
|
||||
def read_sf_header(path):
|
||||
with open(path, "rb") as f:
|
||||
n = struct.unpack("<Q", f.read(8))[0]
|
||||
meta = json.loads(f.read(n))
|
||||
meta.pop("__metadata__", None)
|
||||
return meta, 8 + n
|
||||
|
||||
# read a tensor from safetensors as a numpy view of the right dtype
|
||||
def read_sf_tensor(sf_path, hdr_size, info):
|
||||
off0, off1 = info["data_offsets"]
|
||||
nbytes = off1 - off0
|
||||
with open(sf_path, "rb") as f:
|
||||
f.seek(hdr_size + off0)
|
||||
raw = f.read(nbytes)
|
||||
dtype_str = info["dtype"]
|
||||
shape = info["shape"]
|
||||
if dtype_str == "F32":
|
||||
return np.frombuffer(raw, dtype=np.float32).reshape(shape).copy()
|
||||
if dtype_str == "BF16":
|
||||
u16 = np.frombuffer(raw, dtype=np.uint16).reshape(shape)
|
||||
return (u16.astype(np.uint32) << 16).view(np.float32).copy()
|
||||
if dtype_str == "F16":
|
||||
return np.frombuffer(raw, dtype=np.float16).reshape(shape).astype(np.float32).copy()
|
||||
raise RuntimeError("unsupported dtype %s" % dtype_str)
|
||||
|
||||
# write a tensor to GGUF preserving the source dtype : F32 stays F32, BF16 stays
|
||||
# BF16, F16 stays F16. No cast, no precision invented or destroyed. The OmniVoice
|
||||
# checkpoint is integrally F32 in the reference release ; the BF16 / F16 branches
|
||||
# stay as defensive code in case a future release switches dtype upstream.
|
||||
def add_tensor_passthrough(w, name, sf_path, hdr_size, info):
|
||||
dtype_str = info["dtype"]
|
||||
shape = info["shape"]
|
||||
off0, off1 = info["data_offsets"]
|
||||
nbytes = off1 - off0
|
||||
with open(sf_path, "rb") as f:
|
||||
f.seek(hdr_size + off0)
|
||||
raw = f.read(nbytes)
|
||||
if dtype_str == "F32":
|
||||
arr = np.frombuffer(raw, dtype=np.float32).reshape(shape)
|
||||
w.add_tensor(name, arr)
|
||||
return nbytes
|
||||
if dtype_str == "BF16":
|
||||
arr = np.frombuffer(raw, dtype=np.uint16).reshape(shape)
|
||||
w.add_tensor(name, arr, raw_dtype=gguf.GGMLQuantizationType.BF16)
|
||||
return nbytes
|
||||
if dtype_str == "F16":
|
||||
arr = np.frombuffer(raw, dtype=np.float16).reshape(shape)
|
||||
w.add_tensor(name, arr)
|
||||
return nbytes
|
||||
raise RuntimeError("unsupported dtype %s for %s" % (dtype_str, name))
|
||||
|
||||
# BPE tokenizer: extract vocab + merges + added_tokens from tokenizer.json
|
||||
def add_bpe_tokenizer(w, tokenizer_json_path, tag):
|
||||
with open(tokenizer_json_path, "r", encoding="utf-8") as f:
|
||||
tj = json.load(f)
|
||||
|
||||
model = tj.get("model", {})
|
||||
if model.get("type") != "BPE":
|
||||
raise RuntimeError("expected BPE tokenizer, got %s" % model.get("type"))
|
||||
|
||||
vocab_dict = model.get("vocab", {})
|
||||
added_tokens = tj.get("added_tokens", [])
|
||||
|
||||
max_id = max(
|
||||
max(vocab_dict.values(), default=-1),
|
||||
max((a["id"] for a in added_tokens), default=-1),
|
||||
)
|
||||
tokens = [""] * (max_id + 1)
|
||||
for tok_str, tok_id in vocab_dict.items():
|
||||
if 0 <= tok_id <= max_id:
|
||||
tokens[tok_id] = tok_str
|
||||
for a in added_tokens:
|
||||
tokens[a["id"]] = a["content"]
|
||||
|
||||
merges = []
|
||||
for m in model.get("merges", []):
|
||||
if isinstance(m, list):
|
||||
merges.append(" ".join(m))
|
||||
else:
|
||||
merges.append(m)
|
||||
|
||||
w.add_tokenizer_model("gpt2")
|
||||
w.add_token_list(tokens)
|
||||
w.add_token_merges(merges)
|
||||
|
||||
log(tag, "tokenizer: %d vocab, %d merges, %d added" %
|
||||
(len(tokens), len(merges), len(added_tokens)))
|
||||
|
||||
# Tensors to skip in model.safetensors (recomputable at runtime).
|
||||
def is_skip_base_tensor(name):
|
||||
# codebook_layer_offsets = arange(num_audio_codebook) * audio_vocab_size,
|
||||
# trivially recomputed C++ side from the audio metadata.
|
||||
if name == "codebook_layer_offsets":
|
||||
return True
|
||||
return False
|
||||
|
||||
# omnivoice-base: LLM + audio_embeddings + audio_heads + tokenizer
|
||||
def convert_base(ckpt_dir, output_path):
|
||||
tag = "BASE"
|
||||
cfg_path = os.path.join(ckpt_dir, "config.json")
|
||||
sf_path = os.path.join(ckpt_dir, "model.safetensors")
|
||||
tok_path = os.path.join(ckpt_dir, "tokenizer.json")
|
||||
|
||||
with open(cfg_path, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
llm_cfg = cfg["llm_config"]
|
||||
|
||||
log(tag, "writing %s" % os.path.basename(output_path))
|
||||
w = gguf.GGUFWriter(output_path, "omnivoice-lm", use_temp_file=True)
|
||||
w.add_name("OmniVoice")
|
||||
|
||||
# standard LLM metadata
|
||||
w.add_block_count(llm_cfg["num_hidden_layers"])
|
||||
w.add_embedding_length(llm_cfg["hidden_size"])
|
||||
w.add_feed_forward_length(llm_cfg["intermediate_size"])
|
||||
w.add_head_count(llm_cfg["num_attention_heads"])
|
||||
w.add_head_count_kv(llm_cfg["num_key_value_heads"])
|
||||
w.add_key_length(llm_cfg["head_dim"])
|
||||
w.add_value_length(llm_cfg["head_dim"])
|
||||
w.add_vocab_size(llm_cfg["vocab_size"])
|
||||
w.add_context_length(llm_cfg["max_position_embeddings"])
|
||||
w.add_layer_norm_rms_eps(llm_cfg["rms_norm_eps"])
|
||||
w.add_rope_freq_base(float(llm_cfg["rope_parameters"]["rope_theta"]))
|
||||
w.add_bool("omnivoice.tie_word_embeddings", bool(llm_cfg.get("tie_word_embeddings", False)))
|
||||
|
||||
# audio IO (LLM-side custom)
|
||||
w.add_uint32("omnivoice.num_audio_codebook", cfg["num_audio_codebook"])
|
||||
w.add_uint32("omnivoice.audio_vocab_size", cfg["audio_vocab_size"])
|
||||
w.add_uint32("omnivoice.audio_mask_id", cfg["audio_mask_id"])
|
||||
w.add_array("omnivoice.audio_codebook_weights",
|
||||
[int(x) for x in cfg["audio_codebook_weights"]])
|
||||
|
||||
# special token IDs from tokenizer.json (denoise / lang / instruct / text bracketing)
|
||||
with open(tok_path, "r", encoding="utf-8") as f:
|
||||
tj = json.load(f)
|
||||
special_map = {a["content"]: a["id"] for a in tj.get("added_tokens", [])}
|
||||
for tag_name in [
|
||||
"<|denoise|>",
|
||||
"<|lang_start|>", "<|lang_end|>",
|
||||
"<|instruct_start|>", "<|instruct_end|>",
|
||||
"<|text_start|>", "<|text_end|>",
|
||||
]:
|
||||
if tag_name in special_map:
|
||||
key = "omnivoice.special." + tag_name.strip("<|>").replace("|", "_")
|
||||
w.add_uint32(key, special_map[tag_name])
|
||||
|
||||
# tokenizer
|
||||
add_bpe_tokenizer(w, tok_path, tag)
|
||||
|
||||
# tensors
|
||||
meta, hdr_size = read_sf_header(sf_path)
|
||||
n_tensors, n_bytes = 0, 0
|
||||
n_skip = 0
|
||||
for name in sorted(meta.keys()):
|
||||
if is_skip_base_tensor(name):
|
||||
n_skip += 1
|
||||
continue
|
||||
nb = add_tensor_passthrough(w, name, sf_path, hdr_size, meta[name])
|
||||
n_tensors += 1
|
||||
n_bytes += nb
|
||||
log(tag, "total: %d tensors, %.1f MB (%d skipped)" %
|
||||
(n_tensors, n_bytes / (1 << 20), n_skip))
|
||||
|
||||
w.write_header_to_file()
|
||||
w.write_kv_data_to_file()
|
||||
w.write_tensors_to_file(progress=True)
|
||||
w.close()
|
||||
|
||||
out_mb = os.path.getsize(output_path) / (1 << 20)
|
||||
log(tag, "wrote %.0f MB -> %s" % (out_mb, output_path))
|
||||
|
||||
# Tensors to skip in audio_tokenizer (training-only, not used at inference)
|
||||
def is_skip_tokenizer_tensor(name):
|
||||
# decoder_semantic + fc1: auxiliary HuBERT-feature reconstruction loss
|
||||
if name.startswith("decoder_semantic.") or name.startswith("fc1."):
|
||||
return True
|
||||
# RVQ codebook EMA buffers (cluster_size, embed_avg, inited): training only
|
||||
if "quantizer.quantizers." in name:
|
||||
if name.endswith(".cluster_size") or name.endswith(".embed_avg") or name.endswith(".inited"):
|
||||
return True
|
||||
# parametrizations.weight.original{0,1}: replaced by folded weight_norm below
|
||||
if "parametrizations.weight.original" in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Fold HuBERT pos_conv weight_norm: weight = v * g / ||v||_{dim=(0,1)}
|
||||
# Convention: torch.nn.utils.weight_norm(conv, dim=2) for grouped conv1d.
|
||||
# Verified bit-identical to torch._weight_norm(v, g, dim=2) up to FP32 noise.
|
||||
def fold_pos_conv_weight_norm(meta, hdr_size, sf_path):
|
||||
g_name = "semantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original0"
|
||||
v_name = "semantic_model.encoder.pos_conv_embed.conv.parametrizations.weight.original1"
|
||||
target = "semantic_model.encoder.pos_conv_embed.conv.weight"
|
||||
|
||||
g = read_sf_tensor(sf_path, hdr_size, meta[g_name]) # (1, 1, 128)
|
||||
v = read_sf_tensor(sf_path, hdr_size, meta[v_name]) # (768, 48, 128)
|
||||
norm = np.sqrt(np.sum(v * v, axis=(0, 1), keepdims=True))
|
||||
weight = v * g / norm
|
||||
return target, weight.astype(np.float32, copy=False)
|
||||
|
||||
# omnivoice-tokenizer: HuBERT + DAC + RVQ + fc/fc2
|
||||
def convert_tokenizer(ckpt_dir, output_path):
|
||||
tag = "TOK"
|
||||
audio_dir = os.path.join(ckpt_dir, "audio_tokenizer")
|
||||
cfg_path = os.path.join(audio_dir, "config.json")
|
||||
sf_path = os.path.join(audio_dir, "model.safetensors")
|
||||
|
||||
with open(cfg_path, "r", encoding="utf-8") as f:
|
||||
cfg = json.load(f)
|
||||
|
||||
log(tag, "writing %s" % os.path.basename(output_path))
|
||||
w = gguf.GGUFWriter(output_path, "omnivoice-tokenizer", use_temp_file=True)
|
||||
w.add_name("OmniVoice-tokenizer")
|
||||
|
||||
# global audio config
|
||||
w.add_uint32("omnivoice.sample_rate", cfg["sample_rate"])
|
||||
w.add_uint32("omnivoice.semantic_sample_rate", cfg["semantic_sample_rate"])
|
||||
w.add_uint32("omnivoice.downsample_factor", cfg["downsample_factor"])
|
||||
w.add_uint32("omnivoice.codebook_size", cfg["codebook_size"])
|
||||
w.add_uint32("omnivoice.codebook_dim", cfg["codebook_dim"])
|
||||
|
||||
# acoustic (DAC) config
|
||||
ac = cfg["acoustic_model_config"]
|
||||
w.add_uint32("omnivoice.acoustic.encoder_hidden_size", ac["encoder_hidden_size"])
|
||||
w.add_uint32("omnivoice.acoustic.decoder_hidden_size", ac["decoder_hidden_size"])
|
||||
w.add_uint32("omnivoice.acoustic.hidden_size", ac["hidden_size"])
|
||||
w.add_uint32("omnivoice.acoustic.n_codebooks", ac["n_codebooks"])
|
||||
w.add_uint32("omnivoice.acoustic.hop_length", ac["hop_length"])
|
||||
w.add_array("omnivoice.acoustic.upsampling_ratios",
|
||||
[int(x) for x in ac["upsampling_ratios"]])
|
||||
w.add_array("omnivoice.acoustic.downsampling_ratios",
|
||||
[int(x) for x in ac["downsampling_ratios"]])
|
||||
|
||||
# semantic (HuBERT) config
|
||||
sm = cfg["semantic_model_config"]
|
||||
w.add_uint32("omnivoice.semantic.hidden_size", sm["hidden_size"])
|
||||
w.add_uint32("omnivoice.semantic.intermediate_size", sm["intermediate_size"])
|
||||
w.add_uint32("omnivoice.semantic.num_attention_heads", sm["num_attention_heads"])
|
||||
w.add_uint32("omnivoice.semantic.num_hidden_layers", sm["num_hidden_layers"])
|
||||
w.add_uint32("omnivoice.semantic.num_feat_extract_layers", sm["num_feat_extract_layers"])
|
||||
w.add_array("omnivoice.semantic.conv_dim", [int(x) for x in sm["conv_dim"]])
|
||||
w.add_array("omnivoice.semantic.conv_kernel", [int(x) for x in sm["conv_kernel"]])
|
||||
w.add_array("omnivoice.semantic.conv_stride", [int(x) for x in sm["conv_stride"]])
|
||||
w.add_uint32("omnivoice.semantic.num_conv_pos_embeddings", sm["num_conv_pos_embeddings"])
|
||||
w.add_uint32("omnivoice.semantic.num_conv_pos_embedding_groups", sm["num_conv_pos_embedding_groups"])
|
||||
w.add_float32("omnivoice.semantic.layer_norm_eps", float(sm["layer_norm_eps"]))
|
||||
|
||||
# tensors
|
||||
meta, hdr_size = read_sf_header(sf_path)
|
||||
|
||||
# fold pos_conv weight_norm before iterating, so the folded weight replaces
|
||||
# both parametrizations entries in the output GGUF
|
||||
folded_name, folded_arr = fold_pos_conv_weight_norm(meta, hdr_size, sf_path)
|
||||
|
||||
n_tensors, n_bytes = 0, 0
|
||||
n_skip = 0
|
||||
for name in sorted(meta.keys()):
|
||||
if is_skip_tokenizer_tensor(name):
|
||||
n_skip += 1
|
||||
continue
|
||||
nb = add_tensor_passthrough(w, name, sf_path, hdr_size, meta[name])
|
||||
n_tensors += 1
|
||||
n_bytes += nb
|
||||
|
||||
# folded pos_conv weight is computed F32 from F32 source params, write F32
|
||||
w.add_tensor(folded_name, folded_arr)
|
||||
n_tensors += 1
|
||||
n_bytes += folded_arr.nbytes
|
||||
|
||||
log(tag, "total: %d tensors, %.1f MB (%d skipped: training-only)" %
|
||||
(n_tensors, n_bytes / (1 << 20), n_skip))
|
||||
|
||||
w.write_header_to_file()
|
||||
w.write_kv_data_to_file()
|
||||
w.write_tensors_to_file(progress=True)
|
||||
w.close()
|
||||
|
||||
out_mb = os.path.getsize(output_path) / (1 << 20)
|
||||
log(tag, "wrote %.0f MB -> %s" % (out_mb, output_path))
|
||||
|
||||
def main():
|
||||
if not os.path.isdir(CHECKPOINT_DIR):
|
||||
log("GGUF", "checkpoints/OmniVoice not found, run checkpoints.sh first")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
|
||||
base_path = os.path.join(OUTPUT_DIR, "omnivoice-base-F32.gguf")
|
||||
tok_path = os.path.join(OUTPUT_DIR, "omnivoice-tokenizer-F32.gguf")
|
||||
|
||||
if os.path.exists(base_path):
|
||||
log("BASE", "skip: %s exists" % os.path.basename(base_path))
|
||||
else:
|
||||
convert_base(CHECKPOINT_DIR, base_path)
|
||||
|
||||
if os.path.exists(tok_path):
|
||||
log("TOK", "skip: %s exists" % os.path.basename(tok_path))
|
||||
else:
|
||||
convert_tokenizer(CHECKPOINT_DIR, tok_path)
|
||||
|
||||
log("GGUF", "done -> %s" % OUTPUT_DIR)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
services:
|
||||
omnivoice-server:
|
||||
build: .
|
||||
security_opt:
|
||||
- apparmor:flatpak
|
||||
ports:
|
||||
- "8000:8000"
|
||||
volumes:
|
||||
- ./models:/models
|
||||
devices:
|
||||
- "/dev/kfd:/dev/kfd"
|
||||
- "/dev/dri:/dev/dri"
|
||||
group_add:
|
||||
- video
|
||||
ipc: host
|
||||
cap_add:
|
||||
- SYS_PTRACE
|
||||
environment:
|
||||
- HSA_OVERRIDE_GFX_VERSION=11.0.0
|
||||
command: >
|
||||
/app/tts-server
|
||||
--model /models/omnivoice-base-Q4_K_M.gguf
|
||||
--codec /models/omnivoice-tokenizer-Q4_K_M.gguf
|
||||
--host 0.0.0.0
|
||||
--port 8000
|
||||
# command: >
|
||||
# vulkaninfo | grep "deviceName"
|
||||
logging:
|
||||
options:
|
||||
max-size: "1m"
|
||||
max-file: "1"
|
||||
@@ -0,0 +1,900 @@
|
||||
# Architecture
|
||||
|
||||
Technical reference for omnivoice.cpp, the GGML port of OmniVoice
|
||||
(k2-fsa/OmniVoice). This document covers the model, the conversion to
|
||||
GGUF, the inference pipeline, the GGML graph conventions, and the CLI
|
||||
tools.
|
||||
|
||||
## Upstream model
|
||||
|
||||
OmniVoice (Xiaomi / k2-fsa, Apache 2.0) is a multilingual zero-shot
|
||||
text-to-speech system covering 646 languages. It targets three modes :
|
||||
|
||||
voice cloning reference audio plus reference transcript drive the
|
||||
target speaker identity
|
||||
voice design six attribute categories (gender, age, pitch, style,
|
||||
volume, emotion) drive a synthesised speaker
|
||||
auto voice no reference, the model picks a coherent speaker per
|
||||
utterance
|
||||
|
||||
The system is a non autoregressive, mask-predict (MaskGIT) generative
|
||||
model running on top of a Qwen3 backbone with custom audio
|
||||
input/output and a separate audio tokenizer. The audio tokenizer is
|
||||
the Higgs Audio v2 codec (`bosonai/higgs-audio-v2-tokenizer`,
|
||||
Apache 2.0), which combines a HuBERT semantic stream, a DAC acoustic
|
||||
stream, and an 8-codebook residual vector quantiser at 25 frames per
|
||||
second over 24 kHz mono audio.
|
||||
|
||||
Single public checkpoint : `k2-fsa/OmniVoice` (3.1 GB).
|
||||
|
||||
Backbone Qwen3 0.6B (28 layers, hidden 1024, GQA 16/8)
|
||||
Audio codebooks 8 residual, 1024 entries each plus 1 mask token
|
||||
Audio framerate 25 Hz
|
||||
Hop length 960 samples
|
||||
Sample rate 24 kHz mono
|
||||
Semantic SR 16 kHz (HuBERT input)
|
||||
MaskGIT steps 32 default, configurable
|
||||
|
||||
## Build
|
||||
|
||||
```
|
||||
git clone --recurse-submodules https://github.com/ServeurpersoCom/omnivoice.cpp.git
|
||||
cd omnivoice.cpp
|
||||
./buildcuda.sh # NVIDIA GPU
|
||||
./buildvulkan.sh # AMD/Intel GPU (Vulkan)
|
||||
./buildcpu.sh # CPU only
|
||||
./buildall.sh # all backends, runtime DL loading
|
||||
```
|
||||
|
||||
The GGML submodule lives at `https://github.com/ServeurpersoCom/ggml.git`
|
||||
and provides two custom ops required by the codec :
|
||||
`GGML_OP_SNAKE` and `GGML_OP_COL2IM_1D`. Both have CPU, CUDA, Metal,
|
||||
and Vulkan kernels.
|
||||
|
||||
## Model conversion
|
||||
|
||||
```
|
||||
./checkpoints.sh # hf download k2-fsa/OmniVoice -> checkpoints/OmniVoice/
|
||||
./convert.py # 2 GGUFs in BF16 -> models/
|
||||
./quantize.sh # base LM Q8_0 (tokenizer stays at native dtype)
|
||||
```
|
||||
|
||||
Outputs :
|
||||
|
||||
```
|
||||
models/omnivoice-base-BF16.gguf 1.2 GB LLM + audio_emb + audio_heads + tokenizer
|
||||
models/omnivoice-base-Q8_0.gguf 626 MB quantized base, 1.9x smaller
|
||||
models/omnivoice-tokenizer-F32.gguf 702 MB HuBERT + DAC + RVQ + fc/fc2 (native F32)
|
||||
```
|
||||
|
||||
The audio tokenizer GGUF preserves the source dtype 1:1. The reference
|
||||
checkpoint stores the codec at F32, so the GGUF stays F32 to avoid
|
||||
truncation noise across the 8-stage RVQ residual chain. Late codebooks
|
||||
fall below 50 percent codebook match against the reference if any
|
||||
intermediate weight is rounded to BF16.
|
||||
|
||||
Quantisation policy : Q8_0 only on the base LM. The 612 M parameter
|
||||
backbone is small enough that lower quants degrade quality without
|
||||
meaningful size gains.
|
||||
|
||||
## GGUF layout
|
||||
|
||||
`omnivoice-base-{quant}.gguf` (arch `omnivoice-lm`) :
|
||||
|
||||
```
|
||||
metadata
|
||||
general.architecture omnivoice-lm
|
||||
block_count 28
|
||||
embedding_length 1024
|
||||
feed_forward_length 3072
|
||||
head_count 16
|
||||
head_count_kv 8 (GQA 2:1)
|
||||
key_length 128
|
||||
vocab_size 151676
|
||||
context_length 40960
|
||||
layer_norm_rms_eps 1e-6
|
||||
rope_freq_base 1e6
|
||||
omnivoice.tie_word_embeddings true
|
||||
omnivoice.num_audio_codebook 8
|
||||
omnivoice.audio_vocab_size 1025
|
||||
omnivoice.audio_mask_id 1024
|
||||
omnivoice.audio_codebook_weights [8, 8, 6, 6, 4, 4, 2, 2]
|
||||
omnivoice.special.denoise 151669
|
||||
omnivoice.special.lang_start 151670
|
||||
omnivoice.special.lang_end 151671
|
||||
omnivoice.special.instruct_start 151672
|
||||
omnivoice.special.instruct_end 151673
|
||||
omnivoice.special.text_start 151674
|
||||
omnivoice.special.text_end 151675
|
||||
tokenizer (Qwen2 BPE, 151676 vocab, 151387 merges, 33 added_tokens)
|
||||
|
||||
tensors (312)
|
||||
llm.embed_tokens.weight (151676, 1024)
|
||||
llm.norm.weight (1024,)
|
||||
llm.layers.0..27.{q,k,v,o}_proj.weight GQA, no bias
|
||||
llm.layers.0..27.self_attn.{q_norm, k_norm}.weight per-head RMSNorm (128,)
|
||||
llm.layers.0..27.{input,post_attention}_layernorm.weight RMSNorm
|
||||
llm.layers.0..27.mlp.{gate,up,down}_proj.weight SwiGLU, no bias
|
||||
audio_embeddings.weight (8200, 1024) 8 codebooks * 1025 vocab
|
||||
audio_heads.weight (8200, 1024) audio output, no bias
|
||||
```
|
||||
|
||||
`omnivoice-tokenizer-{quant}.gguf` (arch `omnivoice-tokenizer`) :
|
||||
|
||||
```
|
||||
metadata
|
||||
omnivoice.sample_rate 24000
|
||||
omnivoice.semantic_sample_rate 16000
|
||||
omnivoice.downsample_factor 320
|
||||
omnivoice.codebook_size 1024
|
||||
omnivoice.codebook_dim 64
|
||||
omnivoice.acoustic.encoder_hidden_size 64
|
||||
omnivoice.acoustic.decoder_hidden_size 1024
|
||||
omnivoice.acoustic.hidden_size 256
|
||||
omnivoice.acoustic.n_codebooks 9 (only 8 used)
|
||||
omnivoice.acoustic.hop_length 960
|
||||
omnivoice.acoustic.upsampling_ratios [8, 5, 4, 2, 3]
|
||||
omnivoice.acoustic.downsampling_ratios [8, 5, 4, 2, 3]
|
||||
omnivoice.semantic.hidden_size 768 (HuBERT base)
|
||||
omnivoice.semantic.intermediate_size 3072
|
||||
omnivoice.semantic.num_attention_heads 12
|
||||
omnivoice.semantic.num_hidden_layers 12
|
||||
omnivoice.semantic.num_feat_extract_layers 7
|
||||
omnivoice.semantic.conv_dim [512]*7
|
||||
omnivoice.semantic.conv_kernel [10, 3, 3, 3, 3, 2, 2]
|
||||
omnivoice.semantic.conv_stride [5, 2, 2, 2, 2, 2, 2]
|
||||
omnivoice.semantic.num_conv_pos_embeddings 128
|
||||
omnivoice.semantic.num_conv_pos_embedding_groups 16
|
||||
omnivoice.semantic.layer_norm_eps 1e-5
|
||||
|
||||
tensors (486)
|
||||
acoustic_encoder.* DAC encoder, 5 blocks, downsamples 8 5 4 2 3
|
||||
acoustic_decoder.* DAC decoder, 5 blocks, upsamples 8 5 4 2 3
|
||||
encoder_semantic.* semantic conv blocks
|
||||
semantic_model.* HuBERT base, weight_norm folded
|
||||
quantizer.quantizers.0..7.{codebook.embed, project_in.{w,b}, project_out.{w,b}}
|
||||
fc.{weight, bias} 1024 -> 1024 (after concat acoustic + semantic)
|
||||
fc2.{weight, bias} 1024 -> 256 (before DAC decoder)
|
||||
```
|
||||
|
||||
Single weight_norm fold at convert time :
|
||||
`semantic_model.encoder.pos_conv_embed.conv.weight`, formula
|
||||
`weight = v * g / ||v||_{dim=(0,1)}` matching
|
||||
`torch._weight_norm(v, g, dim=2)`. Validated bit-perfect, max abs diff
|
||||
3.9e-7 against the PyTorch reference.
|
||||
|
||||
## Component architecture
|
||||
|
||||
### Qwen3 0.6B backbone with custom IO
|
||||
|
||||
Standard Qwen3 modulo two changes :
|
||||
|
||||
input embed hybrid text plus audio, weighted sum across 8
|
||||
codebooks gated by `audio_mask`
|
||||
output head custom `audio_heads` Linear (8200, 1024), no text
|
||||
`lm_head`
|
||||
|
||||
```
|
||||
input_ids [B, 8, S] int (text on row 0, audio codes on rows 1..7)
|
||||
audio_mask [B, S] bool
|
||||
|
||||
text_emb = embed_tokens(input_ids[:, 0, :]) (B, S, 1024)
|
||||
shifted = input_ids * audio_mask + offsets[None, :, None]
|
||||
offsets = arange(8) * 1025
|
||||
audio_emb = audio_embeddings(shifted).sum(dim=1) (B, S, 1024)
|
||||
inputs = where(audio_mask, audio_emb, text_emb) (B, S, 1024)
|
||||
|
||||
x = qwen3_forward(inputs, attention_mask, position_ids) (B, S, 1024)
|
||||
|
||||
logits_flat = x @ audio_heads.weight.T (B, S, 8200)
|
||||
logits = reshape (B, 8, S, 1025)
|
||||
```
|
||||
|
||||
Qwen3 specifics already in llama.cpp :
|
||||
|
||||
28 layers, hidden 1024, intermediate 3072
|
||||
16 query heads + 8 KV heads (GQA 2:1), head_dim 128
|
||||
per-head RMSNorm on Q and K (q_norm, k_norm shape (128,)) before RoPE
|
||||
no bias on Q/K/V/O/MLP
|
||||
RoPE theta = 1e6
|
||||
SwiGLU MLP
|
||||
tie_word_embeddings = true (`lm_head` absent, output goes through audio_heads)
|
||||
|
||||
### MaskGIT decoder
|
||||
|
||||
Iterative non autoregressive decoder, no KV cache. Each step is a full
|
||||
prefill of the LLM on the current input.
|
||||
|
||||
Prompt (per item, broadcast across 8 codebooks) :
|
||||
|
||||
```
|
||||
[<|denoise|>]?
|
||||
<|lang_start|> {iso_code or "None"} <|lang_end|>
|
||||
<|instruct_start|> {style or "None"} <|instruct_end|>
|
||||
<|text_start|> {ref_text + " " + text} <|text_end|>
|
||||
{ref_audio_codes}?
|
||||
{MASK x num_target_tokens}
|
||||
```
|
||||
|
||||
Unconditional prompt for CFG = the trailing `num_target_tokens` mask
|
||||
tokens only. Batched (cond + uncond) doubles the batch dim.
|
||||
|
||||
```
|
||||
for step in 0..num_step-1 :
|
||||
forward(input_ids, audio_mask, attention_mask) (2B, 8, S, 1025)
|
||||
log_probs = log_softmax(c + cfg_scale * (c - u))
|
||||
log_probs[..., MASK_ID] = -inf
|
||||
if class_temp > 0 :
|
||||
keep_top_k_ratio(log_probs, 0.1)
|
||||
gumbel_sample(temp = class_temp)
|
||||
pred = argmax(log_probs)
|
||||
score = log_probs.max - layer_idx * layer_penalty (5.0)
|
||||
if pos_temp > 0 :
|
||||
score += gumbel * pos_temp
|
||||
score[already_unmasked] = -inf
|
||||
topk_idx = topk(score.flatten(), schedule[step])
|
||||
tokens[topk_idx] = pred[topk_idx]
|
||||
update batch_input_ids cond and uncond
|
||||
```
|
||||
|
||||
Schedule of newly unmasked positions per step is computed from
|
||||
`_get_time_steps(t_start=0, t_end=1, num_step, t_shift=0.1)` then
|
||||
`ceil(N_total * (t[step+1] - t[step]))`. 32 steps default.
|
||||
|
||||
KV cache is not usable across MaskGIT steps. The attention is fully
|
||||
bidirectional, so the prefix hidden states depend on the current
|
||||
target state through every layer. As tokens get progressively
|
||||
unmasked the K and V tensors of the prefix at every layer above the
|
||||
embeddings drift, which forbids the standard prefix-cache trick that
|
||||
works for causal LMs. Each step is therefore a full prefill of the
|
||||
LLM at cost `2 * forward_full(B, S)` (the 2 accounts for the cond +
|
||||
uncond CFG rows).
|
||||
|
||||
Determinism. With `class_temperature = 0` and `position_temperature = 0`
|
||||
the decoder is bit deterministic. Higher temperatures rely on a
|
||||
seedable Philox4x32-10 PRNG. The pipeline threads the Philox counter
|
||||
across MaskGIT calls so that chunked inference matches the global RNG
|
||||
drift of the PyTorch reference.
|
||||
|
||||
#### Inner-loop optimisations
|
||||
|
||||
The num_step iterations of one chunk run on a fixed shape (`S`, `K`, `B'`) so
|
||||
the per-step overhead can be cut without touching the math.
|
||||
|
||||
`pipeline_tts_llm_forward_batched` accepts a `T_audio` parameter that
|
||||
narrows the GPU output to the audio window only. The MaskGIT decoder
|
||||
reads cond logits at `[S - T, S)` on row 0 and uncond logits at
|
||||
`[0, T)` on row 1, so the full `[B', V, K, S]` tensor is wasteful.
|
||||
With `T_audio > 0` the function builds two `ggml_view_4d` over those
|
||||
ranges, makes them contiguous via `ggml_cont`, and only those two
|
||||
sub-tensors are flagged as graph outputs. The GPU to CPU copy shrinks
|
||||
from `B' * V * K * S` floats to `2 * V * K * T_audio` floats, around
|
||||
5.6x less on the typical voice cloning shape (S ~ 1880, T ~ 350).
|
||||
When `T_audio == 0` the function falls back to the full output, used
|
||||
by the debug dump path that needs every position.
|
||||
|
||||
`MaskgitBatchedCtx` holds the inputs that stay constant across the
|
||||
steps : the F32 audio mask and its complement, the RoPE position
|
||||
vector, and the F16 attention bias. The bias is the heaviest piece,
|
||||
`B' * S * S` F16 conversions per step (about 7 M ops on the typical
|
||||
shape). `pipeline_tts_llm_batched_ctx_init` precomputes the bias once
|
||||
per chunk, with a single `ggml_fp32_to_fp16` call for each of the two
|
||||
distinct values (1.0 and 0.0) hoisted out of the conversion loop. The
|
||||
context also keeps the original int32 pointers so the debug loop path
|
||||
can hand them down to the single forward unchanged.
|
||||
|
||||
Both optimisations preserve the math exactly, the only side effect is
|
||||
a slight reordering of the GPU FP32 reductions when the scheduler
|
||||
fuses the new output nodes differently, which moves the audio cosine
|
||||
similarity by a few times 1e-6. Token-level results stay 100 percent
|
||||
exact against the PyTorch reference.
|
||||
|
||||
### Audio tokenizer pipeline
|
||||
|
||||
Encode (voice cloning reference path) :
|
||||
|
||||
```
|
||||
ref_audio @ 24 kHz (1, 1, T_samples)
|
||||
-> resample 16 kHz (kaiser polyphase)
|
||||
-> pad 160 each side
|
||||
-> HuBERT.feature_extractor (320x downsample)
|
||||
-> HuBERT.feature_projection (LayerNorm + Linear)
|
||||
-> + pos_conv_embed (folded)
|
||||
-> 12 transformer layers
|
||||
-> mean over 13 hidden states (1, 768, T_sem)
|
||||
-> downsample by 2 (semantic_downsample_factor)
|
||||
-> SemanticEncoder (conv blocks) (1, 768, T_frames)
|
||||
-> e_acoustic (DAC encoder, 5 down-blocks) (1, 256, T_frames)
|
||||
-> concat dim=1 (1, 1024, T_frames)
|
||||
-> fc Linear (1024 -> 1024) (1, 1024, T_frames)
|
||||
-> RVQ encode (8 codebooks residual) (1, 8, T_frames) int @ 25 fps
|
||||
```
|
||||
|
||||
Decode (TTS path) :
|
||||
|
||||
```
|
||||
codes [B, 8, T] int
|
||||
-> RVQ decode :
|
||||
for k in 0..7 :
|
||||
e_k = codebook[k].embed[codes[k, :]] (B, T, 64)
|
||||
p_k = e_k @ project_out[k].W.T + bias[k] (B, T, 1024)
|
||||
out += p_k
|
||||
-> transpose (B, 1024, T)
|
||||
-> fc2 Linear (1024 -> 256) (B, 256, T)
|
||||
-> acoustic_decoder DAC :
|
||||
conv1 (256 -> 1024, k=7, pad=3)
|
||||
for block in 0..4, ratios [8, 5, 4, 2, 3] :
|
||||
snake1 (alpha)
|
||||
conv_t1 (IC -> OC, k=2*r, stride=r,
|
||||
padding=ceil(r/2), output_padding=r%2)
|
||||
for res_unit in 0..2, dilations [1, 3, 9] :
|
||||
snake1 (alpha)
|
||||
conv1 (OC, OC, k=7, dil=d, pad=3*d)
|
||||
snake2 (alpha)
|
||||
conv2 (OC, OC, k=1)
|
||||
residual add
|
||||
snake1 (alpha) final
|
||||
conv2 (32 -> 1, k=7, pad=3)
|
||||
-> audio (B, 1, 960*T)
|
||||
```
|
||||
|
||||
960x upsample = 8 * 5 * 4 * 2 * 3. T_in @ 25 fps -> T_out @ 24 kHz exact.
|
||||
|
||||
### DAC decoder block channels
|
||||
|
||||
```
|
||||
block 0 : IC=1024 OC=512 stride=8 K=16 pad=4 output_pad=0
|
||||
block 1 : IC=512 OC=256 stride=5 K=10 pad=3 output_pad=1
|
||||
block 2 : IC=256 OC=128 stride=4 K=8 pad=2 output_pad=0
|
||||
block 3 : IC=128 OC=64 stride=2 K=4 pad=1 output_pad=0
|
||||
block 4 : IC=64 OC=32 stride=3 K=6 pad=2 output_pad=1
|
||||
final : 32 -> 1
|
||||
```
|
||||
|
||||
PyTorch ConvTranspose1d formula :
|
||||
`T_out = (T_in - 1)*stride - 2*padding + dilation*(kernel - 1) + output_padding + 1`
|
||||
|
||||
With our parameters (d=1, k=2*s, p=ceil(s/2), op=s%2) the formula
|
||||
collapses to `T_out = stride * T_in` exactly for all five blocks.
|
||||
|
||||
### Snake activation
|
||||
|
||||
DAC reference formula (Hugging Face `Snake1d.forward`) :
|
||||
`y = x + (alpha + 1e-9).reciprocal() * sin(alpha * x)^2`
|
||||
|
||||
`ggml_snake(x, a, inv_b)` computes `y = x + sin^2(a * x) * inv_b`.
|
||||
Mapping :
|
||||
|
||||
a = alpha (loaded direct, BF16 to F32)
|
||||
inv_b = 1/(alpha + 1e-9) (precomputed CPU side at load, F32)
|
||||
|
||||
Both stored as F32 `[1, C]` tensors. `alpha` shape in checkpoint :
|
||||
`(1, C, 1)`, ggml ne = (1, C, 1). C lives on ne[1]. Loader reads C from
|
||||
`mt->ne[1]`.
|
||||
|
||||
### ConvTranspose1d via GEMM + col2im_1d
|
||||
|
||||
PyTorch `nn.ConvTranspose1d(IC, OC, kernel=K, stride=s, padding=p)` with
|
||||
weight shape `(IC, OC, K)`. GGML decomposition :
|
||||
|
||||
```
|
||||
1. Permute weight (IC, OC, K) PyTorch -> (IC, K*OC) ggml at load time.
|
||||
Layout : dst[(oc*K + k) * IC + ic] = src[ic*OC*K + oc*K + k]
|
||||
This makes k vary faster than oc inside the K*OC axis, matching
|
||||
what ggml_compute_forward_col2im_1d_impl expects :
|
||||
col_data[(oc * K + k) + t_in * K_OC]
|
||||
|
||||
2. Build runtime graph :
|
||||
xt = ggml_cont(ctx, ggml_transpose(ctx, x)) # [IC, T_in]
|
||||
col = ggml_mul_mat(ctx, w, xt) # [K*OC, T_in]
|
||||
y = ggml_col2im_1d(ctx, col, stride, OC, padding) # [T_no_op, OC]
|
||||
if (output_pad > 0)
|
||||
y = ggml_pad(ctx, y, output_pad, 0, 0, 0) # right-pad zeros
|
||||
if (bias)
|
||||
y = ggml_add(ctx, y, bias_2d)
|
||||
```
|
||||
|
||||
Validated math : `T_no_op = (T_in - 1)*stride + K - 2*pad`. Adding
|
||||
`output_pad` right-pad gives the PyTorch output size exactly.
|
||||
|
||||
### RVQ codec
|
||||
|
||||
Per-codebook tensors (k = 0..7) :
|
||||
|
||||
```
|
||||
codebook.embed (1024, 64) PyTorch -> ggml ne=(64, 1024)
|
||||
project_in.weight (64, 1024) PyTorch -> ggml ne=(1024, 64) encode-only
|
||||
project_in.bias (64,) encode-only
|
||||
project_out.weight (1024, 64) PyTorch -> ggml ne=(64, 1024)
|
||||
project_out.bias (1024,)
|
||||
```
|
||||
|
||||
Decode graph (per codebook k, accumulated) :
|
||||
|
||||
```
|
||||
codes_k = ggml_view_1d(codes, T, k * stride) # [T] i32
|
||||
e_k = ggml_get_rows(embed[k], codes_k) # [64, T]
|
||||
p_k = ggml_mul_mat(project_out_w[k], e_k) # [1024, T]
|
||||
p_k = ggml_add(p_k, project_out_b[k])
|
||||
acc += p_k
|
||||
```
|
||||
|
||||
Encode (residual loop) :
|
||||
|
||||
```
|
||||
residual = embeddings_in
|
||||
for k in 0..7 :
|
||||
e_k = project_in[k](residual)
|
||||
codes_k = argmin_i ||e_k - codebook[k].embed[i]||^2
|
||||
quantized = project_out[k](codebook[k].embed[codes_k])
|
||||
residual -= quantized
|
||||
```
|
||||
|
||||
### HuBERT semantic encoder
|
||||
|
||||
12 transformer layers Pre-LN, GELU FFN, MHA 12 heads * 64 dim, biases
|
||||
on all QKVO. Pre-conv feature extractor : 7 Conv1D layers, kernels
|
||||
`[10, 3, 3, 3, 3, 2, 2]`, strides `[5, 2, 2, 2, 2, 2, 2]`, GroupNorm on
|
||||
the first only, GELU between. Feature projection LayerNorm + Linear
|
||||
(512 -> 768). Positional embedding via grouped Conv1D (128 kernel,
|
||||
16 groups), `weight_norm` folded at convert time. Final LayerNorm.
|
||||
|
||||
Output computation :
|
||||
|
||||
```
|
||||
mean(stack(all_13_hidden_states, dim=1), dim=1) # (B, T_sem, 768)
|
||||
```
|
||||
|
||||
This is unusual : the encoder averages across the initial input plus
|
||||
the 12 transformer layer outputs, not just the last hidden state.
|
||||
|
||||
## Long-form TTS pipeline
|
||||
|
||||
`pipeline_tts_synthesize_long` orchestrates inputs longer than the
|
||||
chunking threshold. It mirrors `_generate_chunked` in
|
||||
`omnivoice/models/omnivoice.py`.
|
||||
|
||||
```
|
||||
1. Estimate total target tokens via duration_estimate_tokens.
|
||||
2. If T_total <= chunk_threshold_sec * frame_rate, run a single shot
|
||||
pipeline_tts_synthesize and skip chunking.
|
||||
3. Otherwise split text on punctuation with chunk_text_punctuation,
|
||||
targeting chunk_duration_sec seconds per chunk.
|
||||
4. Generate chunks sequentially :
|
||||
- chunk 0 with no reference (auto voice / voice design path) or
|
||||
with the external reference (cloning path)
|
||||
- in the auto voice case, the audio tokens of chunk 0 become the
|
||||
voice prompt for chunks 1..N, locking in the speaker identity
|
||||
5. Cross-fade decoded chunks with cross_fade_chunks(rate, 0.3 s).
|
||||
6. Apply post-processing on the merged waveform.
|
||||
```
|
||||
|
||||
A shared Philox counter `ctr_lo` is threaded across MaskGIT calls so
|
||||
PRNG state advances continuously between chunks, matching the global
|
||||
`torch.cuda.manual_seed` behaviour on the reference side.
|
||||
|
||||
### Text chunking
|
||||
|
||||
`chunk_text_punctuation(text, chunk_len, min_chunk_len)` splits text on
|
||||
sentence-ending punctuation (skipping abbreviation periods), then
|
||||
merges sentences into chunks of at most `chunk_len` UTF-8 codepoints.
|
||||
Undersized chunks (< `min_chunk_len`) are merged into a neighbour.
|
||||
The function operates on UTF-8 strings and treats length as codepoints,
|
||||
matching Python `len(str)` semantics. Per chunk character budget :
|
||||
|
||||
```
|
||||
n_chars = utf8_codepoint_count(full_text)
|
||||
avg_tokens_per_char = T_total / n_chars
|
||||
chunk_len = (int)(chunk_duration_sec * frame_rate / avg_tokens_per_char)
|
||||
```
|
||||
|
||||
`add_punctuation(text)` appends a terminal `.` (Latin) or its CJK
|
||||
equivalent when missing. Used on the reference transcript when
|
||||
`preprocess_prompt` is on.
|
||||
|
||||
### Audio post-processing
|
||||
|
||||
`audio-postproc.h` is a strict math port of `omnivoice/utils/audio.py`
|
||||
plus the relevant `pydub.silence` routines. All public functions take
|
||||
and return float32 mono PCM in [-1, 1] at the pipeline rate (24 kHz).
|
||||
Silence detection runs on int16 samples to match pydub bit-for-bit.
|
||||
|
||||
```
|
||||
remove_silence(buf, min_silence_ms, keep_silence_ms,
|
||||
seek_step_ms, threshold_dbfs)
|
||||
|
||||
Splits buf on contiguous silent regions where every
|
||||
seek_step_ms-long frame stays below threshold_dbfs (RMS, S16,
|
||||
default -50 dBFS), keeps keep_silence_ms of leading and trailing
|
||||
silence around each retained segment, and concatenates the result.
|
||||
|
||||
cross_fade_chunks(chunks, rate, fade_seconds)
|
||||
|
||||
Concatenates audio chunks with a linear cross-fade of fade_seconds
|
||||
at each junction.
|
||||
|
||||
peak_normalize_half(buf)
|
||||
|
||||
Scales buf so peak |x| equals 0.5. Used in pure auto voice when no
|
||||
reference RMS is available.
|
||||
|
||||
fade_and_pad(buf, rate, fade_seconds, pad_seconds)
|
||||
|
||||
Applies a linear fade in / fade out and pads silence at the start
|
||||
and end. Default fade 0.05 s, pad 0.05 s. Mirrors final reference
|
||||
post-step.
|
||||
```
|
||||
|
||||
`ref_rms` is plumbed end to end and decides the volume branch :
|
||||
|
||||
ref_rms < 0 pure auto voice, peak_normalize_half on the
|
||||
cross-faded waveform
|
||||
ref_rms < 0.1 quiet reference, rescale by ref_rms / 0.1
|
||||
otherwise no-op
|
||||
|
||||
When a reference WAV is provided, the CLI computes its RMS on the F32
|
||||
samples after optional silence trimming and passes it down. The same
|
||||
quantity is used on the PyTorch side.
|
||||
|
||||
### Voice modes
|
||||
|
||||
```
|
||||
auto voice no ref-wav. Chunk 0 generates with no reference,
|
||||
subsequent chunks reuse chunk 0 audio tokens as the
|
||||
voice prompt. peak_normalize_half on output.
|
||||
|
||||
voice design no ref-wav, --instruct provides one or more attribute
|
||||
markers (gender, age, pitch, style, volume, emotion)
|
||||
resolved by voice_design.h to the EN/ZH instruct
|
||||
string the reference uses. Chunking behaves like
|
||||
auto voice.
|
||||
|
||||
voice cloning --ref-wav and --ref-text provided. The reference is
|
||||
resampled to 16 kHz, run through the audio tokenizer
|
||||
encoder, and the resulting RVQ codes are reused as
|
||||
the voice prompt for every chunk. The reference RMS
|
||||
sets the target loudness.
|
||||
```
|
||||
|
||||
## Public API
|
||||
|
||||
Two layers, picked by use case.
|
||||
|
||||
### Top-level public ABI : src/omnivoice.h
|
||||
|
||||
Single-header, plain C99, linkage `extern "C"`. The opaque `ov_context`
|
||||
handle aggregates the GGML backend pair, the LM pipeline, the audio
|
||||
tokenizer codec, the BPE tokenizer and the voice-design vocabulary.
|
||||
One init, one free, one synthesize call covers the full TTS path.
|
||||
Same names, same struct layout, same calling convention from C, C++,
|
||||
Python ctypes, Rust bindgen, Go cgo or any other binding generator.
|
||||
|
||||
```c
|
||||
#include "omnivoice.h"
|
||||
|
||||
struct ov_init_params iparams;
|
||||
ov_init_default_params(&iparams);
|
||||
iparams.model_path = "models/omnivoice-base-Q8_0.gguf";
|
||||
iparams.codec_path = "models/omnivoice-tokenizer-F32.gguf";
|
||||
|
||||
struct ov_context * ov = ov_init(&iparams);
|
||||
|
||||
struct ov_tts_params params;
|
||||
ov_tts_default_params(¶ms);
|
||||
params.text = "Hello world.";
|
||||
params.lang = "English";
|
||||
|
||||
struct ov_audio audio = { 0 };
|
||||
enum ov_status rc = ov_synthesize(ov, ¶ms, &audio);
|
||||
if (rc == OV_STATUS_OK) {
|
||||
/* audio.samples is a malloc'd buffer of audio.n_samples floats
|
||||
at audio.sample_rate Hz, audio.channels = 1 (mono) */
|
||||
}
|
||||
ov_audio_free(&audio);
|
||||
ov_free(ov);
|
||||
```
|
||||
|
||||
Status codes :
|
||||
|
||||
```
|
||||
OV_STATUS_OK 0
|
||||
OV_STATUS_INVALID_PARAMS -1 (mutually exclusive ref inputs etc.)
|
||||
OV_STATUS_INSTRUCT_INVALID -2 (instruct rejected by VoiceDesign)
|
||||
OV_STATUS_GENERATE_FAILED -3 (any internal generate / decode fail)
|
||||
OV_STATUS_OOM -4 (output samples allocation failed)
|
||||
OV_STATUS_CANCELLED -5 (cancel callback returned true)
|
||||
```
|
||||
|
||||
`ov_tts_params` exposes `cancel` and `cancel_user_data`. The pipeline
|
||||
polls between chunks of long-form output, so cancel granularity is
|
||||
roughly `chunk_duration_sec` (15 s by default).
|
||||
|
||||
The MaskGIT sampler config is flattened directly into `ov_tts_params`
|
||||
as seven `mg_*` fields ; `ov_tts_default_params` initialises them to
|
||||
the reference defaults (`num_step=32, guidance_scale=2.0, t_shift=0.1,
|
||||
layer_penalty_factor=5.0, position_temperature=5.0,
|
||||
class_temperature=0.0, seed=42`).
|
||||
|
||||
`ov_version()` returns a static string of the form
|
||||
`"MAJOR.MINOR.PATCH (git-hash, date)"`. The macros `OV_VERSION_MAJOR`,
|
||||
`OV_VERSION_MINOR`, `OV_VERSION_PATCH` are also available at
|
||||
compile time for feature-detection.
|
||||
|
||||
### ABI guarantee
|
||||
|
||||
`tests/abi-c.c` is built on every build with
|
||||
`-std=c99 -Wall -Werror -pedantic`. It includes the public header,
|
||||
calls every entry through its early-return path, and is wired into
|
||||
the default build target. Any regression that breaks plain C
|
||||
consumability fails the main build, not an opt-in step.
|
||||
|
||||
The static library `libomnivoice-core.a` is the default build
|
||||
artefact. For binding consumers, configure with
|
||||
`-DOMNIVOICE_SHARED=ON` to add a `libomnivoice.so` (or `.dll` /
|
||||
`.dylib`) shared target that exports only the `ov_*` symbols ;
|
||||
every internal `pipeline_*` and `backend_*` stays hidden behind
|
||||
`-fvisibility=hidden`. Install rules follow `GNUInstallDirs`.
|
||||
|
||||
### Low-level API : src/pipeline-tts.h, src/pipeline-codec.h
|
||||
|
||||
Direct access to the LM forward (`pipeline_tts_llm_forward`,
|
||||
`pipeline_tts_llm_forward_batched`), the MaskGIT-only path
|
||||
(`pipeline_tts_generate`), the codec encode / decode
|
||||
(`pipeline_codec_encode`, `pipeline_codec_decode`), the instruct
|
||||
resolver (`pipeline_tts_resolve_instruct`) and the manual init / free
|
||||
(`pipeline_tts_load`, `pipeline_codec_load`, `backend_init`).
|
||||
|
||||
Used by `--llm-test` and `--maskgit-test` in `omnivoice-tts`, by
|
||||
`omnivoice-codec` for the standalone codec roundtrip, and by the
|
||||
Python cossim harness through dump files.
|
||||
|
||||
This layer is intentionally not part of the public ABI (C++ types in
|
||||
the signatures, no visibility export). It exists for the in-tree
|
||||
debug paths and stays available as long as the bundled CLI tools
|
||||
need it. The handle layer above is the recommended entry for
|
||||
everything else.
|
||||
|
||||
## CLI tools
|
||||
|
||||
### omnivoice-tts
|
||||
|
||||
End-to-end synthesis : text on stdin, WAV file on disk. Verbatim
|
||||
`--help` (the binary also prints an `omnivoice.cpp <hash> (<date>)`
|
||||
banner line first) :
|
||||
|
||||
```
|
||||
Usage: omnivoice-tts --model <gguf> --codec <gguf> [options] -o <out.wav> < text.txt
|
||||
|
||||
Required:
|
||||
--model <gguf> LLM GGUF (F32 / BF16 / Q8_0)
|
||||
--codec <gguf> Codec GGUF (omnivoice-tokenizer-*.gguf)
|
||||
-o <path> Output WAV (24 kHz mono). '-' streams to stdout (pipe friendly).
|
||||
|
||||
Input:
|
||||
stdin Target text to synthesise. With -o '-', stdin is read
|
||||
incrementally and synthesis starts as soon as the first
|
||||
sentence boundary is reached. With -o file.wav, stdin is
|
||||
read fully then synthesised in one shot.
|
||||
--srt <path> Dub an SRT: synth each cue into its time slot, write one
|
||||
timeline WAV ready to mux. Pairs with --ref-wav / --ref-rvq
|
||||
for a cloned voice. Per cue duration comes from the SRT.
|
||||
|
||||
Optional:
|
||||
--format <fmt> WAV output format: wav16, wav24, wav32 (default: wav16)
|
||||
--lang <str> Language label (default 'None')
|
||||
--instruct <str> Style instruction (default 'None')
|
||||
--duration <sec> Output duration in seconds (default: estimate from text)
|
||||
--no-denoise Omit the <|denoise|> prefix
|
||||
--ref-wav <path> Reference WAV for voice cloning
|
||||
--ref-text <path> Transcript file for the reference (required with --ref-wav / --ref-rvq)
|
||||
--ref-rvq <path> Pre-encoded reference codes from omnivoice-codec (replaces --ref-wav)
|
||||
--seed <int> Sampling seed (default: -1 for random)
|
||||
--steps <int> MaskGIT decode steps (default: 32, fewer is faster)
|
||||
--no-preprocess-prompt Skip ref-wav silence trim and ref-text terminal punctuation
|
||||
--chunk-duration <sec> Long-form chunk duration (default: 15.0, <= 0 disables chunking)
|
||||
--chunk-threshold <sec> Activate chunking above this estimated duration (default: 30.0)
|
||||
--stream-by-line Flush synthesis at each newline, one WAV header per line (-o '-')
|
||||
|
||||
Debug:
|
||||
--no-fa Disable flash attention
|
||||
--clamp-fp16 Clamp hidden states to FP16 range
|
||||
--dump <dir> Dump intermediate tensors (f32) to <dir>
|
||||
--llm-test <input.bin> Full LLM forward, dump audio_logits
|
||||
--maskgit-test Greedy MaskGIT decoder, dump audio_tokens [K, T]
|
||||
(no codec decode, reads target text from stdin)
|
||||
```
|
||||
|
||||
### omnivoice-codec
|
||||
|
||||
Audio tokenizer round-trip : WAV to RVQ codes, RVQ codes to WAV.
|
||||
Verbatim `--help` :
|
||||
|
||||
```
|
||||
Usage: omnivoice-codec --model <gguf> -i <input>
|
||||
|
||||
Required:
|
||||
--model <gguf> Codec GGUF (omnivoice-tokenizer-*.gguf)
|
||||
-i <path> Input. WAV -> encode, .rvq -> decode
|
||||
|
||||
Optional:
|
||||
--format <fmt> WAV output format: wav16, wav24, wav32 (default: wav16)
|
||||
|
||||
Output is auto-named next to input : clip.wav -> clip.rvq, clip.rvq -> clip.wav.
|
||||
Encode applies the TTS reference preprocessing (RMS auto-gain, silence trim,
|
||||
hop truncation); the resulting .rvq feeds omnivoice-tts --ref-rvq directly.
|
||||
```
|
||||
|
||||
The `.rvq` file is a small binary container with shape `[8, T]` int32
|
||||
codes plus a header carrying the sample rate and frame rate.
|
||||
|
||||
## Module map
|
||||
|
||||
```
|
||||
src/
|
||||
backend.h GGML backend init, scheduler factory, env override
|
||||
weight-ctx.h Generic weight context for GGUF loaders
|
||||
gguf-weights.h mmap GGUF, gf_load_tensor, gf_get_*
|
||||
audio-io.h WAV read, mono write (S16 / S24 / F32)
|
||||
audio-resample.h Kaiser polyphase 24 kHz <-> 16 kHz
|
||||
audio-postproc.h remove_silence, peak_normalize_half, fade_and_pad,
|
||||
cross_fade_chunks. Strict pydub / utils.audio port.
|
||||
wav.h WAV header reader (PCM16/24/F32, mono/stereo)
|
||||
philox.h Philox4x32-10 counter-based PRNG, PyTorch CUDA aligned
|
||||
debug.h Tensor dumper for cossim tests
|
||||
|
||||
bpe.h Qwen2 / GPT-2 byte-level BPE tokenizer, GGUF loader
|
||||
lang-map.h Language name to ISO 639-3 ID resolution
|
||||
(auto-generated from omnivoice/utils/lang_map.py)
|
||||
voice-design.h Speaker attribute validation and EN / ZH instruct
|
||||
resolution (mirrors voice_design.py)
|
||||
text-chunker.h chunk_text_punctuation, add_punctuation, END_PUNCTUATION
|
||||
duration-estimator.h RuleDurationEstimator port (per-script weights,
|
||||
Unicode category fallback)
|
||||
|
||||
rvq-codec.h Residual VQ encode + decode (8 codebooks)
|
||||
dac-decoder.h DAC acoustic decoder (5 blocks, ratios 8 5 4 2 3)
|
||||
dac-encoder.h DAC acoustic encoder (mirror of decoder)
|
||||
semantic-enc.h SemanticEncoder convs (768 -> 768)
|
||||
hubert-enc.h HuBERT base (feature extractor + pos_conv +
|
||||
12 transformer layers + final LN)
|
||||
|
||||
qwen3-enc.h Qwen3 transformer building blocks
|
||||
omnivoice-llm.h OmniVoice TTS LLM weights and graph helpers
|
||||
prompt-tts.h Prompt builder (denoise + lang + instruct + text +
|
||||
ref + mask) and CFG batch stacking
|
||||
maskgit-tts.h Iterative non autoregressive decoder, configurable
|
||||
step count (32 default), CFG, layer penalty, gumbel
|
||||
sampling, deterministic in greedy mode
|
||||
|
||||
pipeline-codec.{h,cpp} Audio tokenizer end-to-end (encode and decode)
|
||||
pipeline-tts.{h,cpp} Full TTS orchestration, single shot and chunked,
|
||||
plus low-level entries kept available for the
|
||||
debug paths and the cossim test harness
|
||||
omnivoice.{h,cpp} Public ABI : opaque ov_context handle, plain C99
|
||||
header in extern "C", consumable from C, C++,
|
||||
Python ctypes, Rust bindgen, Go cgo
|
||||
|
||||
tools/
|
||||
omnivoice-tts.cpp CLI : text to WAV (auto / design / clone)
|
||||
omnivoice-codec.cpp CLI : codes <-> WAV
|
||||
quantize.cpp GGUF requantizer
|
||||
version.cmake Embeds the git short hash into the binary
|
||||
|
||||
tests/
|
||||
debug-tts-cossim.py Byte-level comparison of every pipeline stage
|
||||
against the PyTorch reference, voice design path
|
||||
debug-clone-cossim.py Same, voice cloning path
|
||||
cross-decode.py Cross check : decode C++ tokens through PyTorch
|
||||
codec and vice versa
|
||||
prompt.txt Long-form English TTS sample
|
||||
ref-audio.wav Voice cloning reference clip
|
||||
ref-text.txt Transcript matching ref-audio.wav
|
||||
abi-c.c Plain C99 smoke test for the public ABI ; built
|
||||
with -Wall -Werror -pedantic on every build,
|
||||
locks in C consumability and symbol linkage
|
||||
```
|
||||
|
||||
## GGML conventions
|
||||
|
||||
### Tensor shape and layout
|
||||
|
||||
PyTorch shape `(out, in)` for a Linear weight stores as ggml
|
||||
`ne[0]=in, ne[1]=out`. The GGUF tensor-shape array is reversed, so
|
||||
reading `reversed(t.shape)` from gguf-py yields the PyTorch shape
|
||||
directly.
|
||||
|
||||
For PyTorch `Conv1d` weight `(OC, IC, K)`, ggml ne is `(K, IC, OC)`.
|
||||
The kernel axis is innermost (contiguous in memory).
|
||||
|
||||
For PyTorch `ConvTranspose1d` weight `(IC, OC, K)`, the convert-time
|
||||
permutation to ggml `(IC, K*OC)` rearranges
|
||||
`(oc*K + k) * IC + ic` so that `ggml_col2im_1d` receives the correct
|
||||
column matrix.
|
||||
|
||||
`ggml_mul_mat(A, B)` : with A.ne[0] = K (must match B.ne[0]),
|
||||
A.ne[1] = M, B.ne[1] = N, output has ne = (N, M). In PyTorch terms,
|
||||
A is `(M, K)`, B is `(N, K)`, output is `(M, N)`, which equals
|
||||
`A @ B^T`.
|
||||
|
||||
### Custom GGML ops
|
||||
|
||||
Provided by the `ServeurpersoCom/ggml` fork :
|
||||
|
||||
`ggml_snake(ctx, x, a, inv_b)` : `y = x + sin^2(a * x) * inv_b`.
|
||||
F32 / F16 / BF16 input/output. CPU + CUDA + Metal + Vulkan.
|
||||
|
||||
`ggml_col2im_1d(ctx, a, s0, oc, p0)` : scatter-add `[K*OC, T_in]`
|
||||
columns into `[T_out, OC]` signal where
|
||||
`T_out = (T_in - 1)*s0 + K - 2*p0`. Layout requires k to vary faster
|
||||
than oc inside the K*OC axis. F32 / F16 / BF16. All backends. The
|
||||
fork also folds the padding crop into this op via the `p0` parameter,
|
||||
removing a follow-up `ggml_view` for the typical ConvTranspose1d use
|
||||
case.
|
||||
|
||||
### Backend lifecycle
|
||||
|
||||
`backend_init("MOD")` then `backend_sched_new(bp, max_nodes)`. Backend
|
||||
handles are shared across modules in the same binary, refcounted. The
|
||||
GPU backend is the default, the CPU backend is kept as a scheduler
|
||||
fallback.
|
||||
|
||||
## Validation
|
||||
|
||||
The reference comparison harness is `tests/debug-tts-cossim.py` and
|
||||
`tests/debug-clone-cossim.py`. They run the same input through the
|
||||
PyTorch reference (with TF32 disabled, eager attention) and through the
|
||||
C++ binary, dump each pipeline stage to disk, and report cosine
|
||||
similarity per stage. Latest run, chunked path, English long-form
|
||||
prompt :
|
||||
|
||||
```
|
||||
TTS chunked: Logits cos=1.000000 max 3.5e-04
|
||||
Step0 pred_tokens 99.93% (2 FP flips)
|
||||
Tokens 1.000000 exact 100.00%
|
||||
Audio 0.999991
|
||||
|
||||
Clone chunked: Lf hidden cos=1.000000 max 1.9e-03
|
||||
Logits cos=1.000000
|
||||
Step0 pred_tokens 100.00%
|
||||
Tokens 1.000000 exact 100.00%
|
||||
Audio 0.999989
|
||||
```
|
||||
|
||||
The few Step0 token flips are argmax ties at the FP epsilon (~2e-5
|
||||
between top1 and top2 at those positions), inherent to the mixed cuBLAS
|
||||
vs GGML kernel arithmetic. They resorb over the 32 MaskGIT steps so
|
||||
the final tokens match bit for bit and decoded audio cosine is
|
||||
> 0.9999.
|
||||
|
||||
## Glossary
|
||||
|
||||
RVQ Residual Vector Quantisation. Stack of codebooks where
|
||||
each one quantises the residual from the previous
|
||||
codebook reconstruction.
|
||||
|
||||
DAC Descript Audio Codec. Convolutional encoder/decoder over
|
||||
residual VQ codes.
|
||||
|
||||
HuBERT Hidden-Unit BERT. Transformer encoder pretrained with
|
||||
masked acoustic unit prediction. Used here to extract
|
||||
semantic embeddings from raw audio.
|
||||
|
||||
Snake Periodic activation introduced in BigVGAN,
|
||||
`y = x + (1/alpha) * sin^2(alpha * x)`. Replaces
|
||||
LeakyReLU in the DAC encoder/decoder.
|
||||
|
||||
CFG Classifier-Free Guidance. The model is run twice
|
||||
(conditional and unconditional) and the outputs combined
|
||||
as `c + scale * (c - u)` to amplify the conditional
|
||||
signal.
|
||||
|
||||
MaskGIT Masked Generative Image Transformer (Chang et al.,
|
||||
arXiv:2202.04200). Iterative non autoregressive decoder
|
||||
where masked tokens are progressively unmasked over a
|
||||
fixed number of steps, prioritising high-confidence
|
||||
positions per step. Originally introduced for image
|
||||
generation, adapted here to audio codes.
|
||||
|
||||
Philox Counter-based PRNG used by PyTorch CUDA. Thread safe and
|
||||
skip-ahead friendly, well suited to deterministic
|
||||
chunked inference.
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
# Call the omnivoice OpenAI-compatible TTS server.
|
||||
# Default response_format is pcm : audio streams chunked as it is generated.
|
||||
|
||||
host="${1:-127.0.0.1}"
|
||||
port="${2:-8080}"
|
||||
|
||||
# Streaming pcm, piped straight into a player as it arrives.
|
||||
# s16le mono 24 kHz. ffplay reads the raw stream from stdin.
|
||||
curl -s -X POST "http://${host}:${port}/v1/audio/speech" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"The quick brown fox jumps over the lazy dog."}' \
|
||||
| ffplay -f s16le -ar 24000 -ch_layout mono -nodisp -autoexit -i -
|
||||
|
||||
# One-shot wav written to a file.
|
||||
curl -s -X POST "http://${host}:${port}/v1/audio/speech" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"input":"This one is written to a file.","response_format":"wav"}' \
|
||||
--output out.wav
|
||||
echo "wrote out.wav"
|
||||
@@ -0,0 +1,13 @@
|
||||
@echo off
|
||||
|
||||
set PATH=%~dp0..\build\Release;%PATH%
|
||||
|
||||
omnivoice-tts.exe ^
|
||||
--model ..\models\omnivoice-base-Q8_0.gguf ^
|
||||
--codec ..\models\omnivoice-tokenizer-Q8_0.gguf ^
|
||||
--ref-rvq freeman.rvq ^
|
||||
--ref-text freeman.txt ^
|
||||
--lang English ^
|
||||
-o clone.wav < prompt.txt
|
||||
|
||||
pause
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
|
||||
../build/omnivoice-tts \
|
||||
--model ../models/omnivoice-base-Q8_0.gguf \
|
||||
--codec ../models/omnivoice-tokenizer-Q8_0.gguf \
|
||||
--ref-rvq freeman.rvq \
|
||||
--ref-text freeman.txt \
|
||||
--lang English \
|
||||
-o clone.wav < prompt.txt
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
If you go into different cultures, they have different concepts of creation. They have their own creation story and what an afterlife is. Where you go, what you do, who you're going to be with. People say, well...
|
||||
@@ -0,0 +1 @@
|
||||
omnivoice.cpp is a minimal C++17 and GGML port of OmniVoice for zero shot multilingual text to speech.
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/bash
|
||||
# Start the omnivoice OpenAI-compatible TTS server.
|
||||
|
||||
./build/tts-server \
|
||||
--model models/omnivoice-base-Q8_0.gguf \
|
||||
--codec models/omnivoice-tokenizer-F32.gguf \
|
||||
--host 127.0.0.1 --port 8080 --lang None
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
|
||||
set PATH=%~dp0..\build\Release;%PATH%
|
||||
|
||||
omnivoice-tts.exe ^
|
||||
--model ..\models\omnivoice-base-Q8_0.gguf ^
|
||||
--codec ..\models\omnivoice-tokenizer-Q8_0.gguf ^
|
||||
--instruct "male, young adult, moderate pitch" ^
|
||||
--lang English ^
|
||||
-o tts.wav < prompt.txt
|
||||
|
||||
pause
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -eu
|
||||
|
||||
../build/omnivoice-tts \
|
||||
--model ../models/omnivoice-base-Q8_0.gguf \
|
||||
--codec ../models/omnivoice-tokenizer-Q8_0.gguf \
|
||||
--instruct "male, young adult, moderate pitch" \
|
||||
--lang English \
|
||||
-o tts.wav < prompt.txt
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
find . -name "*.cpp" -o -name "*.h" | grep -v -e build/ -e ggml/ -e vendor/ | xargs clang-format -i
|
||||
@@ -0,0 +1,22 @@
|
||||
# https://EditorConfig.org
|
||||
|
||||
# Top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
# Unix-style newlines with a newline ending every file, utf-8 charset
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
|
||||
[*.md]
|
||||
indent_size = 2
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[prompts/*.txt]
|
||||
insert_final_newline = unset
|
||||
+1
@@ -0,0 +1 @@
|
||||
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
|
||||
Vendored
+272
@@ -0,0 +1,272 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
libraries: [shared, static]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Dependencies for Ubuntu
|
||||
if: matrix.os == 'ubuntu-latest'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install llvm
|
||||
|
||||
- name: Add msbuild to PATH
|
||||
if: matrix.os == 'windows-latest'
|
||||
uses: microsoft/setup-msbuild@v2
|
||||
|
||||
- name: Create Build Environment
|
||||
run: mkdir build
|
||||
|
||||
- name: Configure CMake
|
||||
working-directory: ./build
|
||||
run: cmake ..
|
||||
${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
|
||||
${{ matrix.libraries == 'static' && '-DBUILD_SHARED_LIBS=OFF' || '-DBUILD_SHARED_LIBS=ON' }}
|
||||
-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed
|
||||
-DGGML_METAL=OFF
|
||||
|
||||
- name: Build
|
||||
working-directory: ./build
|
||||
run: cmake --build . ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
|
||||
|
||||
- name: Test
|
||||
working-directory: ./build
|
||||
run: ctest --verbose --timeout 900 ${{ contains(matrix.os, 'windows') && '--build-config Release' || '' }}
|
||||
|
||||
- name: Install
|
||||
working-directory: ./build
|
||||
run: cmake --build . --target install ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
|
||||
|
||||
- name: Test CMake config
|
||||
run: |
|
||||
mkdir test-cmake
|
||||
cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
|
||||
cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
|
||||
|
||||
# TODO: simplify the following workflows using a matrix
|
||||
ggml-ci-x64-cpu-low-perf:
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.16
|
||||
with:
|
||||
key: ggml-ci-x64-cpu-low-perf
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libcurl4-openssl-dev
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
|
||||
ggml-ci-arm64-cpu-low-perf:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.16
|
||||
with:
|
||||
key: ggml-ci-arm64-cpu-low-perf
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libcurl4-openssl-dev
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
|
||||
ggml-ci-x64-cpu-high-perf:
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.16
|
||||
with:
|
||||
key: ggml-ci-x64-cpu-high-perf
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libcurl4-openssl-dev
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
|
||||
ggml-ci-arm64-cpu-high-perf:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.16
|
||||
with:
|
||||
key: ggml-ci-arm64-cpu-high-perf
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libcurl4-openssl-dev
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
|
||||
ggml-ci-arm64-cpu-high-perf-sve:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.16
|
||||
with:
|
||||
key: ggml-ci-arm64-cpu-high-perf-sve
|
||||
evict-old-files: 1d
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install build-essential libcurl4-openssl-dev
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
|
||||
ggml-ci-x64-nvidia-cuda:
|
||||
runs-on: [self-hosted, Linux, X64, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
nvidia-smi
|
||||
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
|
||||
|
||||
ggml-ci-x64-nvidia-vulkan-cm:
|
||||
runs-on: [self-hosted, Linux, X64, NVIDIA]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
|
||||
|
||||
ggml-ci-x64-nvidia-vulkan-cm2:
|
||||
runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
|
||||
|
||||
# TODO: provision AMX-compatible machine
|
||||
#ggml-ci-x64-cpu-amx:
|
||||
# runs-on: [self-hosted, Linux, X64, CPU, AMX]
|
||||
|
||||
# steps:
|
||||
# - name: Clone
|
||||
# id: checkout
|
||||
# uses: actions/checkout@v6
|
||||
|
||||
# - name: Test
|
||||
# id: ggml-ci
|
||||
# run: |
|
||||
# bash ./ci/run.sh ~/results/ggml /mnt/ggml
|
||||
|
||||
ggml-ci-mac-metal:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
|
||||
|
||||
ggml-ci-mac-vulkan:
|
||||
runs-on: [self-hosted, macOS, ARM64]
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Test
|
||||
id: ggml-ci
|
||||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
|
||||
Vendored
+27
@@ -0,0 +1,27 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Create Release
|
||||
id: create_release
|
||||
uses: ggml-org/action-create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
release_name: ${{ github.ref }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -0,0 +1,39 @@
|
||||
build/
|
||||
build-*/
|
||||
out/
|
||||
tmp/
|
||||
models/
|
||||
models-mnt
|
||||
|
||||
compile_commands.json
|
||||
CMakeSettings.json
|
||||
.vs/
|
||||
.vscode/
|
||||
.idea/
|
||||
.clangd
|
||||
|
||||
.venv/
|
||||
ggml_env/
|
||||
.exrc
|
||||
.cache
|
||||
.DS_Store
|
||||
.stablelm
|
||||
.gpt-2
|
||||
|
||||
src/arm_neon.h
|
||||
tests/arm_neon.h
|
||||
|
||||
zig-out/
|
||||
zig-cache/
|
||||
|
||||
*.o
|
||||
*.d
|
||||
*.dot
|
||||
|
||||
*.sw?
|
||||
|
||||
__pycache__/
|
||||
|
||||
# Model files
|
||||
ggml-model-f16.bin
|
||||
*.bat
|
||||
+335
@@ -0,0 +1,335 @@
|
||||
# date: Tue Feb 4 13:03:51 EET 2025
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
0cc4m <picard12@live.de>
|
||||
65a <10104049+65a@users.noreply.github.com>
|
||||
AT <manyoso@users.noreply.github.com>
|
||||
Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
|
||||
Adam Tazi <52357206+ad1tazi@users.noreply.github.com>
|
||||
Adrien Gallouët <adrien@gallouet.fr>
|
||||
Adrien Gallouët <angt@huggingface.co>
|
||||
Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>
|
||||
AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>
|
||||
AidanBeltonS <aidan.belton@codeplay.com>
|
||||
Akarshan Biswas <akarshan.biswas@gmail.com>
|
||||
Akarshan Biswas <akarshanbiswas@fedoraproject.org>
|
||||
Albert Jin <albert.jin@gmail.com>
|
||||
Alberto Cabrera Pérez <alberto.cabrera@codeplay.com>
|
||||
Alberto Cabrera Pérez <alberto.cabrera@intel.com>
|
||||
Alex Azarov <alex@azarov.by>
|
||||
Alex O'Connell <35843486+acon96@users.noreply.github.com>
|
||||
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||
AmbientL <107641468+AmbientL@users.noreply.github.com>
|
||||
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||
Ananta Bastola <anantarajbastola@gmail.com>
|
||||
Andreas (Andi) Kunar <andreask@msn.com>
|
||||
Andreas Kieslinger <47689530+aendk@users.noreply.github.com>
|
||||
Andrei <abetlen@gmail.com>
|
||||
Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>
|
||||
Andrii Ryzhkov <andriiryzhkov@users.noreply.github.com>
|
||||
Arjun <ccldarjun@icloud.com>
|
||||
Ashraful Islam <ashraful.meche@gmail.com>
|
||||
Astariul <43774355+astariul@users.noreply.github.com>
|
||||
AsukaMinato <asukaminato@nyan.eu.org>
|
||||
Avi Lumelsky <avilume@gmail.com>
|
||||
Bart Pelle <3662930+Velocity-@users.noreply.github.com>
|
||||
Ben Ashbaugh <ben.ashbaugh@intel.com>
|
||||
Bernhard M. Wiedemann <githubbmwprimary@lsmod.de>
|
||||
Borislav Stanimirov <b.stanimirov@abv.bg>
|
||||
Brad Ito <phlogisticfugu@users.noreply.github.com>
|
||||
Brad Murray <59848399+bradmurray-dt@users.noreply.github.com>
|
||||
Brian <mofosyne@gmail.com>
|
||||
Bryan Lozano <b.lozano.havoc@gmail.com>
|
||||
Carolinabanana <140120812+Carolinabanana@users.noreply.github.com>
|
||||
CarterLi999 <664681047@qq.com>
|
||||
Cebtenzzre <cebtenzzre@gmail.com>
|
||||
Changyeon Kim <cyzero.kim@samsung.com>
|
||||
Charles Xu <63788048+chaxu01@users.noreply.github.com>
|
||||
Charles Xu <charles.xu@arm.com>
|
||||
Chen Xi <xi2.chen@intel.com>
|
||||
Chen Xi <xixichen08@foxmail.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
Chris Elrod <elrodc@gmail.com>
|
||||
Christian Kastner <ckk@kvr.at>
|
||||
Clint Herron <hanclinto@gmail.com>
|
||||
Conrad Kramer <conrad@conradkramer.com>
|
||||
Cordeiro <1471463+ocordeiro@users.noreply.github.com>
|
||||
Cristiano Calcagno <cristianoc@users.noreply.github.com>
|
||||
DAN™ <dranger003@gmail.com>
|
||||
Dan Forbes <dan@danforbes.dev>
|
||||
Dan Johansson <164997844+eddnjjn@users.noreply.github.com>
|
||||
Dan Johansson <dan.johansson@arm.com>
|
||||
Daniel Bevenius <daniel.bevenius@gmail.com>
|
||||
Daniel Ziegenberg <daniel@ziegenberg.at>
|
||||
Daniele <57776841+daniandtheweb@users.noreply.github.com>
|
||||
Daulet Zhanguzin <daulet@users.noreply.github.com>
|
||||
Dave <dave-fl@users.noreply.github.com>
|
||||
Dave Airlie <airlied@gmail.com>
|
||||
Dave Airlie <airlied@redhat.com>
|
||||
David Miller <david@patagona.ca>
|
||||
DavidKorczynski <david@adalogics.com>
|
||||
Davidson Francis <davidsondfgl@gmail.com>
|
||||
Dibakar Gope <dibakar.gope@arm.com>
|
||||
Didzis Gosko <didzis@users.noreply.github.com>
|
||||
Diego Devesa <slarengh@gmail.com>
|
||||
Diogo <dgcruz983@gmail.com>
|
||||
Djip007 <3705339+Djip007@users.noreply.github.com>
|
||||
Djip007 <djip.perois@free.fr>
|
||||
Dou Xinpeng <15529241576@163.com>
|
||||
Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com>
|
||||
Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>
|
||||
Ebey Abraham <ebey97@gmail.com>
|
||||
Eldar Yusupov <eyusupov@gmail.com>
|
||||
Emmanuel Durand <emmanueldurand@protonmail.com>
|
||||
Engininja2 <139037756+Engininja2@users.noreply.github.com>
|
||||
Eric Zhang <34133756+EZForever@users.noreply.github.com>
|
||||
Erik Scholz <Green-Sky@users.noreply.github.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
Eve <139727413+netrunnereve@users.noreply.github.com>
|
||||
F1L1P <78918286+F1L1Pv2@users.noreply.github.com>
|
||||
Faisal Zaghloul <quic_fzaghlou@quicinc.com>
|
||||
FantasyGmm <16450052+FantasyGmm@users.noreply.github.com>
|
||||
Felix <stenbackfelix@gmail.com>
|
||||
Finn Voorhees <finnvoorhees@gmail.com>
|
||||
FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>
|
||||
Frankie Robertson <frankier@users.noreply.github.com>
|
||||
GainLee <perfecter.gen@gmail.com>
|
||||
George Hindle <george@georgehindle.com>
|
||||
Georgi Gerganov <ggerganov@gmail.com>
|
||||
Gilad S <7817232+giladgd@users.noreply.github.com>
|
||||
Gilad S <giladgd@users.noreply.github.com>
|
||||
Gilad S. <7817232+giladgd@users.noreply.github.com>
|
||||
Guillaume Wenzek <gwenzek@users.noreply.github.com>
|
||||
Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>
|
||||
Haus1 <haus.xda@gmail.com>
|
||||
Herman Semenov <GermanAizek@yandex.ru>
|
||||
HimariO <dsfhe49854@gmail.com>
|
||||
Hirochika Matsumoto <git@hkmatsumoto.com>
|
||||
Hong Bo PENG <penghb@cn.ibm.com>
|
||||
Hugo Rosenkranz-Costa <hugo.rosenkranz@gmail.com>
|
||||
Hyunsung Lee <ita9naiwa@gmail.com>
|
||||
IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com>
|
||||
Ian Bull <irbull@eclipsesource.com>
|
||||
Ihar Hrachyshka <ihrachys@redhat.com>
|
||||
Ikko Eltociear Ashimine <eltociear@gmail.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
|
||||
Ivan Stepanov <ivanstepanovftw@gmail.com>
|
||||
Ivan Zdane <accounts@ivanzdane.com>
|
||||
Jack Mousseau <jmousseau@users.noreply.github.com>
|
||||
Jack Vial <vialjack@gmail.com>
|
||||
JacobLinCool <jacoblincool@gmail.com>
|
||||
Jakob Frick <jakob.maria.frick@gmail.com>
|
||||
Jan Ploski <jpl@plosquare.com>
|
||||
Jared Van Bortel <jared@nomic.ai>
|
||||
Jeff Bolz <jbolz@nvidia.com>
|
||||
Jeffrey Quesnelle <jquesnelle@gmail.com>
|
||||
Jeroen Mostert <jeroen.mostert@cm.com>
|
||||
Jiahao Li <liplus17@163.com>
|
||||
JidongZhang-THU <1119708529@qq.com>
|
||||
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||
Jo Liss <joliss42@gmail.com>
|
||||
Joe Todd <joe.todd@codeplay.com>
|
||||
Johannes Gäßler <johannesg@5d6.de>
|
||||
John Balis <phobossystems@gmail.com>
|
||||
Josh Bleecher Snyder <josharian@gmail.com>
|
||||
Judd <foldl@users.noreply.github.com>
|
||||
Jun Hee Yoo <contact.jhyoo@gmail.com>
|
||||
Junil Kim <logyourself@gmail.com>
|
||||
Justina Cho <justcho5@gmail.com>
|
||||
Justine Tunney <jtunney@gmail.com>
|
||||
Justine Tunney <jtunney@mozilla.com>
|
||||
Karol Kontny <82021046+kkontny@users.noreply.github.com>
|
||||
Kawrakow <48489457+ikawrakow@users.noreply.github.com>
|
||||
Kevin Gibbons <bakkot@gmail.com>
|
||||
Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
|
||||
Kylin <56434533+KyL0N@users.noreply.github.com>
|
||||
LoganDark <git@logandark.mozmail.com>
|
||||
LoganDark <github@logandark.mozmail.com>
|
||||
LostRuins <39025047+LostRuins@users.noreply.github.com>
|
||||
Lukas Möller <mail@lukas-moeller.ch>
|
||||
M Refi D.A <24388107+refinism@users.noreply.github.com>
|
||||
M. Yusuf Sarıgöz <yusufsarigoz@gmail.com>
|
||||
Ma Mingfei <mingfei.ma@intel.com>
|
||||
Mahesh Madhav <67384846+heshpdx@users.noreply.github.com>
|
||||
MaiHD <maihd.dev@gmail.com>
|
||||
Mark Zhuang <zhuangqiubin@gmail.com>
|
||||
Markus Tavenrath <mtavenrath@users.noreply.github.com>
|
||||
Masaya, Kato <62578291+msy-kato@users.noreply.github.com>
|
||||
Mathieu Baudier <mbaudier@argeo.org>
|
||||
Mathijs de Bruin <mathijs@mathijsfietst.nl>
|
||||
Matt Stephenson <mstephenson6@users.noreply.github.com>
|
||||
Max Krasnyansky <max.krasnyansky@gmail.com>
|
||||
Max Krasnyansky <quic_maxk@quicinc.com>
|
||||
Mayank Kumar Pal <mynkpl1998@gmail.com>
|
||||
Meng, Hengyu <hengyu.meng@intel.com>
|
||||
Mengqing Cao <cmq0113@163.com>
|
||||
Metal Whale <45712559+metalwhale@users.noreply.github.com>
|
||||
Michael Klimenko <mklimenko29@gmail.com>
|
||||
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||
Michael Verrilli <msv@pobox.com>
|
||||
Molly Sophia <mollysophia379@gmail.com>
|
||||
Natsu <chino@hotococoa.moe>
|
||||
Neo Zhang <14088817+arthw@users.noreply.github.com>
|
||||
Neo Zhang Jianyu <jianyu.zhang@intel.com>
|
||||
Neuman Vong <neuman.vong@gmail.com>
|
||||
Nevin <nevinpuri1901@gmail.com>
|
||||
Nicholai Tukanov <nicholaitukanov@gmail.com>
|
||||
Nico Bosshard <nico@bosshome.ch>
|
||||
Nicolò Scipione <nicolo.scipione@codeplay.com>
|
||||
Nikita Sarychev <42014488+sARY77@users.noreply.github.com>
|
||||
Nouamane Tazi <nouamane98@gmail.com>
|
||||
Olivier Chafik <ochafik@google.com>
|
||||
Olivier Chafik <ochafik@users.noreply.github.com>
|
||||
Ondřej Čertík <ondrej@certik.us>
|
||||
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||
PAB <pierreantoine.bannier@gmail.com>
|
||||
Paul Tsochantaris <ptsochantaris@icloud.com>
|
||||
Peter <peter277@users.noreply.github.com>
|
||||
Philpax <me@philpax.me>
|
||||
Pierre Alexandre SCHEMBRI <pa.schembri@gmail.com>
|
||||
Plamen Minev <pacominev@gmail.com>
|
||||
Playdev <josang1204@gmail.com>
|
||||
Prashant Vithule <119530321+Vithulep@users.noreply.github.com>
|
||||
Przemysław Pawełczyk <przemoc@gmail.com>
|
||||
R0CKSTAR <xiaodong.ye@mthreads.com>
|
||||
R0CKSTAR <yeahdongcn@gmail.com>
|
||||
Radoslav Gerganov <rgerganov@gmail.com>
|
||||
Radosław Gryta <radek.gryta@gmail.com>
|
||||
Ravindra Marella <marella@users.noreply.github.com>
|
||||
Ray Cromwell <cromwellian@gmail.com>
|
||||
Reinforce-II <fate@eastal.com>
|
||||
Rémy Oudompheng <oudomphe@phare.normalesup.org>
|
||||
Reza Rezvan <reza@rezvan.xyz>
|
||||
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||
RiverZhou <riverzhou2000@gmail.com>
|
||||
Robert Ormandi <52251610+ormandi@users.noreply.github.com>
|
||||
Romain Biessy <romain.biessy@codeplay.com>
|
||||
Ronsor <ronsor@ronsor.pw>
|
||||
Rotem Dan <rotemdan@gmail.com>
|
||||
Ryan Hitchman <hitchmanr@gmail.com>
|
||||
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
|
||||
SXX <sxx1136965276@gmail.com>
|
||||
Salvatore Mesoraca <s.mesoraca16@gmail.com>
|
||||
Sam Spilsbury <smspillaz@gmail.com>
|
||||
Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>
|
||||
Santtu Keskinen <santtu.keskinen@gmail.com>
|
||||
Sergio López <slp@redhat.com>
|
||||
Sergio López <slp@sinrega.org>
|
||||
Shanshan Shen <467638484@qq.com>
|
||||
Shijie <821898965@qq.com>
|
||||
Shupei Fan <dymarkfan@outlook.com>
|
||||
Siddharth Ramakrishnan <srr2141@columbia.edu>
|
||||
Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
|
||||
Skyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com>
|
||||
Slava Primenko <primenko.s@gmail.com>
|
||||
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
|
||||
Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
|
||||
Supreet Sethi <supreet.sethi@gmail.com>
|
||||
Takuya Takeuchi <takuya.takeuchi.dev@gmail.com>
|
||||
Tamotsu Takahashi <ttakah+github@gmail.com>
|
||||
Tanmay <tnmysachan@gmail.com>
|
||||
Tanmay Sachan <tnmysachan@gmail.com>
|
||||
Timothy Cronin <40186632+4imothy@users.noreply.github.com>
|
||||
Tom Bailey <tombailey@users.noreply.github.com>
|
||||
Tom Jobbins <784313+TheBloke@users.noreply.github.com>
|
||||
Tony Wasserka <4840017+neobrain@users.noreply.github.com>
|
||||
Tristan Druyen <tristan@vault81.mozmail.com>
|
||||
Tyé singwa <92231658+tye-singwa@users.noreply.github.com>
|
||||
UEXTM.com <84163508+uextm@users.noreply.github.com>
|
||||
WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com>
|
||||
William Tambellini <william.tambellini@gmail.com>
|
||||
William Tambellini <wtambellini@sdl.com>
|
||||
XiaotaoChen <chenxiaotao1234@gmail.com>
|
||||
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
|
||||
Xuan Son Nguyen <thichthat@gmail.com>
|
||||
Yavor Ivanov <yivanov@viewray.com>
|
||||
YavorGIvanov <yivanov@viewray.com>
|
||||
Yilong Guo <vfirst218@gmail.com>
|
||||
Yilong Guo <yilong.guo@intel.com>
|
||||
Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
|
||||
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
|
||||
Zhiyuan Li <lizhiyuan@uniartisan.com>
|
||||
Zhiyuan Li <uniartisan2017@gmail.com>
|
||||
a3sh <38979186+A3shTnT@users.noreply.github.com>
|
||||
ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>
|
||||
agray3 <agray3@users.noreply.github.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amritahs-ibm <amritahs@linux.vnet.ibm.com>
|
||||
apcameron <37645737+apcameron@users.noreply.github.com>
|
||||
appvoid <78444142+appvoid@users.noreply.github.com>
|
||||
ariez-xyz <41232910+ariez-xyz@users.noreply.github.com>
|
||||
automaticcat <daogiatuank54@gmail.com>
|
||||
bandoti <141645996+bandoti@users.noreply.github.com>
|
||||
bmwl <brian.marshall@tolko.com>
|
||||
bobqianic <129547291+bobqianic@users.noreply.github.com>
|
||||
bssrdf <merlintiger@hotmail.com>
|
||||
chengchi <davesjoewang@gmail.com>
|
||||
compilade <113953597+compilade@users.noreply.github.com>
|
||||
compilade <git@compilade.net>
|
||||
ddpasa <112642920+ddpasa@users.noreply.github.com>
|
||||
denersc <denerstassun@gmail.com>
|
||||
dscripka <dscripka@users.noreply.github.com>
|
||||
fitzsim <fitzsim@fitzsim.org>
|
||||
fj-y-saito <85871716+fj-y-saito@users.noreply.github.com>
|
||||
fraxy-v <65565042+fraxy-v@users.noreply.github.com>
|
||||
gn64 <yukikaze.jp@gmail.com>
|
||||
goerch <jhr.walter@t-online.de>
|
||||
goldwaving <77494627+goldwaving@users.noreply.github.com>
|
||||
haopeng <657407891@qq.com>
|
||||
hidenorly <hidenorly@users.noreply.github.com>
|
||||
hipudding <huafengchun@gmail.com>
|
||||
hydai <z54981220@gmail.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
jaeminSon <woalsdnd@gmail.com>
|
||||
jdomke <28772296+jdomke@users.noreply.github.com>
|
||||
jiez <373447296@qq.com>
|
||||
johnson442 <56517414+johnson442@users.noreply.github.com>
|
||||
junchao-loongson <68935141+junchao-loongson@users.noreply.github.com>
|
||||
k.h.lai <adrian.k.h.lai@outlook.com>
|
||||
katsu560 <118887472+katsu560@users.noreply.github.com>
|
||||
klosax <131523366+klosax@users.noreply.github.com>
|
||||
kunnis <kunnis@users.noreply.github.com>
|
||||
l3utterfly <gc.pthzfoldr@gmail.com>
|
||||
le.chang <cljs118@126.com>
|
||||
leejet <31925346+leejet@users.noreply.github.com>
|
||||
leejet <leejet714@gmail.com>
|
||||
leo-pony <nengjunma@outlook.com>
|
||||
lhez <quic_lih@quicinc.com>
|
||||
liuwei-git <14815172+liuwei-git@users.noreply.github.com>
|
||||
luoyu-intel <yu.luo@intel.com>
|
||||
magicse <magicse@users.noreply.github.com>
|
||||
mahorozte <41834471+mahorozte@users.noreply.github.com>
|
||||
mashizora <30516315+mashizora@users.noreply.github.com>
|
||||
matt23654 <matthew.webber@protonmail.com>
|
||||
matteo <matteogeniaccio@yahoo.it>
|
||||
ochafik <ochafik@google.com>
|
||||
otaGran <ujt2h8@gmail.com>
|
||||
pengxin99 <pengxin.yuan@intel.com>
|
||||
pikalover6 <49179590+pikalover6@users.noreply.github.com>
|
||||
postmasters <namnguyen@google.com>
|
||||
sjinzh <sjinzh@gmail.com>
|
||||
skirodev <57715494+skirodev@users.noreply.github.com>
|
||||
slaren <slarengh@gmail.com>
|
||||
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||
someone13574 <81528246+someone13574@users.noreply.github.com>
|
||||
stduhpf <stephduh@live.fr>
|
||||
taher <8665427+nullhook@users.noreply.github.com>
|
||||
texmex76 <40733439+texmex76@users.noreply.github.com>
|
||||
the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com>
|
||||
thewh1teagle <61390950+thewh1teagle@users.noreply.github.com>
|
||||
ucag.li <ucag@qq.com>
|
||||
ulatekh <ulatekh@yahoo.com>
|
||||
uvos <devnull@uvos.xyz>
|
||||
uvos <philipp@uvos.xyz>
|
||||
wangshuai09 <391746016@qq.com>
|
||||
woachk <24752637+woachk@users.noreply.github.com>
|
||||
xctan <axunlei@gmail.com>
|
||||
yangyaofei <yangyaofei@gmail.com>
|
||||
yuri@FreeBSD <yuri@FreeBSD>
|
||||
zhentaoyu <zhentao.yu@intel.com>
|
||||
zhouwg <6889919+zhouwg@users.noreply.github.com>
|
||||
zhouwg <zhouwg2000@gmail.com>
|
||||
谢乃闻 <sienaiwun@users.noreply.github.com>
|
||||
布客飞龙 <562826179@qq.com>
|
||||
旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com>
|
||||
@@ -0,0 +1,505 @@
|
||||
cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.
|
||||
|
||||
project("ggml" C CXX ASM)
|
||||
|
||||
### GGML Version
|
||||
set(GGML_VERSION_MAJOR 0)
|
||||
set(GGML_VERSION_MINOR 15)
|
||||
set(GGML_VERSION_PATCH 2)
|
||||
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
|
||||
|
||||
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
|
||||
if(GIT_EXE)
|
||||
# Get current git commit hash
|
||||
execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
OUTPUT_VARIABLE GGML_BUILD_COMMIT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
ERROR_QUIET
|
||||
)
|
||||
|
||||
# Check if the working directory is dirty (i.e., has uncommitted changes)
|
||||
execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- .
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
RESULT_VARIABLE GGML_GIT_DIRTY
|
||||
ERROR_QUIET
|
||||
)
|
||||
endif()
|
||||
|
||||
set(GGML_VERSION "${GGML_VERSION_BASE}")
|
||||
|
||||
if(NOT GGML_BUILD_COMMIT)
|
||||
set(GGML_BUILD_COMMIT "unknown")
|
||||
endif()
|
||||
|
||||
# Build the commit string with optional dirty flag
|
||||
if(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1)
|
||||
set(GGML_BUILD_COMMIT "${GGML_BUILD_COMMIT}-dirty")
|
||||
endif()
|
||||
|
||||
include(CheckIncludeFileCXX)
|
||||
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
|
||||
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
|
||||
endif()
|
||||
|
||||
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
|
||||
set(GGML_STANDALONE ON)
|
||||
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
|
||||
# configure project version
|
||||
# TODO
|
||||
else()
|
||||
set(GGML_STANDALONE OFF)
|
||||
|
||||
if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (EMSCRIPTEN)
|
||||
set(BUILD_SHARED_LIBS_DEFAULT OFF)
|
||||
|
||||
option(GGML_WASM_SINGLE_FILE "ggml: embed WASM inside the generated ggml.js" ON)
|
||||
else()
|
||||
if (MINGW)
|
||||
set(BUILD_SHARED_LIBS_DEFAULT OFF)
|
||||
else()
|
||||
set(BUILD_SHARED_LIBS_DEFAULT ON)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# remove the lib prefix on win32 mingw
|
||||
if (WIN32)
|
||||
set(CMAKE_STATIC_LIBRARY_PREFIX "")
|
||||
set(CMAKE_SHARED_LIBRARY_PREFIX "")
|
||||
set(CMAKE_SHARED_MODULE_PREFIX "")
|
||||
endif()
|
||||
|
||||
option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
|
||||
option(GGML_BACKEND_DL "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
|
||||
set(GGML_BACKEND_DIR "" CACHE PATH "ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL")
|
||||
|
||||
#
|
||||
# option list
|
||||
#
|
||||
|
||||
# TODO: mark all options as advanced when not GGML_STANDALONE
|
||||
|
||||
if (APPLE)
|
||||
set(GGML_METAL_DEFAULT ON)
|
||||
set(GGML_BLAS_DEFAULT ON)
|
||||
set(GGML_BLAS_VENDOR_DEFAULT "Apple")
|
||||
else()
|
||||
set(GGML_METAL_DEFAULT OFF)
|
||||
set(GGML_BLAS_DEFAULT OFF)
|
||||
set(GGML_BLAS_VENDOR_DEFAULT "Generic")
|
||||
endif()
|
||||
|
||||
if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH})
|
||||
message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF")
|
||||
set(GGML_NATIVE_DEFAULT OFF)
|
||||
else()
|
||||
set(GGML_NATIVE_DEFAULT ON)
|
||||
endif()
|
||||
|
||||
# defaults
|
||||
if (NOT GGML_LLAMAFILE_DEFAULT)
|
||||
set(GGML_LLAMAFILE_DEFAULT OFF)
|
||||
endif()
|
||||
|
||||
if (NOT GGML_CUDA_GRAPHS_DEFAULT)
|
||||
set(GGML_CUDA_GRAPHS_DEFAULT OFF)
|
||||
endif()
|
||||
|
||||
# general
|
||||
option(GGML_STATIC "ggml: static link libraries" OFF)
|
||||
option(GGML_NATIVE "ggml: optimize the build for the current system" ${GGML_NATIVE_DEFAULT})
|
||||
option(GGML_LTO "ggml: enable link time optimization" OFF)
|
||||
option(GGML_CCACHE "ggml: use ccache if available" ON)
|
||||
|
||||
# debug
|
||||
option(GGML_ALL_WARNINGS "ggml: enable all compiler warnings" ON)
|
||||
option(GGML_ALL_WARNINGS_3RD_PARTY "ggml: enable all compiler warnings in 3rd party libs" OFF)
|
||||
option(GGML_GPROF "ggml: enable gprof" OFF)
|
||||
|
||||
# build
|
||||
option(GGML_FATAL_WARNINGS "ggml: enable -Werror flag" OFF)
|
||||
|
||||
# sanitizers
|
||||
option(GGML_SANITIZE_THREAD "ggml: enable thread sanitizer" OFF)
|
||||
option(GGML_SANITIZE_ADDRESS "ggml: enable address sanitizer" OFF)
|
||||
option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF)
|
||||
|
||||
# instruction set specific
|
||||
if (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)
|
||||
set(INS_ENB OFF)
|
||||
else()
|
||||
set(INS_ENB ON)
|
||||
endif()
|
||||
|
||||
message(DEBUG "GGML_NATIVE : ${GGML_NATIVE}")
|
||||
message(DEBUG "GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}")
|
||||
message(DEBUG "INS_ENB : ${INS_ENB}")
|
||||
|
||||
option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF)
|
||||
option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
|
||||
option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF)
|
||||
option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB})
|
||||
option(GGML_AVX "ggml: enable AVX" ${INS_ENB})
|
||||
option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF)
|
||||
option(GGML_AVX2 "ggml: enable AVX2" ${INS_ENB})
|
||||
option(GGML_BMI2 "ggml: enable BMI2" ${INS_ENB})
|
||||
option(GGML_AVX512 "ggml: enable AVX512F" OFF)
|
||||
option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI" OFF)
|
||||
option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI" OFF)
|
||||
option(GGML_AVX512_BF16 "ggml: enable AVX512-BF16" OFF)
|
||||
if (NOT MSVC)
|
||||
# in MSVC F16C and FMA is implied with AVX2/AVX512
|
||||
option(GGML_FMA "ggml: enable FMA" ${INS_ENB})
|
||||
option(GGML_F16C "ggml: enable F16C" ${INS_ENB})
|
||||
# MSVC does not seem to support AMX
|
||||
option(GGML_AMX_TILE "ggml: enable AMX-TILE" OFF)
|
||||
option(GGML_AMX_INT8 "ggml: enable AMX-INT8" OFF)
|
||||
option(GGML_AMX_BF16 "ggml: enable AMX-BF16" OFF)
|
||||
endif()
|
||||
option(GGML_LASX "ggml: enable lasx" ON)
|
||||
option(GGML_LSX "ggml: enable lsx" ON)
|
||||
option(GGML_RVV "ggml: enable rvv" ON)
|
||||
option(GGML_RV_ZFH "ggml: enable riscv zfh" ON)
|
||||
option(GGML_RV_ZVFH "ggml: enable riscv zvfh" ON)
|
||||
option(GGML_RV_ZICBOP "ggml: enable riscv zicbop" ON)
|
||||
option(GGML_RV_ZIHINTPAUSE "ggml: enable riscv zihintpause" ON)
|
||||
option(GGML_RV_ZVFBFWMA "ggml: enable riscv zvfbfwma" OFF)
|
||||
option(GGML_XTHEADVECTOR "ggml: enable xtheadvector" OFF)
|
||||
option(GGML_VXE "ggml: enable vxe" ${GGML_NATIVE})
|
||||
|
||||
option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
|
||||
set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM")
|
||||
set(GGML_CPU_POWERPC_CPUTYPE "" CACHE STRING "ggml: CPU type for PowerPC")
|
||||
|
||||
# ggml core
|
||||
set(GGML_SCHED_MAX_COPIES "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
|
||||
option(GGML_CPU "ggml: enable CPU backend" ON)
|
||||
option(GGML_SCHED_NO_REALLOC "ggml: disallow reallocations in ggml-alloc (for debugging)" OFF)
|
||||
|
||||
# 3rd party libs / backends
|
||||
option(GGML_ACCELERATE "ggml: enable Accelerate framework" ON)
|
||||
option(GGML_BLAS "ggml: use BLAS" ${GGML_BLAS_DEFAULT})
|
||||
set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING
|
||||
"ggml: BLAS library vendor")
|
||||
option(GGML_LLAMAFILE "ggml: use LLAMAFILE" ${GGML_LLAMAFILE_DEFAULT})
|
||||
|
||||
option(GGML_CUDA "ggml: use CUDA" OFF)
|
||||
option(GGML_MUSA "ggml: use MUSA" OFF)
|
||||
option(GGML_CUDA_FORCE_MMQ "ggml: use mmq kernels instead of cuBLAS" OFF)
|
||||
option(GGML_CUDA_FORCE_CUBLAS "ggml: always use cuBLAS instead of mmq kernels" OFF)
|
||||
set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
|
||||
"ggml: max. batch size for using peer access")
|
||||
option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF)
|
||||
option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF)
|
||||
option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON)
|
||||
option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF)
|
||||
option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT})
|
||||
option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON)
|
||||
set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING
|
||||
"ggml: cuda link binary compression mode; requires cuda 12.8+")
|
||||
set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size")
|
||||
|
||||
option(GGML_HIP "ggml: use HIP" OFF)
|
||||
option(GGML_HIP_GRAPHS "ggml: use HIP graph" ON)
|
||||
option(GGML_HIP_RCCL "ggml: use ROCm Collective Comm. Library" OFF)
|
||||
option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON)
|
||||
option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF)
|
||||
option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON)
|
||||
option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF)
|
||||
option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF)
|
||||
option(GGML_MUSA_MUDNN_COPY "ggml: enable muDNN for accelerated copy" OFF)
|
||||
option(GGML_VULKAN "ggml: use Vulkan" OFF)
|
||||
option(GGML_VULKAN_CHECK_RESULTS "ggml: run Vulkan op checks" OFF)
|
||||
option(GGML_VULKAN_DEBUG "ggml: enable Vulkan debug output" OFF)
|
||||
option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug output" OFF)
|
||||
option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF)
|
||||
option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF)
|
||||
option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF)
|
||||
option(GGML_WEBGPU "ggml: use WebGPU" OFF)
|
||||
option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF)
|
||||
option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF)
|
||||
option(GGML_WEBGPU_GPU_PROFILE "ggml: enable WebGPU profiling (GPU)" OFF)
|
||||
option(GGML_WEBGPU_JSPI "ggml: use JSPI for WebGPU" ON)
|
||||
option(GGML_ZDNN "ggml: use zDNN" OFF)
|
||||
option(GGML_VIRTGPU "ggml: use the VirtGPU/Virglrenderer API Remoting frontend" OFF)
|
||||
option(GGML_VIRTGPU_BACKEND "ggml: build the VirtGPU/Virglrenderer API Remoting backend" OFF)
|
||||
option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT})
|
||||
option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF)
|
||||
option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF)
|
||||
option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" ${GGML_METAL})
|
||||
set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
|
||||
"ggml: metal minimum macOS version")
|
||||
set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)")
|
||||
option(GGML_OPENMP "ggml: use OpenMP" ON)
|
||||
option(GGML_RPC "ggml: use RPC" OFF)
|
||||
option(GGML_SYCL "ggml: use SYCL" OFF)
|
||||
option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF)
|
||||
option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON)
|
||||
option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
|
||||
option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON)
|
||||
option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON)
|
||||
set (GGML_SYCL_TARGET "INTEL" CACHE STRING
|
||||
"ggml: sycl target device")
|
||||
set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
|
||||
"ggml: sycl device architecture")
|
||||
|
||||
option(GGML_OPENVINO "ggml: use OPENVINO" OFF)
|
||||
|
||||
option(GGML_OPENCL "ggml: use OpenCL" OFF)
|
||||
option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF)
|
||||
option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON)
|
||||
option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON)
|
||||
set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
|
||||
"ggml: OpenCL API version to target")
|
||||
|
||||
option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF)
|
||||
set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)")
|
||||
|
||||
# toolchain for vulkan-shaders-gen
|
||||
set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
|
||||
|
||||
option(GGML_ZENDNN "ggml: use ZenDNN" OFF)
|
||||
option(ZENDNN_ROOT "ggml: path to ZenDNN installation" "")
|
||||
|
||||
# extra artifacts
|
||||
option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE})
|
||||
option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
|
||||
|
||||
#
|
||||
# dependencies
|
||||
#
|
||||
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED true)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED true)
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
|
||||
#
|
||||
# build the library
|
||||
#
|
||||
|
||||
add_subdirectory(src)
|
||||
|
||||
#
|
||||
# tests and examples
|
||||
#
|
||||
|
||||
if (GGML_BUILD_TESTS)
|
||||
enable_testing()
|
||||
add_subdirectory(tests)
|
||||
endif ()
|
||||
|
||||
if (GGML_BUILD_EXAMPLES)
|
||||
add_subdirectory(examples)
|
||||
endif ()
|
||||
|
||||
#
|
||||
# install
|
||||
#
|
||||
|
||||
include(CMakePackageConfigHelpers)
|
||||
|
||||
# all public headers
|
||||
set(GGML_PUBLIC_HEADERS
|
||||
include/ggml.h
|
||||
include/ggml-cpu.h
|
||||
include/ggml-alloc.h
|
||||
include/ggml-backend.h
|
||||
include/ggml-blas.h
|
||||
include/ggml-cann.h
|
||||
include/ggml-cpp.h
|
||||
include/ggml-cuda.h
|
||||
include/ggml-opt.h
|
||||
include/ggml-metal.h
|
||||
include/ggml-rpc.h
|
||||
include/ggml-virtgpu.h
|
||||
include/ggml-sycl.h
|
||||
include/ggml-vulkan.h
|
||||
include/ggml-webgpu.h
|
||||
include/ggml-zendnn.h
|
||||
include/ggml-openvino.h
|
||||
include/gguf.h)
|
||||
|
||||
set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
|
||||
#if (GGML_METAL)
|
||||
# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
|
||||
#endif()
|
||||
install(TARGETS ggml LIBRARY PUBLIC_HEADER)
|
||||
install(TARGETS ggml-base LIBRARY)
|
||||
|
||||
if (GGML_STANDALONE)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in
|
||||
${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
|
||||
@ONLY)
|
||||
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
|
||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
|
||||
endif()
|
||||
|
||||
#
|
||||
# Create CMake package
|
||||
#
|
||||
|
||||
|
||||
|
||||
# Capture variables prefixed with GGML_.
|
||||
|
||||
set(variable_set_statements
|
||||
"
|
||||
####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() #######
|
||||
####### Any changes to this file will be overwritten by the next CMake run #######
|
||||
|
||||
")
|
||||
|
||||
set(GGML_SHARED_LIB ${BUILD_SHARED_LIBS})
|
||||
|
||||
get_cmake_property(all_variables VARIABLES)
|
||||
foreach(variable_name IN LISTS all_variables)
|
||||
if(variable_name MATCHES "^GGML_")
|
||||
string(REPLACE ";" "\\;"
|
||||
variable_value "${${variable_name}}")
|
||||
|
||||
set(variable_set_statements
|
||||
"${variable_set_statements}set(${variable_name} \"${variable_value}\")\n")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(GGML_VARIABLES_EXPANDED ${variable_set_statements})
|
||||
|
||||
# Create the CMake package and set install location.
|
||||
|
||||
set(GGML_INSTALL_VERSION ${GGML_VERSION})
|
||||
set(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files")
|
||||
set(GGML_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files")
|
||||
set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files")
|
||||
|
||||
configure_package_config_file(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in
|
||||
${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
|
||||
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml
|
||||
PATH_VARS GGML_INCLUDE_INSTALL_DIR
|
||||
GGML_LIB_INSTALL_DIR
|
||||
GGML_BIN_INSTALL_DIR)
|
||||
|
||||
write_basic_package_version_file(
|
||||
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
|
||||
VERSION ${GGML_INSTALL_VERSION}
|
||||
COMPATIBILITY SameMajorVersion)
|
||||
|
||||
target_compile_definitions(ggml-base PRIVATE
|
||||
GGML_VERSION="${GGML_INSTALL_VERSION}"
|
||||
GGML_COMMIT="${GGML_BUILD_COMMIT}"
|
||||
)
|
||||
message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
|
||||
message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}")
|
||||
|
||||
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
|
||||
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
|
||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
|
||||
|
||||
if (MSVC)
|
||||
set(MSVC_WARNING_FLAGS
|
||||
/wd4005 # Macro redefinition
|
||||
/wd4244 # Conversion from one type to another type, possible loss of data
|
||||
/wd4267 # Conversion from 'size_t' to a smaller type, possible loss of data
|
||||
/wd4305 # Conversion from 'type1' to 'type2', possible loss of data
|
||||
/wd4566 # Conversion from 'char' to 'wchar_t', possible loss of data
|
||||
/wd4996 # Disable POSIX deprecation warnings
|
||||
/wd4702 # Unreachable code warnings
|
||||
)
|
||||
set(MSVC_COMPILE_OPTIONS
|
||||
"$<$<COMPILE_LANGUAGE:C>:/utf-8>"
|
||||
"$<$<COMPILE_LANGUAGE:CXX>:/utf-8>"
|
||||
)
|
||||
function(configure_msvc_target target_name)
|
||||
if(TARGET ${target_name})
|
||||
target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})
|
||||
target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS})
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
configure_msvc_target(ggml-base)
|
||||
configure_msvc_target(ggml)
|
||||
configure_msvc_target(ggml-cpu)
|
||||
configure_msvc_target(ggml-cpu-x64)
|
||||
configure_msvc_target(ggml-cpu-sse42)
|
||||
configure_msvc_target(ggml-cpu-sandybridge)
|
||||
# __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
|
||||
# skipping ggml-cpu-ivybridge
|
||||
# skipping ggml-cpu-piledriver
|
||||
configure_msvc_target(ggml-cpu-haswell)
|
||||
configure_msvc_target(ggml-cpu-skylakex)
|
||||
configure_msvc_target(ggml-cpu-cannonlake)
|
||||
configure_msvc_target(ggml-cpu-cascadelake)
|
||||
configure_msvc_target(ggml-cpu-icelake)
|
||||
# MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!
|
||||
# https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170
|
||||
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170
|
||||
# skipping ggml-cpu-cooperlake
|
||||
# skipping ggml-cpu-zen4
|
||||
configure_msvc_target(ggml-cpu-alderlake)
|
||||
# MSVC doesn't support AMX
|
||||
# skipping ggml-cpu-sapphirerapids
|
||||
|
||||
if (GGML_BUILD_EXAMPLES)
|
||||
configure_msvc_target(common-ggml)
|
||||
configure_msvc_target(common)
|
||||
|
||||
configure_msvc_target(mnist-common)
|
||||
configure_msvc_target(mnist-eval)
|
||||
configure_msvc_target(mnist-train)
|
||||
|
||||
configure_msvc_target(gpt-2-ctx)
|
||||
configure_msvc_target(gpt-2-alloc)
|
||||
configure_msvc_target(gpt-2-backend)
|
||||
configure_msvc_target(gpt-2-sched)
|
||||
configure_msvc_target(gpt-2-quantize)
|
||||
configure_msvc_target(gpt-2-batched)
|
||||
|
||||
configure_msvc_target(gpt-j)
|
||||
configure_msvc_target(gpt-j-quantize)
|
||||
|
||||
configure_msvc_target(magika)
|
||||
configure_msvc_target(yolov3-tiny)
|
||||
configure_msvc_target(sam)
|
||||
|
||||
configure_msvc_target(simple-ctx)
|
||||
configure_msvc_target(simple-backend)
|
||||
endif()
|
||||
|
||||
if (GGML_BUILD_TESTS)
|
||||
configure_msvc_target(test-mul-mat)
|
||||
configure_msvc_target(test-arange)
|
||||
configure_msvc_target(test-backend-ops)
|
||||
configure_msvc_target(test-cont)
|
||||
configure_msvc_target(test-conv-transpose)
|
||||
configure_msvc_target(test-conv-transpose-1d)
|
||||
configure_msvc_target(test-conv1d)
|
||||
configure_msvc_target(test-conv2d)
|
||||
configure_msvc_target(test-conv2d-dw)
|
||||
configure_msvc_target(test-customop)
|
||||
configure_msvc_target(test-dup)
|
||||
configure_msvc_target(test-opt)
|
||||
configure_msvc_target(test-pool)
|
||||
endif ()
|
||||
endif()
|
||||
@@ -0,0 +1,3 @@
|
||||
Please use [llama.cpp's contribution guidelines](https://github.com/ggml-org/llama.cpp/blob/master/CONTRIBUTING.md) for this project.
|
||||
|
||||
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023-2026 The ggml authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,50 @@
|
||||
# ggml
|
||||
|
||||
[Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205)
|
||||
|
||||
Tensor library for machine learning
|
||||
|
||||
***Note that this project is under active development. \
|
||||
Some of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos***
|
||||
|
||||
## Features
|
||||
|
||||
- Low-level cross-platform implementation
|
||||
- Integer quantization support
|
||||
- Broad hardware support
|
||||
- Automatic differentiation
|
||||
- ADAM and L-BFGS optimizers
|
||||
- No third-party dependencies
|
||||
- Zero memory allocations during runtime
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggml-org/ggml
|
||||
cd ggml
|
||||
|
||||
# install python dependencies in a virtual environment
|
||||
python3.10 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
# build the examples
|
||||
mkdir build && cd build
|
||||
cmake ..
|
||||
cmake --build . --config Release -j 8
|
||||
```
|
||||
|
||||
## GPT inference (example)
|
||||
|
||||
```bash
|
||||
# run the GPT-2 small 117M model
|
||||
../examples/gpt-2/download-ggml-model.sh 117M
|
||||
./bin/gpt-2-backend -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
|
||||
```
|
||||
|
||||
For more information, checkout the corresponding programs in the [examples](examples) folder.
|
||||
|
||||
## Resources
|
||||
|
||||
- [Introduction to ggml](https://huggingface.co/blog/introduction-to-ggml)
|
||||
- [The GGUF file format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md)
|
||||
+395
@@ -0,0 +1,395 @@
|
||||
#/bin/bash
|
||||
#
|
||||
# sample usage:
|
||||
#
|
||||
# mkdir tmp
|
||||
#
|
||||
# # CPU-only build
|
||||
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
#
|
||||
# # with CUDA support
|
||||
# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
#
|
||||
# # With SYCL support
|
||||
# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
||||
#
|
||||
|
||||
if [ -z "$2" ]; then
|
||||
echo "usage: $0 <output-dir> <mnt-dir>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$1"
|
||||
mkdir -p "$2"
|
||||
|
||||
OUT=$(realpath "$1")
|
||||
MNT=$(realpath "$2")
|
||||
|
||||
rm -v $OUT/*.log
|
||||
rm -v $OUT/*.exit
|
||||
rm -v $OUT/*.md
|
||||
|
||||
sd=`dirname $0`
|
||||
cd $sd/../
|
||||
SRC=`pwd`
|
||||
|
||||
CMAKE_EXTRA=""
|
||||
CTEST_EXTRA=""
|
||||
|
||||
if [ ! -z ${GG_BUILD_METAL} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_CUDA} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON"
|
||||
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
|
||||
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
|
||||
else
|
||||
echo "Warning: Using fallback CUDA architectures"
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
|
||||
fi
|
||||
else
|
||||
echo "Error: nvidia-smi not found, cannot build with CUDA"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_ROCM} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
|
||||
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
|
||||
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_SYCL} ]; then
|
||||
if [ -z ${ONEAPI_ROOT} ]; then
|
||||
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
|
||||
echo "source /opt/intel/oneapi/setvars.sh"
|
||||
exit 1
|
||||
fi
|
||||
# Use only main GPU
|
||||
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
|
||||
# Enable sysman for correct memory reporting
|
||||
export ZES_ENABLE_SYSMAN=1
|
||||
# to circumvent precision issues on CPY operations
|
||||
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
|
||||
|
||||
# if on Mac, disable METAL
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_MUSA} ]; then
|
||||
# Use qy1 by default (MTT S80)
|
||||
MUSA_ARCH=${MUSA_ARCH:-21}
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
|
||||
fi
|
||||
|
||||
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
|
||||
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
|
||||
fi
|
||||
|
||||
## helpers
|
||||
|
||||
# download a file if it does not exist or if it is outdated
|
||||
function gg_wget {
|
||||
local out=$1
|
||||
local url=$2
|
||||
|
||||
local cwd=`pwd`
|
||||
|
||||
mkdir -p $out
|
||||
cd $out
|
||||
|
||||
# should not re-download if file is the same
|
||||
wget -nv -N $url
|
||||
|
||||
cd $cwd
|
||||
}
|
||||
|
||||
function gg_printf {
|
||||
printf -- "$@" >> $OUT/README.md
|
||||
}
|
||||
|
||||
function gg_run {
|
||||
ci=$1
|
||||
|
||||
set -o pipefail
|
||||
set -x
|
||||
|
||||
gg_run_$ci | tee $OUT/$ci.log
|
||||
cur=$?
|
||||
echo "$cur" > $OUT/$ci.exit
|
||||
|
||||
set +x
|
||||
set +o pipefail
|
||||
|
||||
gg_sum_$ci
|
||||
|
||||
ret=$((ret | cur))
|
||||
}
|
||||
|
||||
## ci
|
||||
|
||||
# ctest_debug
|
||||
|
||||
function gg_run_ctest_debug {
|
||||
cd ${SRC}
|
||||
|
||||
rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
|
||||
|
||||
set -e
|
||||
|
||||
(time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
(time ctest ${CTEST_EXTRA} --output-on-failure -E "test-opt|test-backend-ops" ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_debug {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in debug mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '\n'
|
||||
}
|
||||
|
||||
# ctest_release
|
||||
|
||||
function gg_run_ctest_release {
|
||||
cd ${SRC}
|
||||
|
||||
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
|
||||
|
||||
set -e
|
||||
|
||||
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
|
||||
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
|
||||
|
||||
if [ -z $GG_BUILD_LOW_PERF ]; then
|
||||
(time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
else
|
||||
(time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
|
||||
fi
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_ctest_release {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs ctest in release mode\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# gpt_2
|
||||
|
||||
function gg_run_gpt_2 {
|
||||
cd ${SRC}
|
||||
|
||||
gg_wget models-mnt/gpt-2 https://huggingface.co/ggerganov/ggml/resolve/main/ggml-model-gpt-2-117M.bin
|
||||
|
||||
cd build-ci-release
|
||||
|
||||
set -e
|
||||
|
||||
model="../models-mnt/gpt-2/ggml-model-gpt-2-117M.bin"
|
||||
prompts="../examples/prompts/gpt-2.txt"
|
||||
|
||||
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -tt ${prompts} ) 2>&1 | tee -a $OUT/${ci}-tg.log
|
||||
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
|
||||
(time ./bin/gpt-2-sched --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
|
||||
|
||||
(time ./bin/gpt-2-batched --model ${model} -s 1234 -n 64 -np 8 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_gpt_2 {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Runs short GPT-2 text generation\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-tg.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# TODO: update
|
||||
## mnist
|
||||
#
|
||||
#function gg_run_mnist {
|
||||
# cd ${SRC}
|
||||
#
|
||||
# cd build-ci-release
|
||||
#
|
||||
# set -e
|
||||
#
|
||||
# mkdir -p models/mnist
|
||||
# python3 ../examples/mnist/convert-h5-to-ggml.py ../examples/mnist/models/mnist/mnist_model.state_dict
|
||||
#
|
||||
# model_f32="./models/mnist/ggml-model-f32.bin"
|
||||
# samples="../examples/mnist/models/mnist/t10k-images.idx3-ubyte"
|
||||
#
|
||||
# # first command runs and exports "mnist.ggml", the second command runs the exported model
|
||||
#
|
||||
# (time ./bin/mnist ${model_f32} ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
|
||||
# (time ./bin/mnist-cpu ./mnist.ggml ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
|
||||
#
|
||||
# set +e
|
||||
#}
|
||||
#
|
||||
#function gg_sum_mnist {
|
||||
# gg_printf '### %s\n\n' "${ci}"
|
||||
#
|
||||
# gg_printf 'MNIST\n'
|
||||
# gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
# gg_printf '```\n'
|
||||
# gg_printf '%s\n' "$(cat $OUT/${ci}-mnist.log)"
|
||||
# gg_printf '```\n'
|
||||
#}
|
||||
|
||||
# sam
|
||||
|
||||
function gg_run_sam {
|
||||
cd ${SRC}
|
||||
|
||||
gg_wget models-mnt/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth
|
||||
gg_wget models-mnt/sam/ https://raw.githubusercontent.com/YavorGIvanov/sam.cpp/ceafb7467bff7ec98e0c4f952e58a9eb8fd0238b/img.jpg
|
||||
|
||||
cd build-ci-release
|
||||
|
||||
set -e
|
||||
|
||||
path_models="../models-mnt/sam/"
|
||||
model_f16="${path_models}/ggml-model-f16.bin"
|
||||
img_0="${path_models}/img.jpg"
|
||||
|
||||
python3 ../examples/sam/convert-pth-to-ggml.py ${path_models}/sam_vit_b_01ec64.pth ${path_models}/ 1
|
||||
|
||||
# Test default parameters
|
||||
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 ) 2>&1 | tee -a $OUT/${ci}-main.log
|
||||
grep -q "point prompt" $OUT/${ci}-main.log
|
||||
grep -q "bbox (371, 436), (144, 168)" $OUT/${ci}-main.log ||
|
||||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
|
||||
|
||||
# Test box prompt and single mask output
|
||||
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 -b 368,144,441,173 -sm) 2>&1 | tee -a $OUT/${ci}-main.log
|
||||
grep -q "box prompt" $OUT/${ci}-main.log
|
||||
grep -q "bbox (370, 439), (144, 169)" $OUT/${ci}-main.log ||
|
||||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_sam {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Run SAM\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
# yolo
|
||||
|
||||
function gg_run_yolo {
|
||||
cd ${SRC}
|
||||
|
||||
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/yolov3-tiny.weights
|
||||
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/dog.jpg
|
||||
|
||||
cd build-ci-release
|
||||
cp -r ../examples/yolo/data .
|
||||
|
||||
set -e
|
||||
|
||||
path_models="../models-mnt/yolo/"
|
||||
|
||||
python3 ../examples/yolo/convert-yolov3-tiny.py ${path_models}/yolov3-tiny.weights
|
||||
|
||||
(time ./bin/yolov3-tiny -m yolov3-tiny.gguf -i ${path_models}/dog.jpg ) 2>&1 | tee -a $OUT/${ci}-main.log
|
||||
|
||||
grep -qE "dog: (55|56|57|58|59)%" $OUT/${ci}-main.log
|
||||
grep -qE "car: (50|51|52|53|54)%" $OUT/${ci}-main.log
|
||||
grep -qE "truck: (54|55|56|57|58)%" $OUT/${ci}-main.log
|
||||
grep -qE "bicycle: (57|58|59|60|61)%" $OUT/${ci}-main.log
|
||||
|
||||
set +e
|
||||
}
|
||||
|
||||
function gg_sum_yolo {
|
||||
gg_printf '### %s\n\n' "${ci}"
|
||||
|
||||
gg_printf 'Run YOLO\n'
|
||||
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
|
||||
gg_printf '```\n'
|
||||
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
|
||||
gg_printf '```\n'
|
||||
}
|
||||
|
||||
## main
|
||||
|
||||
if true ; then
|
||||
# Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt
|
||||
rm -rf ${SRC}/models-mnt
|
||||
mnt_models=${MNT}/models
|
||||
mkdir -p ${mnt_models}
|
||||
ln -sfn ${mnt_models} ${SRC}/models-mnt
|
||||
|
||||
# Create a fresh python3 venv and enter it
|
||||
if ! python3 -m venv "$MNT/venv"; then
|
||||
echo "Error: Failed to create Python virtual environment at $MNT/venv."
|
||||
exit 1
|
||||
fi
|
||||
source "$MNT/venv/bin/activate"
|
||||
|
||||
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
|
||||
fi
|
||||
|
||||
|
||||
ret=0
|
||||
|
||||
test $ret -eq 0 && gg_run ctest_debug
|
||||
test $ret -eq 0 && gg_run ctest_release
|
||||
|
||||
test $ret -eq 0 && gg_run gpt_2
|
||||
#test $ret -eq 0 && gg_run mnist
|
||||
test $ret -eq 0 && gg_run sam
|
||||
test $ret -eq 0 && gg_run yolo
|
||||
|
||||
if [ -z $GG_BUILD_LOW_PERF ]; then
|
||||
# run tests meant for low-perf runners
|
||||
date
|
||||
fi
|
||||
|
||||
cat $OUT/README.md
|
||||
|
||||
exit $ret
|
||||
@@ -0,0 +1,36 @@
|
||||
# cmake/FindNCCL.cmake
|
||||
|
||||
# NVIDIA does not distribute CMake files with NCCl, therefore use this file to find it instead.
|
||||
|
||||
find_path(NCCL_INCLUDE_DIR
|
||||
NAMES nccl.h
|
||||
HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda
|
||||
PATH_SUFFIXES include
|
||||
)
|
||||
|
||||
find_library(NCCL_LIBRARY
|
||||
NAMES nccl
|
||||
HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda
|
||||
PATH_SUFFIXES lib lib64
|
||||
)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(NCCL
|
||||
DEFAULT_MSG
|
||||
NCCL_LIBRARY NCCL_INCLUDE_DIR
|
||||
)
|
||||
|
||||
if(NCCL_FOUND)
|
||||
set(NCCL_LIBRARIES ${NCCL_LIBRARY})
|
||||
set(NCCL_INCLUDE_DIRS ${NCCL_INCLUDE_DIR})
|
||||
|
||||
if(NOT TARGET NCCL::NCCL)
|
||||
add_library(NCCL::NCCL UNKNOWN IMPORTED)
|
||||
set_target_properties(NCCL::NCCL PROPERTIES
|
||||
IMPORTED_LOCATION "${NCCL_LIBRARY}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}"
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
mark_as_advanced(NCCL_INCLUDE_DIR NCCL_LIBRARY)
|
||||
@@ -0,0 +1,22 @@
|
||||
find_package(Git)
|
||||
|
||||
# the commit's SHA1
|
||||
execute_process(COMMAND
|
||||
"${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_SHA1
|
||||
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
|
||||
# the date of the commit
|
||||
execute_process(COMMAND
|
||||
"${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_DATE
|
||||
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
|
||||
# the subject of the commit
|
||||
execute_process(COMMAND
|
||||
"${GIT_EXECUTABLE}" log -1 --format=%s
|
||||
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
|
||||
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
@@ -0,0 +1,50 @@
|
||||
function(ggml_get_flags CCID CCVER)
|
||||
set(C_FLAGS "")
|
||||
set(CXX_FLAGS "")
|
||||
|
||||
if (CCID MATCHES "Clang")
|
||||
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
|
||||
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
|
||||
|
||||
if (
|
||||
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
|
||||
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
|
||||
)
|
||||
list(APPEND C_FLAGS -Wdouble-promotion)
|
||||
endif()
|
||||
elseif (CCID STREQUAL "GNU")
|
||||
set(C_FLAGS -Wdouble-promotion)
|
||||
set(CXX_FLAGS -Wno-array-bounds)
|
||||
|
||||
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
|
||||
list(APPEND CXX_FLAGS -Wextra-semi)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
|
||||
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(ggml_get_system_arch)
|
||||
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
|
||||
CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
|
||||
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
||||
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
|
||||
set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
|
||||
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
|
||||
CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
|
||||
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
|
||||
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
|
||||
set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
|
||||
set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
|
||||
set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE)
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
|
||||
set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
|
||||
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
|
||||
set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
|
||||
else()
|
||||
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,201 @@
|
||||
@PACKAGE_INIT@
|
||||
|
||||
@GGML_VARIABLES_EXPANDED@
|
||||
|
||||
# Find all dependencies before creating any target.
|
||||
include(CMakeFindDependencyMacro)
|
||||
find_dependency(Threads)
|
||||
if (NOT GGML_SHARED_LIB)
|
||||
set(GGML_BASE_INTERFACE_LINK_LIBRARIES "")
|
||||
set(GGML_CPU_INTERFACE_LINK_LIBRARIES "")
|
||||
set(GGML_CPU_INTERFACE_LINK_OPTIONS "")
|
||||
|
||||
if (APPLE AND GGML_ACCELERATE)
|
||||
find_library(ACCELERATE_FRAMEWORK Accelerate)
|
||||
if(NOT ACCELERATE_FRAMEWORK)
|
||||
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
|
||||
return()
|
||||
endif()
|
||||
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK})
|
||||
endif()
|
||||
|
||||
if (GGML_OPENMP_ENABLED)
|
||||
find_dependency(OpenMP)
|
||||
set(GGML_OPENMP_INTERFACE_LINK_LIBRARIES "")
|
||||
if (TARGET OpenMP::OpenMP_C)
|
||||
list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C)
|
||||
endif()
|
||||
if (TARGET OpenMP::OpenMP_CXX)
|
||||
list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_CXX)
|
||||
endif()
|
||||
list(APPEND GGML_BASE_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES})
|
||||
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES})
|
||||
endif()
|
||||
|
||||
if (GGML_CPU_HBM)
|
||||
find_library(memkind memkind)
|
||||
if(NOT memkind)
|
||||
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
|
||||
return()
|
||||
endif()
|
||||
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind)
|
||||
endif()
|
||||
|
||||
if (GGML_BLAS)
|
||||
find_dependency(BLAS)
|
||||
list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES})
|
||||
list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS ${BLAS_LINKER_FLAGS})
|
||||
endif()
|
||||
|
||||
if (GGML_CUDA)
|
||||
set(GGML_CUDA_INTERFACE_LINK_LIBRARIES "")
|
||||
find_dependency(CUDAToolkit)
|
||||
if (GGML_STATIC)
|
||||
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cudart_static>)
|
||||
if (WIN32)
|
||||
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas> $<LINK_ONLY:CUDA::cublasLt>)
|
||||
else()
|
||||
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas_static> $<LINK_ONLY:CUDA::cublasLt_static>)
|
||||
endif()
|
||||
endif()
|
||||
if (NOT GGML_CUDA_NO_VMM)
|
||||
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cuda_driver>)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (GGML_METAL)
|
||||
find_library(FOUNDATION_LIBRARY Foundation)
|
||||
find_library(METAL_FRAMEWORK Metal)
|
||||
find_library(METALKIT_FRAMEWORK MetalKit)
|
||||
if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK)
|
||||
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
|
||||
return()
|
||||
endif()
|
||||
set(GGML_METAL_INTERFACE_LINK_LIBRARIES
|
||||
${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK})
|
||||
endif()
|
||||
|
||||
if (GGML_OPENCL)
|
||||
find_dependency(OpenCL)
|
||||
set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:OpenCL::OpenCL>)
|
||||
endif()
|
||||
|
||||
if (GGML_VULKAN)
|
||||
find_dependency(Vulkan)
|
||||
set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:Vulkan::Vulkan>)
|
||||
endif()
|
||||
|
||||
if (GGML_HIP)
|
||||
find_dependency(hip)
|
||||
find_dependency(hipblas)
|
||||
find_dependency(rocblas)
|
||||
set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas)
|
||||
endif()
|
||||
|
||||
if (GGML_SYCL)
|
||||
set(GGML_SYCL_INTERFACE_LINK_LIBRARIES "")
|
||||
find_package(DNNL)
|
||||
if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL")
|
||||
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl)
|
||||
endif()
|
||||
if (WIN32)
|
||||
find_dependency(IntelSYCL)
|
||||
find_dependency(MKL)
|
||||
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@")
|
||||
set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
|
||||
#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@")
|
||||
|
||||
if(NOT TARGET ggml::ggml)
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
find_library(GGML_LIBRARY ggml
|
||||
REQUIRED
|
||||
HINTS ${GGML_LIB_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH)
|
||||
|
||||
add_library(ggml::ggml UNKNOWN IMPORTED)
|
||||
set_target_properties(ggml::ggml
|
||||
PROPERTIES
|
||||
IMPORTED_LOCATION "${GGML_LIBRARY}")
|
||||
|
||||
find_library(GGML_BASE_LIBRARY ggml-base
|
||||
REQUIRED
|
||||
HINTS ${GGML_LIB_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH)
|
||||
|
||||
add_library(ggml::ggml-base UNKNOWN IMPORTED)
|
||||
set_target_properties(ggml::ggml-base
|
||||
PROPERTIES
|
||||
IMPORTED_LOCATION "${GGML_BASE_LIBRARY}"
|
||||
INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}")
|
||||
|
||||
set(_ggml_all_targets "")
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS})
|
||||
string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}")
|
||||
string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx)
|
||||
|
||||
find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}
|
||||
REQUIRED
|
||||
HINTS ${GGML_LIB_DIR}
|
||||
NO_CMAKE_FIND_ROOT_PATH)
|
||||
|
||||
message(STATUS "Found ${${_ggml_backend_pfx}_LIBRARY}")
|
||||
|
||||
add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED)
|
||||
set_target_properties(ggml::${_ggml_backend}
|
||||
PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}"
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
|
||||
IMPORTED_LOCATION "${${_ggml_backend_pfx}_LIBRARY}"
|
||||
INTERFACE_COMPILE_FEATURES c_std_90
|
||||
POSITION_INDEPENDENT_CODE ON)
|
||||
|
||||
string(REGEX MATCH "^ggml-cpu" is_cpu_variant "${_ggml_backend}")
|
||||
if(is_cpu_variant)
|
||||
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
|
||||
set_target_properties(ggml::${_ggml_backend}
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES "${GGML_CPU_INTERFACE_LINK_LIBRARIES}")
|
||||
|
||||
if(GGML_CPU_INTERFACE_LINK_OPTIONS)
|
||||
set_target_properties(ggml::${_ggml_backend}
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_OPTIONS "${GGML_CPU_INTERFACE_LINK_OPTIONS}")
|
||||
endif()
|
||||
|
||||
else()
|
||||
list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
|
||||
set_target_properties(ggml::${_ggml_backend}
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES "${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}")
|
||||
|
||||
if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS)
|
||||
set_target_properties(ggml::${_ggml_backend}
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_OPTIONS "${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
list(APPEND _ggml_all_targets ggml::${_ggml_backend})
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base "${_ggml_all_targets}")
|
||||
set_target_properties(ggml::ggml
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES "${GGML_INTERFACE_LINK_LIBRARIES}")
|
||||
|
||||
add_library(ggml::all INTERFACE IMPORTED)
|
||||
set_target_properties(ggml::all
|
||||
PROPERTIES
|
||||
INTERFACE_LINK_LIBRARIES "${_ggml_all_targets}")
|
||||
|
||||
endif()
|
||||
|
||||
check_required_components(ggml)
|
||||
@@ -0,0 +1,828 @@
|
||||
# GGUF
|
||||
|
||||
GGUF is a file format for storing models for inference with GGML and executors based on GGML. GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML.
|
||||
|
||||
It is a successor file format to GGML, GGMF and GGJT, and is designed to be unambiguous by containing all the information needed to load a model. It is also designed to be extensible, so that new information can be added to models without breaking compatibility.
|
||||
|
||||
For more information about the motivation behind GGUF, see [Historical State of Affairs](#historical-state-of-affairs).
|
||||
|
||||
## Specification
|
||||
|
||||
GGUF is a format based on the existing GGJT, but makes a few changes to the format to make it more extensible and easier to use. The following features are desired:
|
||||
|
||||
- Single-file deployment: they can be easily distributed and loaded, and do not require any external files for additional information.
|
||||
- Extensible: new features can be added to GGML-based executors/new information can be added to GGUF models without breaking compatibility with existing models.
|
||||
- `mmap` compatibility: models can be loaded using `mmap` for fast loading and saving.
|
||||
- Easy to use: models can be easily loaded and saved using a small amount of code, with no need for external libraries, regardless of the language used.
|
||||
- Full information: all information needed to load a model is contained in the model file, and no additional information needs to be provided by the user.
|
||||
|
||||
The key difference between GGJT and GGUF is the use of a key-value structure for the hyperparameters (now referred to as metadata), rather than a list of untyped values. This allows for new metadata to be added without breaking compatibility with existing models, and to annotate the model with additional information that may be useful for inference or for identifying the model.
|
||||
|
||||
### GGUF Naming Convention
|
||||
|
||||
GGUF follow a naming convention of `[<Sidecar>]<BaseName><SizeLabel><FineTune><Version><Encoding><Type><Shard>.gguf` where each component is delimitated by a `-` if present. Ultimately this is intended to make it easier for humans to at a glance get the most important details of a model. It is not intended to be perfectly parsable in the field due to the diversity of existing gguf filenames.
|
||||
|
||||
The components are:
|
||||
1. **Sidecar**: (Optional) Prefix marking the file as an auxiliary module loaded alongside a base model, rather than a standalone model. When present, sits at the very front of the filename followed by `-`. Lowercase by convention.
|
||||
- `mmproj` : Multimodal projector (vision/audio encoder and projection layer for use with a base LLM)
|
||||
- `mtp` : Multi-Token Prediction heads (speculative-decoding draft module, intended to be loaded alongside a base model of matching architecture and version). Note that oftentimes the MTP weights can be distributed inside the base model, in which case there is no separate `mtp-` sidecar file.
|
||||
1. **BaseName**: A descriptive name for the model base type or architecture.
|
||||
- This can be derived from gguf metadata `general.basename` substituting spaces for dashes.
|
||||
1. **SizeLabel**: Parameter weight class (useful for leader boards) represented as `<expertCount>x<count><scale-prefix>`
|
||||
- This can be derived from gguf metadata `general.size_label` if available or calculated if missing.
|
||||
- Rounded decimal point is supported in count with a single letter scale prefix to assist in floating point exponent shown below
|
||||
- `Q`: Quadrillion parameters.
|
||||
- `T`: Trillion parameters.
|
||||
- `B`: Billion parameters.
|
||||
- `M`: Million parameters.
|
||||
- `K`: Thousand parameters.
|
||||
- Additional `-<attributes><count><scale-prefix>` can be appended as needed to indicate other attributes of interest
|
||||
1. **FineTune**: A descriptive name for the model fine tuning goal (e.g. Chat, Instruct, etc...)
|
||||
- This can be derived from gguf metadata `general.finetune` substituting spaces for dashes.
|
||||
1. **Version**: (Optional) Denotes the model version number, formatted as `v<Major>.<Minor>`
|
||||
- If model is missing a version number then assume `v1.0` (First Public Release)
|
||||
- This can be derived from gguf metadata `general.version`
|
||||
1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs.
|
||||
1. **Type**: Indicates the kind of gguf file and the intended purpose for it
|
||||
- If missing, then file is by default a typical gguf tensor model file
|
||||
- `LoRA` : GGUF file is a LoRA adapter
|
||||
- `vocab` : GGUF file with only vocab data and metadata
|
||||
1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `<ShardNum>-of-<ShardTotal>`.
|
||||
- *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros.
|
||||
- Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`).
|
||||
- *ShardTotal* : Total number of shards in this model. Must be 5 digits padded by zeros.
|
||||
|
||||
|
||||
#### Validating Above Naming Convention
|
||||
|
||||
At a minimum all model files should have at least BaseName, SizeLabel, Version, in order to be easily validated as a file that is keeping with the GGUF Naming Convention. An example of this issue is that it is easy for Encoding to be mistaken as a FineTune if Version is omitted.
|
||||
|
||||
To validate you can use this regular expression `^(?:(?<Sidecar>mmproj|mtp)-)?(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$` which will check that you got the minimum BaseName, SizeLabel and Version present in the correct order.
|
||||
|
||||
For example:
|
||||
|
||||
* `Mixtral-8x7B-v0.1-KQ2.gguf`:
|
||||
- Model Name: Mixtral
|
||||
- Expert Count: 8
|
||||
- Parameter Count: 7B
|
||||
- Version Number: v0.1
|
||||
- Weight Encoding Scheme: KQ2
|
||||
|
||||
* `Hermes-2-Pro-Llama-3-8B-F16.gguf`:
|
||||
- Model Name: Hermes 2 Pro Llama 3
|
||||
- Expert Count: 0
|
||||
- Parameter Count: 8B
|
||||
- Version Number: v1.0
|
||||
- Weight Encoding Scheme: F16
|
||||
- Shard: N/A
|
||||
|
||||
* `Grok-100B-v1.0-Q4_0-00003-of-00009.gguf`
|
||||
- Model Name: Grok
|
||||
- Expert Count: 0
|
||||
- Parameter Count: 100B
|
||||
- Version Number: v1.0
|
||||
- Weight Encoding Scheme: Q4_0
|
||||
- Shard: 3 out of 9 total shards
|
||||
|
||||
* `mtp-Qwen3-27B-v1.0-Q4_K_M.gguf`
|
||||
- Sidecar: mtp (Multi-Token Prediction draft module)
|
||||
- Model Name: Qwen3
|
||||
- Expert Count: 0
|
||||
- Parameter Count: 27B (of the main model — sidecar tensors are smaller)
|
||||
- Version Number: v1.0
|
||||
- Weight Encoding Scheme: Q4_K_M
|
||||
|
||||
* `mmproj-Qwen2-VL-7B-v1.0-F16.gguf`
|
||||
- Sidecar: mmproj (multimodal projector)
|
||||
- Model Name: Qwen2-VL
|
||||
- Expert Count: 0
|
||||
- Parameter Count: 7B (of the main model — sidecar tensors are smaller)
|
||||
- Version Number: v1.0
|
||||
- Weight Encoding Scheme: F16
|
||||
|
||||
|
||||
<details><summary>Example Node.js Regex Function</summary>
|
||||
|
||||
```js
|
||||
#!/usr/bin/env node
|
||||
const ggufRegex = /^(?:(?<Sidecar>mmproj|mtp)-)?(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$/;
|
||||
|
||||
function parseGGUFFilename(filename) {
|
||||
const match = ggufRegex.exec(filename);
|
||||
if (!match)
|
||||
return null;
|
||||
const {Sidecar = null, BaseName = null, SizeLabel = null, FineTune = null, Version = "v1.0", Encoding = null, Type = null, Shard = null} = match.groups;
|
||||
return {Sidecar: Sidecar, BaseName: BaseName, SizeLabel: SizeLabel, FineTune: FineTune, Version: Version, Encoding: Encoding, Type: Type, Shard: Shard};
|
||||
}
|
||||
|
||||
const testCases = [
|
||||
{filename: 'Mixtral-8x7B-v0.1-KQ2.gguf', expected: { Sidecar: null, BaseName: 'Mixtral', SizeLabel: '8x7B', FineTune: null, Version: 'v0.1', Encoding: 'KQ2', Type: null, Shard: null}},
|
||||
{filename: 'Grok-100B-v1.0-Q4_0-00003-of-00009.gguf', expected: { Sidecar: null, BaseName: 'Grok', SizeLabel: '100B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_0', Type: null, Shard: "00003-of-00009"}},
|
||||
{filename: 'Hermes-2-Pro-Llama-3-8B-v1.0-F16.gguf', expected: { Sidecar: null, BaseName: 'Hermes-2-Pro-Llama-3', SizeLabel: '8B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},
|
||||
{filename: 'Phi-3-mini-3.8B-ContextLength4k-instruct-v1.0.gguf', expected: { Sidecar: null, BaseName: 'Phi-3-mini', SizeLabel: '3.8B-ContextLength4k', FineTune: 'instruct', Version: 'v1.0', Encoding: null, Type: null, Shard: null}},
|
||||
{filename: 'mtp-Qwen3-27B-v1.0-Q4_K_M.gguf', expected: { Sidecar: 'mtp', BaseName: 'Qwen3', SizeLabel: '27B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_K_M', Type: null, Shard: null}},
|
||||
{filename: 'mmproj-Qwen2-VL-7B-v1.0-F16.gguf', expected: { Sidecar: 'mmproj', BaseName: 'Qwen2-VL', SizeLabel: '7B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},
|
||||
{filename: 'not-a-known-arrangement.gguf', expected: null},
|
||||
];
|
||||
|
||||
testCases.forEach(({ filename, expected }) => {
|
||||
const result = parseGGUFFilename(filename);
|
||||
const passed = JSON.stringify(result) === JSON.stringify(expected);
|
||||
console.log(`${filename}: ${passed ? "PASS" : "FAIL"}`);
|
||||
if (!passed) {
|
||||
console.log(result);
|
||||
console.log(expected);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### File Structure
|
||||
|
||||
|
||||
*diagram by [@mishig25](https://github.com/mishig25) (GGUF v3)*
|
||||
|
||||
GGUF files are structured as follows. They use a global alignment specified in the `general.alignment` metadata field, referred to as `ALIGNMENT` below. Where required, the file is padded with `0x00` bytes to the next multiple of `general.alignment`.
|
||||
|
||||
Fields, including arrays, are written sequentially without alignment unless otherwise specified.
|
||||
|
||||
Models are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian.
|
||||
|
||||
```c
|
||||
enum ggml_type: uint32_t {
|
||||
GGML_TYPE_F32 = 0,
|
||||
GGML_TYPE_F16 = 1,
|
||||
GGML_TYPE_Q4_0 = 2,
|
||||
GGML_TYPE_Q4_1 = 3,
|
||||
// GGML_TYPE_Q4_2 = 4, support has been removed
|
||||
// GGML_TYPE_Q4_3 = 5, support has been removed
|
||||
GGML_TYPE_Q5_0 = 6,
|
||||
GGML_TYPE_Q5_1 = 7,
|
||||
GGML_TYPE_Q8_0 = 8,
|
||||
GGML_TYPE_Q8_1 = 9,
|
||||
GGML_TYPE_Q2_K = 10,
|
||||
GGML_TYPE_Q3_K = 11,
|
||||
GGML_TYPE_Q4_K = 12,
|
||||
GGML_TYPE_Q5_K = 13,
|
||||
GGML_TYPE_Q6_K = 14,
|
||||
GGML_TYPE_Q8_K = 15,
|
||||
GGML_TYPE_IQ2_XXS = 16,
|
||||
GGML_TYPE_IQ2_XS = 17,
|
||||
GGML_TYPE_IQ3_XXS = 18,
|
||||
GGML_TYPE_IQ1_S = 19,
|
||||
GGML_TYPE_IQ4_NL = 20,
|
||||
GGML_TYPE_IQ3_S = 21,
|
||||
GGML_TYPE_IQ2_S = 22,
|
||||
GGML_TYPE_IQ4_XS = 23,
|
||||
GGML_TYPE_I8 = 24,
|
||||
GGML_TYPE_I16 = 25,
|
||||
GGML_TYPE_I32 = 26,
|
||||
GGML_TYPE_I64 = 27,
|
||||
GGML_TYPE_F64 = 28,
|
||||
GGML_TYPE_IQ1_M = 29,
|
||||
GGML_TYPE_BF16 = 30,
|
||||
// GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
|
||||
// GGML_TYPE_Q4_0_4_8 = 32,
|
||||
// GGML_TYPE_Q4_0_8_8 = 33,
|
||||
GGML_TYPE_TQ1_0 = 34,
|
||||
GGML_TYPE_TQ2_0 = 35,
|
||||
// GGML_TYPE_IQ4_NL_4_4 = 36,
|
||||
// GGML_TYPE_IQ4_NL_4_8 = 37,
|
||||
// GGML_TYPE_IQ4_NL_8_8 = 38,
|
||||
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
|
||||
GGML_TYPE_COUNT = 40,
|
||||
};
|
||||
|
||||
enum gguf_metadata_value_type: uint32_t {
|
||||
// The value is a 8-bit unsigned integer.
|
||||
GGUF_METADATA_VALUE_TYPE_UINT8 = 0,
|
||||
// The value is a 8-bit signed integer.
|
||||
GGUF_METADATA_VALUE_TYPE_INT8 = 1,
|
||||
// The value is a 16-bit unsigned little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_UINT16 = 2,
|
||||
// The value is a 16-bit signed little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_INT16 = 3,
|
||||
// The value is a 32-bit unsigned little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_UINT32 = 4,
|
||||
// The value is a 32-bit signed little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_INT32 = 5,
|
||||
// The value is a 32-bit IEEE754 floating point number.
|
||||
GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6,
|
||||
// The value is a boolean.
|
||||
// 1-byte value where 0 is false and 1 is true.
|
||||
// Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.
|
||||
GGUF_METADATA_VALUE_TYPE_BOOL = 7,
|
||||
// The value is a UTF-8 non-null-terminated string, with length prepended.
|
||||
GGUF_METADATA_VALUE_TYPE_STRING = 8,
|
||||
// The value is an array of other values, with the length and type prepended.
|
||||
///
|
||||
// Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.
|
||||
GGUF_METADATA_VALUE_TYPE_ARRAY = 9,
|
||||
// The value is a 64-bit unsigned little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_UINT64 = 10,
|
||||
// The value is a 64-bit signed little-endian integer.
|
||||
GGUF_METADATA_VALUE_TYPE_INT64 = 11,
|
||||
// The value is a 64-bit IEEE754 floating point number.
|
||||
GGUF_METADATA_VALUE_TYPE_FLOAT64 = 12,
|
||||
};
|
||||
|
||||
// A string in GGUF.
|
||||
struct gguf_string_t {
|
||||
// The length of the string, in bytes.
|
||||
uint64_t len;
|
||||
// The string as a UTF-8 non-null-terminated string.
|
||||
char string[len];
|
||||
};
|
||||
|
||||
union gguf_metadata_value_t {
|
||||
uint8_t uint8;
|
||||
int8_t int8;
|
||||
uint16_t uint16;
|
||||
int16_t int16;
|
||||
uint32_t uint32;
|
||||
int32_t int32;
|
||||
float float32;
|
||||
uint64_t uint64;
|
||||
int64_t int64;
|
||||
double float64;
|
||||
bool bool_;
|
||||
gguf_string_t string;
|
||||
struct {
|
||||
// Any value type is valid, including arrays.
|
||||
gguf_metadata_value_type type;
|
||||
// Number of elements, not bytes
|
||||
uint64_t len;
|
||||
// The array of values.
|
||||
gguf_metadata_value_t array[len];
|
||||
} array;
|
||||
};
|
||||
|
||||
struct gguf_metadata_kv_t {
|
||||
// The key of the metadata. It is a standard GGUF string, with the following caveats:
|
||||
// - It must be a valid ASCII string.
|
||||
// - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`.
|
||||
// - It must be at most 2^16-1/65535 bytes long.
|
||||
// Any keys that do not follow these rules are invalid.
|
||||
gguf_string_t key;
|
||||
|
||||
// The type of the value.
|
||||
// Must be one of the `gguf_metadata_value_type` values.
|
||||
gguf_metadata_value_type value_type;
|
||||
// The value.
|
||||
gguf_metadata_value_t value;
|
||||
};
|
||||
|
||||
struct gguf_header_t {
|
||||
// Magic number to announce that this is a GGUF file.
|
||||
// Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`.
|
||||
// Your executor might do little-endian byte order, so it might be
|
||||
// check for 0x46554747 and letting the endianness cancel out.
|
||||
// Consider being *very* explicit about the byte order here.
|
||||
uint32_t magic;
|
||||
// The version of the format implemented.
|
||||
// Must be `3` for version described in this spec, which introduces big-endian support.
|
||||
//
|
||||
// This version should only be increased for structural changes to the format.
|
||||
// Changes that do not affect the structure of the file should instead update the metadata
|
||||
// to signify the change.
|
||||
uint32_t version;
|
||||
// The number of tensors in the file.
|
||||
// This is explicit, instead of being included in the metadata, to ensure it is always present
|
||||
// for loading the tensors.
|
||||
uint64_t tensor_count;
|
||||
// The number of metadata key-value pairs.
|
||||
uint64_t metadata_kv_count;
|
||||
// The metadata key-value pairs.
|
||||
gguf_metadata_kv_t metadata_kv[metadata_kv_count];
|
||||
};
|
||||
|
||||
uint64_t align_offset(uint64_t offset) {
|
||||
return offset + (ALIGNMENT - (offset % ALIGNMENT)) % ALIGNMENT;
|
||||
}
|
||||
|
||||
struct gguf_tensor_info_t {
|
||||
// The name of the tensor. It is a standard GGUF string, with the caveat that
|
||||
// it must be at most 64 bytes long.
|
||||
gguf_string_t name;
|
||||
// The number of dimensions in the tensor.
|
||||
// Currently at most 4, but this may change in the future.
|
||||
uint32_t n_dimensions;
|
||||
// The dimensions of the tensor.
|
||||
uint64_t dimensions[n_dimensions];
|
||||
// The type of the tensor.
|
||||
ggml_type type;
|
||||
// The offset of the tensor's data in this file in bytes.
|
||||
//
|
||||
// This offset is relative to `tensor_data`, not to the start
|
||||
// of the file, to make it easier for writers to write the file.
|
||||
// Readers should consider exposing this offset relative to the
|
||||
// file to make it easier to read the data.
|
||||
//
|
||||
// Must be a multiple of `ALIGNMENT`. That is, `align_offset(offset) == offset`.
|
||||
uint64_t offset;
|
||||
};
|
||||
|
||||
struct gguf_file_t {
|
||||
// The header of the file.
|
||||
gguf_header_t header;
|
||||
|
||||
// Tensor infos, which can be used to locate the tensor data.
|
||||
gguf_tensor_info_t tensor_infos[header.tensor_count];
|
||||
|
||||
// Padding to the nearest multiple of `ALIGNMENT`.
|
||||
//
|
||||
// That is, if `sizeof(header) + sizeof(tensor_infos)` is not a multiple of `ALIGNMENT`,
|
||||
// this padding is added to make it so.
|
||||
//
|
||||
// This can be calculated as `align_offset(position) - position`, where `position` is
|
||||
// the position of the end of `tensor_infos` (i.e. `sizeof(header) + sizeof(tensor_infos)`).
|
||||
uint8_t _padding[];
|
||||
|
||||
// Tensor data.
|
||||
//
|
||||
// This is arbitrary binary data corresponding to the weights of the model. This data should be close
|
||||
// or identical to the data in the original model file, but may be different due to quantization or
|
||||
// other optimizations for inference. Any such deviations should be recorded in the metadata or as
|
||||
// part of the architecture definition.
|
||||
//
|
||||
// Each tensor's data must be stored within this array, and located through its `tensor_infos` entry.
|
||||
// The offset of each tensor's data must be a multiple of `ALIGNMENT`, and the space between tensors
|
||||
// should be padded to `ALIGNMENT` bytes.
|
||||
uint8_t tensor_data[];
|
||||
};
|
||||
```
|
||||
|
||||
## Standardized key-value pairs
|
||||
|
||||
The following key-value pairs are standardized. This list may grow in the future as more use cases are discovered. Where possible, names are shared with the original model definitions to make it easier to map between the two.
|
||||
|
||||
Not all of these are required, but they are all recommended. Keys that are required are bolded. For omitted pairs, the reader should assume that the value is unknown and either default or error as appropriate.
|
||||
|
||||
The community can develop their own key-value pairs to carry additional data. However, these should be namespaced with the relevant community name to avoid collisions. For example, the `rustformers` community might use `rustformers.` as a prefix for all of their keys.
|
||||
|
||||
If a particular community key is widely used, it may be promoted to a standardized key.
|
||||
|
||||
By convention, most counts/lengths/etc are `uint64` unless otherwise specified. This is to allow for larger models to be supported in the future. Some models may use `uint32` for their values; it is recommended that readers support both.
|
||||
|
||||
### General
|
||||
|
||||
#### Required
|
||||
|
||||
- **`general.architecture: string`**: describes what architecture this model implements. All lowercase ASCII, with only `[a-z0-9]+` characters allowed. Known values include:
|
||||
- `llama`
|
||||
- `mpt`
|
||||
- `gptneox`
|
||||
- `gptj`
|
||||
- `gpt2`
|
||||
- `bloom`
|
||||
- `falcon`
|
||||
- `mamba`
|
||||
- `rwkv`
|
||||
- **`general.quantization_version: uint32`**: The version of the quantization format. Not required if the model is not quantized (i.e. no tensors are quantized). If any tensors are quantized, this _must_ be present. This is separate to the quantization scheme of the tensors itself; the quantization version may change without changing the scheme's name (e.g. the quantization scheme is Q5_K, and the quantization version is 4).
|
||||
- **`general.alignment: uint32`**: the global alignment to use, as described above. This can vary to allow for different alignment schemes, but it must be a multiple of 8. Some writers may not write the alignment. If the alignment is **not** specified, assume it is `32`.
|
||||
|
||||
#### General metadata
|
||||
|
||||
- `general.name: string`: The name of the model. This should be a human-readable name that can be used to identify the model. It should be unique within the community that the model is defined in.
|
||||
- `general.author: string`: The author of the model.
|
||||
- `general.version: string`: The version of the model.
|
||||
- `general.organization: string`: The organization of the model.
|
||||
- `general.basename: string`: The base model name / architecture of the model
|
||||
- `general.finetune: string`: What has the base model been optimized toward.
|
||||
- `general.description: string`: free-form description of the model including anything that isn't covered by the other fields
|
||||
- `general.quantized_by: string`: The name of the individual who quantized the model
|
||||
- `general.size_label: string`: Size class of the model, such as number of weights and experts. (Useful for leader boards)
|
||||
- `general.license: string`: License of the model, expressed as a [SPDX license expression](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/) (e.g. `"MIT OR Apache-2.0`). Do not include any other information, such as the license text or the URL to the license.
|
||||
- `general.license.name: string`: Human friendly license name
|
||||
- `general.license.link: string`: URL to the license.
|
||||
- `general.url: string`: URL to the model's homepage. This can be a GitHub repo, a paper, etc.
|
||||
- `general.doi: string`: Digital Object Identifier (DOI) https://www.doi.org/
|
||||
- `general.uuid: string`: [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
|
||||
- `general.repo_url: string`: URL to the model's repository such as a GitHub repo or HuggingFace repo
|
||||
- `general.tags: string[]`: List of tags that can be used as search terms for a search engine or social media
|
||||
- `general.languages: string[]`: What languages can the model speak. Encoded as [ISO 639](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) two letter codes
|
||||
- `general.datasets: string[]`: Links or references to datasets that the model was trained upon
|
||||
- `general.file_type: uint32`: An enumerated value describing the type of the majority of the tensors in the file. Optional; can be inferred from the tensor types.
|
||||
- `ALL_F32 = 0`
|
||||
- `MOSTLY_F16 = 1`
|
||||
- `MOSTLY_Q4_0 = 2`
|
||||
- `MOSTLY_Q4_1 = 3`
|
||||
- `MOSTLY_Q4_1_SOME_F16 = 4`
|
||||
- `MOSTLY_Q4_2 = 5` (support removed)
|
||||
- `MOSTLY_Q4_3 = 6` (support removed)
|
||||
- `MOSTLY_Q8_0 = 7`
|
||||
- `MOSTLY_Q5_0 = 8`
|
||||
- `MOSTLY_Q5_1 = 9`
|
||||
- `MOSTLY_Q2_K = 10`
|
||||
- `MOSTLY_Q3_K_S = 11`
|
||||
- `MOSTLY_Q3_K_M = 12`
|
||||
- `MOSTLY_Q3_K_L = 13`
|
||||
- `MOSTLY_Q4_K_S = 14`
|
||||
- `MOSTLY_Q4_K_M = 15`
|
||||
- `MOSTLY_Q5_K_S = 16`
|
||||
- `MOSTLY_Q5_K_M = 17`
|
||||
- `MOSTLY_Q6_K = 18`
|
||||
|
||||
#### Source metadata
|
||||
|
||||
Information about where this model came from. This is useful for tracking the provenance of the model, and for finding the original source if the model is modified. For a model that was converted from GGML, for example, these keys would point to the model that was converted from.
|
||||
|
||||
- `general.source.url: string`: URL to the source of the model's homepage. This can be a GitHub repo, a paper, etc.
|
||||
- `general.source.doi: string`: Source Digital Object Identifier (DOI) https://www.doi.org/
|
||||
- `general.source.uuid: string`: Source [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
|
||||
- `general.source.repo_url: string`: URL to the source of the model's repository such as a GitHub repo or HuggingFace repo
|
||||
|
||||
- `general.base_model.count: uint32`: Number of parent models
|
||||
- `general.base_model.{id}.name: string`: The name of the parent model.
|
||||
- `general.base_model.{id}.author: string`: The author of the parent model.
|
||||
- `general.base_model.{id}.version: string`: The version of the parent model.
|
||||
- `general.base_model.{id}.organization: string`: The organization of the parent model.
|
||||
- `general.base_model.{id}.url: string`: URL to the source of the parent model's homepage. This can be a GitHub repo, a paper, etc.
|
||||
- `general.base_model.{id}.doi: string`: Parent Digital Object Identifier (DOI) https://www.doi.org/
|
||||
- `general.base_model.{id}.uuid: string`: Parent [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
|
||||
- `general.base_model.{id}.repo_url: string`: URL to the source of the parent model's repository such as a GitHub repo or HuggingFace repo
|
||||
|
||||
### LLM
|
||||
|
||||
In the following, `[llm]` is used to fill in for the name of a specific LLM architecture. For example, `llama` for LLaMA, `mpt` for MPT, etc. If mentioned in an architecture's section, it is required for that architecture, but not all keys are required for all architectures. Consult the relevant section for more information.
|
||||
|
||||
- `[llm].context_length: uint64`: Also known as `n_ctx`. length of the context (in tokens) that the model was trained on. For most architectures, this is the hard limit on the length of the input. Architectures, like RWKV, that are not reliant on transformer-style attention may be able to handle larger inputs, but this is not guaranteed.
|
||||
- `[llm].embedding_length: uint64`: Also known as `n_embd`. Embedding layer size.
|
||||
- `[llm].block_count: uint64`: The number of blocks of attention+feed-forward layers (i.e. the bulk of the LLM). Does not include the input or embedding layers.
|
||||
- `[llm].feed_forward_length: uint64`: Also known as `n_ff`. The length of the feed-forward layer.
|
||||
- `[llm].use_parallel_residual: bool`: Whether or not the parallel residual logic should be used.
|
||||
- `[llm].tensor_data_layout: string`: When a model is converted to GGUF, tensors may be rearranged to improve performance. This key describes the layout of the tensor data. This is not required; if not present, it is assumed to be `reference`.
|
||||
- `reference`: tensors are laid out in the same order as the original model
|
||||
- further options can be found for each architecture in their respective sections
|
||||
- `[llm].expert_count: uint32`: Number of experts in MoE models (optional for non-MoE arches).
|
||||
- `[llm].expert_used_count: uint32`: Number of experts used during each token token evaluation (optional for non-MoE arches).
|
||||
|
||||
#### Attention
|
||||
|
||||
- `[llm].attention.head_count: uint64`: Also known as `n_head`. Number of attention heads.
|
||||
- `[llm].attention.head_count_kv: uint64`: The number of heads per group used in Grouped-Query-Attention. If not present or if present and equal to `[llm].attention.head_count`, the model does not use GQA.
|
||||
- `[llm].attention.max_alibi_bias: float32`: The maximum bias to use for ALiBI.
|
||||
- `[llm].attention.clamp_kqv: float32`: Value (`C`) to clamp the values of the `Q`, `K`, and `V` tensors between (`[-C, C]`).
|
||||
- `[llm].attention.layer_norm_epsilon: float32`: Layer normalization epsilon.
|
||||
- `[llm].attention.layer_norm_rms_epsilon: float32`: Layer RMS normalization epsilon.
|
||||
- `[llm].attention.key_length: uint32`: The optional size of a key head, $d_k$. If not specified, it will be `n_embd / n_head`.
|
||||
- `[llm].attention.value_length: uint32`: The optional size of a value head, $d_v$. If not specified, it will be `n_embd / n_head`.
|
||||
|
||||
#### RoPE
|
||||
|
||||
- `[llm].rope.dimension_count: uint64`: The number of rotary dimensions for RoPE.
|
||||
- `[llm].rope.freq_base: float32`: The base frequency for RoPE.
|
||||
|
||||
##### Scaling
|
||||
|
||||
The following keys describe RoPE scaling parameters:
|
||||
|
||||
- `[llm].rope.scaling.type: string`: Can be `none`, `linear`, or `yarn`.
|
||||
- `[llm].rope.scaling.factor: float32`: A scale factor for RoPE to adjust the context length.
|
||||
- `[llm].rope.scaling.original_context_length: uint32_t`: The original context length of the base model.
|
||||
- `[llm].rope.scaling.finetuned: bool`: True if model has been finetuned with RoPE scaling.
|
||||
|
||||
Note that older models may not have these keys, and may instead use the following key:
|
||||
|
||||
- `[llm].rope.scale_linear: float32`: A linear scale factor for RoPE to adjust the context length.
|
||||
|
||||
It is recommended that models use the newer keys if possible, as they are more flexible and allow for more complex scaling schemes. Executors will need to support both indefinitely.
|
||||
|
||||
#### SSM
|
||||
|
||||
- `[llm].ssm.conv_kernel: uint32`: The size of the rolling/shift state.
|
||||
- `[llm].ssm.inner_size: uint32`: The embedding size of the states.
|
||||
- `[llm].ssm.state_size: uint32`: The size of the recurrent state.
|
||||
- `[llm].ssm.time_step_rank: uint32`: The rank of time steps.
|
||||
|
||||
#### Models
|
||||
|
||||
The following sections describe the metadata for each model architecture. Each key specified _must_ be present.
|
||||
|
||||
##### LLaMA
|
||||
|
||||
- `llama.context_length`
|
||||
- `llama.embedding_length`
|
||||
- `llama.block_count`
|
||||
- `llama.feed_forward_length`
|
||||
- `llama.rope.dimension_count`
|
||||
- `llama.attention.head_count`
|
||||
- `llama.attention.layer_norm_rms_epsilon`
|
||||
|
||||
###### Optional
|
||||
|
||||
- `llama.rope.scale`
|
||||
- `llama.attention.head_count_kv`
|
||||
- `llama.tensor_data_layout`:
|
||||
- `Meta AI original pth`:
|
||||
```python
|
||||
def permute(weights: NDArray, n_head: int) -> NDArray:
|
||||
return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
|
||||
.swapaxes(1, 2)
|
||||
.reshape(weights.shape))
|
||||
```
|
||||
- `llama.expert_count`
|
||||
- `llama.expert_used_count`
|
||||
|
||||
##### MPT
|
||||
|
||||
- `mpt.context_length`
|
||||
- `mpt.embedding_length`
|
||||
- `mpt.block_count`
|
||||
- `mpt.attention.head_count`
|
||||
- `mpt.attention.alibi_bias_max`
|
||||
- `mpt.attention.clip_kqv`
|
||||
- `mpt.attention.layer_norm_epsilon`
|
||||
|
||||
##### GPT-NeoX
|
||||
|
||||
- `gptneox.context_length`
|
||||
- `gptneox.embedding_length`
|
||||
- `gptneox.block_count`
|
||||
- `gptneox.use_parallel_residual`
|
||||
- `gptneox.rope.dimension_count`
|
||||
- `gptneox.attention.head_count`
|
||||
- `gptneox.attention.layer_norm_epsilon`
|
||||
|
||||
###### Optional
|
||||
|
||||
- `gptneox.rope.scale`
|
||||
|
||||
##### GPT-J
|
||||
|
||||
- `gptj.context_length`
|
||||
- `gptj.embedding_length`
|
||||
- `gptj.block_count`
|
||||
- `gptj.rope.dimension_count`
|
||||
- `gptj.attention.head_count`
|
||||
- `gptj.attention.layer_norm_epsilon`
|
||||
|
||||
###### Optional
|
||||
|
||||
- `gptj.rope.scale`
|
||||
|
||||
##### GPT-2
|
||||
|
||||
- `gpt2.context_length`
|
||||
- `gpt2.embedding_length`
|
||||
- `gpt2.block_count`
|
||||
- `gpt2.attention.head_count`
|
||||
- `gpt2.attention.layer_norm_epsilon`
|
||||
|
||||
##### BLOOM
|
||||
|
||||
- `bloom.context_length`
|
||||
- `bloom.embedding_length`
|
||||
- `bloom.block_count`
|
||||
- `bloom.feed_forward_length`
|
||||
- `bloom.attention.head_count`
|
||||
- `bloom.attention.layer_norm_epsilon`
|
||||
|
||||
##### Falcon
|
||||
|
||||
- `falcon.context_length`
|
||||
- `falcon.embedding_length`
|
||||
- `falcon.block_count`
|
||||
- `falcon.attention.head_count`
|
||||
- `falcon.attention.head_count_kv`
|
||||
- `falcon.attention.use_norm`
|
||||
- `falcon.attention.layer_norm_epsilon`
|
||||
|
||||
###### Optional
|
||||
|
||||
- `falcon.tensor_data_layout`:
|
||||
|
||||
- `jploski` (author of the original GGML implementation of Falcon):
|
||||
|
||||
```python
|
||||
# The original query_key_value tensor contains n_head_kv "kv groups",
|
||||
# each consisting of n_head/n_head_kv query weights followed by one key
|
||||
# and one value weight (shared by all query heads in the kv group).
|
||||
# This layout makes it a big pain to work with in GGML.
|
||||
# So we rearrange them here,, so that we have n_head query weights
|
||||
# followed by n_head_kv key weights followed by n_head_kv value weights,
|
||||
# in contiguous fashion.
|
||||
|
||||
if "query_key_value" in src:
|
||||
qkv = model[src].view(
|
||||
n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
|
||||
|
||||
q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
|
||||
k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
|
||||
v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
|
||||
|
||||
model[src] = torch.cat((q,k,v)).reshape_as(model[src])
|
||||
```
|
||||
|
||||
##### Mamba
|
||||
|
||||
- `mamba.context_length`
|
||||
- `mamba.embedding_length`
|
||||
- `mamba.block_count`
|
||||
- `mamba.ssm.conv_kernel`
|
||||
- `mamba.ssm.inner_size`
|
||||
- `mamba.ssm.state_size`
|
||||
- `mamba.ssm.time_step_rank`
|
||||
- `mamba.attention.layer_norm_rms_epsilon`
|
||||
|
||||
##### RWKV
|
||||
|
||||
The vocabulary size is the same as the number of rows in the `head` matrix.
|
||||
|
||||
- `rwkv.architecture_version: uint32`: The only allowed value currently is 4. Version 5 is expected to appear some time in the future.
|
||||
- `rwkv.context_length: uint64`: Length of the context used during training or fine-tuning. RWKV is able to handle larger context than this limit, but the output quality may suffer.
|
||||
- `rwkv.block_count: uint64`
|
||||
- `rwkv.embedding_length: uint64`
|
||||
- `rwkv.feed_forward_length: uint64`
|
||||
|
||||
##### Whisper
|
||||
|
||||
Keys that do not have types defined should be assumed to share definitions with `llm.` keys.
|
||||
(For example, `whisper.context_length` is equivalent to `llm.context_length`.)
|
||||
This is because they are both transformer models.
|
||||
|
||||
- `whisper.encoder.context_length`
|
||||
- `whisper.encoder.embedding_length`
|
||||
- `whisper.encoder.block_count`
|
||||
- `whisper.encoder.mels_count: uint64`
|
||||
- `whisper.encoder.attention.head_count`
|
||||
|
||||
- `whisper.decoder.context_length`
|
||||
- `whisper.decoder.embedding_length`
|
||||
- `whisper.decoder.block_count`
|
||||
- `whisper.decoder.attention.head_count`
|
||||
|
||||
#### Prompting
|
||||
|
||||
**TODO**: Include prompt format, and/or metadata about how it should be used (instruction, conversation, autocomplete, etc).
|
||||
|
||||
### LoRA
|
||||
|
||||
**TODO**: Figure out what metadata is needed for LoRA. Probably desired features:
|
||||
|
||||
- match an existing model exactly, so that it can't be misapplied
|
||||
- be marked as a LoRA so executors won't try to run it by itself
|
||||
|
||||
Should this be an architecture, or should it share the details of the original model with additional fields to mark it as a LoRA?
|
||||
|
||||
### Tokenizer
|
||||
|
||||
The following keys are used to describe the tokenizer of the model. It is recommended that model authors support as many of these as possible, as it will allow for better tokenization quality with supported executors.
|
||||
|
||||
#### GGML
|
||||
|
||||
GGML supports an embedded vocabulary that enables inference of the model, but implementations of tokenization using this vocabulary (i.e. `llama.cpp`'s tokenizer) may have lower accuracy than the original tokenizer used for the model. When a more accurate tokenizer is available and supported, it should be used instead.
|
||||
|
||||
It is not guaranteed to be standardized across models, and may change in the future. It is recommended that model authors use a more standardized tokenizer if possible.
|
||||
|
||||
- `tokenizer.ggml.model: string`: The name of the tokenizer model.
|
||||
- `llama`: Llama style SentencePiece (tokens and scores extracted from HF `tokenizer.model`)
|
||||
- `replit`: Replit style SentencePiece (tokens and scores extracted from HF `spiece.model`)
|
||||
- `gpt2`: GPT-2 / GPT-NeoX style BPE (tokens extracted from HF `tokenizer.json`)
|
||||
- `rwkv`: RWKV tokenizer
|
||||
- `tokenizer.ggml.tokens: array[string]`: A list of tokens indexed by the token ID used by the model.
|
||||
- `tokenizer.ggml.scores: array[float32]`: If present, the score/probability of each token. If not present, all tokens are assumed to have equal probability. If present, it must have the same length and index as `tokens`.
|
||||
- `tokenizer.ggml.token_type: array[int32]`: The token type (1=normal, 2=unknown, 3=control, 4=user defined, 5=unused, 6=byte). If present, it must have the same length and index as `tokens`.
|
||||
- `tokenizer.ggml.merges: array[string]`: If present, the merges of the tokenizer. If not present, the tokens are assumed to be atomic.
|
||||
- `tokenizer.ggml.added_tokens: array[string]`: If present, tokens that were added after training.
|
||||
|
||||
##### Special tokens
|
||||
|
||||
- `tokenizer.ggml.bos_token_id: uint32`: Beginning of sequence marker
|
||||
- `tokenizer.ggml.eos_token_id: uint32`: End of sequence marker
|
||||
- `tokenizer.ggml.unknown_token_id: uint32`: Unknown token
|
||||
- `tokenizer.ggml.separator_token_id: uint32`: Separator token
|
||||
- `tokenizer.ggml.padding_token_id: uint32`: Padding token
|
||||
|
||||
#### Hugging Face
|
||||
|
||||
Hugging Face maintains their own `tokenizers` library that supports a wide variety of tokenizers. If your executor uses this library, it may be able to use the model's tokenizer directly.
|
||||
|
||||
- `tokenizer.huggingface.json: string`: the entirety of the HF `tokenizer.json` for a given model (e.g. <https://huggingface.co/mosaicml/mpt-7b-instruct/blob/main/tokenizer.json>). Included for compatibility with executors that support HF tokenizers directly.
|
||||
|
||||
#### Other
|
||||
|
||||
Other tokenizers may be used, but are not necessarily standardized. They may be executor-specific. They will be documented here as they are discovered/further developed.
|
||||
|
||||
- `tokenizer.rwkv.world: string`: a RWKV World tokenizer, like [this](https://github.com/BlinkDL/ChatRWKV/blob/main/tokenizer/rwkv_vocab_v20230424.txt). This text file should be included verbatim.
|
||||
- `tokenizer.chat_template : string`: a Jinja template that specifies the input format expected by the model. For more details see: <https://huggingface.co/docs/transformers/main/en/chat_templating>
|
||||
|
||||
### Computation graph
|
||||
|
||||
This is a future extension and still needs to be discussed, and may necessitate a new GGUF version. At the time of writing, the primary blocker is the stabilization of the computation graph format.
|
||||
|
||||
A sample computation graph of GGML nodes could be included in the model itself, allowing an executor to run the model without providing its own implementation of the architecture. This would allow for a more consistent experience across executors, and would allow for more complex architectures to be supported without requiring the executor to implement them.
|
||||
|
||||
## Standardized tensor names
|
||||
|
||||
To minimize complexity and maximize compatibility, it is recommended that models using the transformer architecture use the following naming convention for their tensors:
|
||||
|
||||
### Base layers
|
||||
|
||||
`AA.weight` `AA.bias`
|
||||
|
||||
where `AA` can be:
|
||||
|
||||
- `token_embd`: Token embedding layer
|
||||
- `pos_embd`: Position embedding layer
|
||||
- `output_norm`: Output normalization layer
|
||||
- `output`: Output layer
|
||||
|
||||
### Attention and feed-forward layer blocks
|
||||
|
||||
`blk.N.BB.weight` `blk.N.BB.bias`
|
||||
|
||||
where N signifies the block number a layer belongs to, and where `BB` could be:
|
||||
|
||||
- `attn_norm`: Attention normalization layer
|
||||
- `attn_norm_2`: Attention normalization layer
|
||||
- `attn_qkv`: Attention query-key-value layer
|
||||
- `attn_q`: Attention query layer
|
||||
- `attn_k`: Attention key layer
|
||||
- `attn_v`: Attention value layer
|
||||
- `attn_output`: Attention output layer
|
||||
|
||||
- `ffn_norm`: Feed-forward network normalization layer
|
||||
- `ffn_up`: Feed-forward network "up" layer
|
||||
- `ffn_gate`: Feed-forward network "gate" layer
|
||||
- `ffn_down`: Feed-forward network "down" layer
|
||||
- `ffn_gate_inp`: Expert-routing layer for the Feed-forward network in MoE models
|
||||
- `ffn_gate_exp`: Feed-forward network "gate" layer per expert in MoE models
|
||||
- `ffn_down_exp`: Feed-forward network "down" layer per expert in MoE models
|
||||
- `ffn_up_exp`: Feed-forward network "up" layer per expert in MoE models
|
||||
|
||||
- `ssm_in`: State space model input projections layer
|
||||
- `ssm_conv1d`: State space model rolling/shift layer
|
||||
- `ssm_x`: State space model selective parametrization layer
|
||||
- `ssm_a`: State space model state compression layer
|
||||
- `ssm_d`: State space model skip connection layer
|
||||
- `ssm_dt`: State space model time step layer
|
||||
- `ssm_out`: State space model output projection layer
|
||||
|
||||
## Version History
|
||||
|
||||
This document is actively updated to describe the current state of the metadata, and these changes are not tracked outside of the commits.
|
||||
|
||||
However, the format _itself_ has changed. The following sections describe the changes to the format itself.
|
||||
|
||||
### v3
|
||||
|
||||
Adds big-endian support.
|
||||
|
||||
### v2
|
||||
|
||||
Most countable values (lengths, etc) were changed from `uint32` to `uint64` to allow for larger models to be supported in the future.
|
||||
|
||||
### v1
|
||||
|
||||
Initial version.
|
||||
|
||||
## Historical State of Affairs
|
||||
|
||||
The following information is provided for context, but is not necessary to understand the rest of this document.
|
||||
|
||||
### Overview
|
||||
|
||||
At present, there are three GGML file formats floating around for LLMs:
|
||||
|
||||
- **GGML** (unversioned): baseline format, with no versioning or alignment.
|
||||
- **GGMF** (versioned): the same as GGML, but with versioning. Only one version exists.
|
||||
- **GGJT**: Aligns the tensors to allow for use with `mmap`, which requires alignment. v1, v2 and v3 are identical, but the latter versions use a different quantization scheme that is incompatible with previous versions.
|
||||
|
||||
GGML is primarily used by the examples in `ggml`, while GGJT is used by `llama.cpp` models. Other executors may use any of the three formats, but this is not 'officially' supported.
|
||||
|
||||
These formats share the same fundamental structure:
|
||||
|
||||
- a magic number with an optional version number
|
||||
- model-specific hyperparameters, including
|
||||
- metadata about the model, such as the number of layers, the number of heads, etc.
|
||||
- a `ftype` that describes the type of the majority of the tensors,
|
||||
- for GGML files, the quantization version is encoded in the `ftype` divided by 1000
|
||||
- an embedded vocabulary, which is a list of strings with length prepended. The GGMF/GGJT formats embed a float32 score next to the strings.
|
||||
- finally, a list of tensors with their length-prepended name, type, and (aligned, in the case of GGJT) tensor data
|
||||
|
||||
Notably, this structure does not identify what model architecture the model belongs to, nor does it offer any flexibility for changing the structure of the hyperparameters. This means that the only way to add new hyperparameters is to add them to the end of the list, which is a breaking change for existing models.
|
||||
|
||||
### Drawbacks
|
||||
|
||||
Unfortunately, over the last few months, there are a few issues that have become apparent with the existing models:
|
||||
|
||||
- There's no way to identify which model architecture a given model is for, because that information isn't present
|
||||
- Similarly, existing programs cannot intelligently fail upon encountering new architectures
|
||||
- Adding or removing any new hyperparameters is a breaking change, which is impossible for a reader to detect without using heuristics
|
||||
- Each model architecture requires its own conversion script to their architecture's variant of GGML
|
||||
- Maintaining backwards compatibility without breaking the structure of the format requires clever tricks, like packing the quantization version into the ftype, which are not guaranteed to be picked up by readers/writers, and are not consistent between the two formats
|
||||
|
||||
### Why not other formats?
|
||||
|
||||
There are a few other formats that could be used, but issues include:
|
||||
|
||||
- requiring additional dependencies to load or save the model, which is complicated in a C environment
|
||||
- limited or no support for 4-bit quantization
|
||||
- existing cultural expectations (e.g. whether or not the model is a directory or a file)
|
||||
- lack of support for embedded vocabularies
|
||||
- lack of control over direction of future development
|
||||
|
||||
Ultimately, it is likely that GGUF will remain necessary for the foreseeable future, and it is better to have a single format that is well-documented and supported by all executors than to contort an existing format to fit the needs of GGML.
|
||||
@@ -0,0 +1,34 @@
|
||||
if (GGML_ALL_WARNINGS)
|
||||
if (NOT MSVC)
|
||||
set(cxx_flags
|
||||
# TODO(marella): Add other warnings.
|
||||
-Wpedantic
|
||||
-Wunused-variable
|
||||
-Wno-unused-function
|
||||
-Wno-multichar
|
||||
)
|
||||
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${cxx_flags}>")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
add_library(common STATIC common.cpp)
|
||||
target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_library(common-ggml STATIC common-ggml.cpp)
|
||||
target_link_libraries(common-ggml PRIVATE ggml)
|
||||
target_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_subdirectory(yolo)
|
||||
|
||||
if (NOT GGML_BACKEND_DL)
|
||||
add_subdirectory(gpt-2)
|
||||
add_subdirectory(gpt-j)
|
||||
add_subdirectory(mnist)
|
||||
add_subdirectory(sam)
|
||||
add_subdirectory(simple)
|
||||
add_subdirectory(magika)
|
||||
endif()
|
||||
|
||||
if (GGML_METAL)
|
||||
add_subdirectory(perf-metal)
|
||||
endif()
|
||||
@@ -0,0 +1,244 @@
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <regex>
|
||||
#include <map>
|
||||
|
||||
static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {
|
||||
{"q4_0", GGML_FTYPE_MOSTLY_Q4_0},
|
||||
{"q4_1", GGML_FTYPE_MOSTLY_Q4_1},
|
||||
{"q5_0", GGML_FTYPE_MOSTLY_Q5_0},
|
||||
{"q5_1", GGML_FTYPE_MOSTLY_Q5_1},
|
||||
{"q8_0", GGML_FTYPE_MOSTLY_Q8_0},
|
||||
{"q2_k", GGML_FTYPE_MOSTLY_Q2_K},
|
||||
{"q3_k", GGML_FTYPE_MOSTLY_Q3_K},
|
||||
{"q4_k", GGML_FTYPE_MOSTLY_Q4_K},
|
||||
{"q5_k", GGML_FTYPE_MOSTLY_Q5_K},
|
||||
{"q6_k", GGML_FTYPE_MOSTLY_Q6_K},
|
||||
};
|
||||
|
||||
void ggml_print_ftypes(FILE * fp) {
|
||||
for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {
|
||||
fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second);
|
||||
}
|
||||
}
|
||||
|
||||
enum ggml_ftype ggml_parse_ftype(const char * str) {
|
||||
enum ggml_ftype ftype;
|
||||
if (str[0] == 'q') {
|
||||
const auto it = GGML_FTYPE_MAP.find(str);
|
||||
if (it == GGML_FTYPE_MAP.end()) {
|
||||
fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);
|
||||
return GGML_FTYPE_UNKNOWN;
|
||||
}
|
||||
ftype = it->second;
|
||||
} else {
|
||||
ftype = (enum ggml_ftype) atoi(str);
|
||||
}
|
||||
|
||||
return ftype;
|
||||
}
|
||||
|
||||
bool ggml_common_quantize_0(
|
||||
std::ifstream & finp,
|
||||
std::ofstream & fout,
|
||||
const ggml_ftype ftype,
|
||||
const std::vector<std::string> & to_quant,
|
||||
const std::vector<std::string> & to_skip) {
|
||||
|
||||
ggml_type qtype = GGML_TYPE_F32;
|
||||
|
||||
switch (ftype) {
|
||||
case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;
|
||||
case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;
|
||||
case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;
|
||||
case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;
|
||||
case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;
|
||||
case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;
|
||||
case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;
|
||||
case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;
|
||||
case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;
|
||||
case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;
|
||||
case GGML_FTYPE_UNKNOWN:
|
||||
case GGML_FTYPE_ALL_F32:
|
||||
case GGML_FTYPE_MOSTLY_F16:
|
||||
case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:
|
||||
case GGML_FTYPE_MOSTLY_IQ2_XXS:
|
||||
case GGML_FTYPE_MOSTLY_IQ2_XS:
|
||||
case GGML_FTYPE_MOSTLY_IQ2_S:
|
||||
case GGML_FTYPE_MOSTLY_IQ3_XXS:
|
||||
case GGML_FTYPE_MOSTLY_IQ3_S:
|
||||
case GGML_FTYPE_MOSTLY_IQ1_S:
|
||||
case GGML_FTYPE_MOSTLY_IQ4_NL:
|
||||
case GGML_FTYPE_MOSTLY_IQ4_XS:
|
||||
case GGML_FTYPE_MOSTLY_IQ1_M:
|
||||
case GGML_FTYPE_MOSTLY_BF16:
|
||||
case GGML_FTYPE_MOSTLY_MXFP4:
|
||||
case GGML_FTYPE_MOSTLY_NVFP4:
|
||||
case GGML_FTYPE_MOSTLY_Q1_0:
|
||||
{
|
||||
fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
if (!ggml_is_quantized(qtype)) {
|
||||
fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t total_size_org = 0;
|
||||
size_t total_size_new = 0;
|
||||
|
||||
std::vector<float> work;
|
||||
|
||||
std::vector<uint8_t> data_u8;
|
||||
std::vector<ggml_fp16_t> data_f16;
|
||||
std::vector<float> data_f32;
|
||||
|
||||
while (true) {
|
||||
int32_t n_dims;
|
||||
int32_t length;
|
||||
int32_t ttype;
|
||||
|
||||
finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
finp.read(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
finp.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
|
||||
if (finp.eof()) {
|
||||
break;
|
||||
}
|
||||
|
||||
int32_t nelements = 1;
|
||||
int32_t ne[4] = { 1, 1, 1, 1 };
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
nelements *= ne[i];
|
||||
}
|
||||
|
||||
std::string name(length, 0);
|
||||
finp.read (&name[0], length);
|
||||
|
||||
printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));
|
||||
|
||||
bool quantize = false;
|
||||
|
||||
// check if we should quantize this tensor
|
||||
for (const auto & s : to_quant) {
|
||||
if (std::regex_match(name, std::regex(s))) {
|
||||
quantize = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// check if we should skip this tensor
|
||||
for (const auto & s : to_skip) {
|
||||
if (std::regex_match(name, std::regex(s))) {
|
||||
quantize = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// quantize only 2D tensors
|
||||
quantize &= (n_dims == 2);
|
||||
|
||||
if (quantize) {
|
||||
if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {
|
||||
fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ttype == GGML_TYPE_F16) {
|
||||
data_f16.resize(nelements);
|
||||
finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));
|
||||
data_f32.resize(nelements);
|
||||
for (int i = 0; i < nelements; ++i) {
|
||||
data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);
|
||||
}
|
||||
} else {
|
||||
data_f32.resize(nelements);
|
||||
finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));
|
||||
}
|
||||
|
||||
ttype = qtype;
|
||||
} else {
|
||||
const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);
|
||||
|
||||
data_u8.resize(nelements*bpe);
|
||||
finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);
|
||||
}
|
||||
|
||||
fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
fout.write(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
fout.write(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
}
|
||||
fout.write(&name[0], length);
|
||||
|
||||
if (quantize) {
|
||||
work.resize(nelements); // for quantization
|
||||
|
||||
size_t cur_size = 0;
|
||||
switch ((ggml_type) ttype) {
|
||||
case GGML_TYPE_Q4_0:
|
||||
case GGML_TYPE_Q4_1:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_Q2_K:
|
||||
case GGML_TYPE_Q3_K:
|
||||
case GGML_TYPE_Q4_K:
|
||||
case GGML_TYPE_Q5_K:
|
||||
case GGML_TYPE_Q6_K:
|
||||
{
|
||||
cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_I8:
|
||||
case GGML_TYPE_I16:
|
||||
case GGML_TYPE_I32:
|
||||
case GGML_TYPE_I64:
|
||||
case GGML_TYPE_F64:
|
||||
case GGML_TYPE_Q8_1:
|
||||
case GGML_TYPE_Q8_K:
|
||||
case GGML_TYPE_IQ2_XXS:
|
||||
case GGML_TYPE_IQ2_XS:
|
||||
case GGML_TYPE_IQ2_S:
|
||||
case GGML_TYPE_IQ3_XXS:
|
||||
case GGML_TYPE_IQ3_S:
|
||||
case GGML_TYPE_IQ1_S:
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
case GGML_TYPE_IQ4_XS:
|
||||
case GGML_TYPE_IQ1_M:
|
||||
case GGML_TYPE_BF16:
|
||||
case GGML_TYPE_TQ1_0:
|
||||
case GGML_TYPE_TQ2_0:
|
||||
case GGML_TYPE_MXFP4:
|
||||
case GGML_TYPE_NVFP4:
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_COUNT:
|
||||
{
|
||||
fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
fout.write(reinterpret_cast<char *>(work.data()), cur_size);
|
||||
total_size_new += cur_size;
|
||||
|
||||
printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);
|
||||
} else {
|
||||
printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);
|
||||
fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());
|
||||
total_size_new += data_u8.size();
|
||||
}
|
||||
|
||||
total_size_org += nelements * sizeof(float);
|
||||
}
|
||||
|
||||
printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
|
||||
printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "ggml.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
enum ggml_ftype ggml_parse_ftype(const char * str);
|
||||
|
||||
void ggml_print_ftypes(FILE * fp = stderr);
|
||||
|
||||
bool ggml_common_quantize_0(
|
||||
std::ifstream & finp,
|
||||
std::ofstream & fout,
|
||||
const ggml_ftype ftype,
|
||||
const std::vector<std::string> & to_quant,
|
||||
const std::vector<std::string> & to_skip);
|
||||
@@ -0,0 +1,675 @@
|
||||
#define _USE_MATH_DEFINES // for M_PI
|
||||
|
||||
#include "common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <codecvt>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <locale>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
|
||||
// Function to check if the next argument exists
|
||||
static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
|
||||
if (i + 1 < argc && argv[i + 1][0] != '-') {
|
||||
return argv[++i];
|
||||
} else {
|
||||
fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
|
||||
gpt_print_usage(argc, argv, params);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
||||
for (int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
|
||||
if (arg == "-s" || arg == "--seed") {
|
||||
params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-t" || arg == "--threads") {
|
||||
params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-p" || arg == "--prompt") {
|
||||
params.prompt = get_next_arg(i, argc, argv, arg, params);
|
||||
} else if (arg == "-n" || arg == "--n_predict") {
|
||||
params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-np" || arg == "--n_parallel") {
|
||||
params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--top_k") {
|
||||
params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--top_p") {
|
||||
params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--temp") {
|
||||
params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--repeat-last-n") {
|
||||
params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--repeat-penalty") {
|
||||
params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-b" || arg == "--batch_size") {
|
||||
params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-c" || arg == "--context") {
|
||||
params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
|
||||
params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "--ignore-eos") {
|
||||
params.ignore_eos = true;
|
||||
} else if (arg == "-m" || arg == "--model") {
|
||||
params.model = get_next_arg(i, argc, argv, arg, params);
|
||||
} else if (arg == "-i" || arg == "--interactive") {
|
||||
params.interactive = true;
|
||||
} else if (arg == "-ip" || arg == "--interactive-port") {
|
||||
params.interactive = true;
|
||||
params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
|
||||
} else if (arg == "-h" || arg == "--help") {
|
||||
gpt_print_usage(argc, argv, params);
|
||||
exit(0);
|
||||
} else if (arg == "-f" || arg == "--file") {
|
||||
get_next_arg(i, argc, argv, arg, params);
|
||||
std::ifstream file(argv[i]);
|
||||
if (!file) {
|
||||
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
|
||||
break;
|
||||
}
|
||||
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
|
||||
if (params.prompt.back() == '\n') {
|
||||
params.prompt.pop_back();
|
||||
}
|
||||
} else if (arg == "-tt" || arg == "--token_test") {
|
||||
params.token_test = get_next_arg(i, argc, argv, arg, params);
|
||||
}
|
||||
else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
gpt_print_usage(argc, argv, params);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
|
||||
fprintf(stderr, "usage: %s [options]\n", argv[0]);
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "options:\n");
|
||||
fprintf(stderr, " -h, --help show this help message and exit\n");
|
||||
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
|
||||
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
|
||||
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
|
||||
fprintf(stderr, " prompt to start generation with (default: random)\n");
|
||||
fprintf(stderr, " -f FNAME, --file FNAME\n");
|
||||
fprintf(stderr, " load prompt from a file\n");
|
||||
fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n");
|
||||
fprintf(stderr, " test tokenization\n");
|
||||
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict);
|
||||
fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k);
|
||||
fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p);
|
||||
fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp);
|
||||
fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n);
|
||||
fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty);
|
||||
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
|
||||
fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx);
|
||||
fprintf(stderr, " --ignore-eos ignore EOS token during generation\n");
|
||||
fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
|
||||
fprintf(stderr, " -m FNAME, --model FNAME\n");
|
||||
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
std::string gpt_random_prompt(std::mt19937 & rng) {
|
||||
const int r = rng() % 10;
|
||||
switch (r) {
|
||||
case 0: return "So";
|
||||
case 1: return "Once upon a time";
|
||||
case 2: return "When";
|
||||
case 3: return "The";
|
||||
case 4: return "After";
|
||||
case 5: return "If";
|
||||
case 6: return "import";
|
||||
case 7: return "He";
|
||||
case 8: return "She";
|
||||
case 9: return "They";
|
||||
}
|
||||
|
||||
return "The";
|
||||
}
|
||||
|
||||
std::string trim(const std::string & s) {
|
||||
std::regex e("^\\s+|\\s+$");
|
||||
return std::regex_replace(s, e, "");
|
||||
}
|
||||
|
||||
std::string replace(const std::string & s, const std::string & from, const std::string & to) {
|
||||
std::string result = s;
|
||||
size_t pos = 0;
|
||||
while ((pos = result.find(from, pos)) != std::string::npos) {
|
||||
result.replace(pos, from.length(), to);
|
||||
pos += to.length();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void gpt_vocab::add_special_token(const std::string & token) {
|
||||
special_tokens.push_back(token);
|
||||
}
|
||||
|
||||
std::map<std::string, int32_t> json_parse(const std::string & fname) {
|
||||
std::map<std::string, int32_t> result;
|
||||
|
||||
// read file into string
|
||||
std::string json;
|
||||
{
|
||||
std::ifstream ifs(fname);
|
||||
if (!ifs) {
|
||||
fprintf(stderr, "Failed to open %s\n", fname.c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
json = std::string((std::istreambuf_iterator<char>(ifs)),
|
||||
(std::istreambuf_iterator<char>()));
|
||||
}
|
||||
|
||||
if (json[0] != '{') {
|
||||
return result;
|
||||
}
|
||||
|
||||
// parse json
|
||||
{
|
||||
bool has_key = false;
|
||||
bool in_token = false;
|
||||
|
||||
std::string str_key = "";
|
||||
std::string str_val = "";
|
||||
|
||||
int n = json.size();
|
||||
for (int i = 1; i < n; ++i) {
|
||||
if (!in_token) {
|
||||
if (json[i] == ' ') continue;
|
||||
if (json[i] == '"') {
|
||||
in_token = true;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
if (json[i] == '\\' && i+1 < n) {
|
||||
if (has_key == false) {
|
||||
str_key += json[i];
|
||||
} else {
|
||||
str_val += json[i];
|
||||
}
|
||||
++i;
|
||||
} else if (json[i] == '"') {
|
||||
if (has_key == false) {
|
||||
has_key = true;
|
||||
++i;
|
||||
while (json[i] == ' ') ++i;
|
||||
++i; // :
|
||||
while (json[i] == ' ') ++i;
|
||||
if (json[i] != '\"') {
|
||||
while (json[i] != ',' && json[i] != '}') {
|
||||
str_val += json[i++];
|
||||
}
|
||||
has_key = false;
|
||||
} else {
|
||||
in_token = true;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
has_key = false;
|
||||
}
|
||||
|
||||
str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
|
||||
str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
|
||||
str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> "
|
||||
|
||||
try {
|
||||
result[str_key] = std::stoi(str_val);
|
||||
} catch (...) {
|
||||
//fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
|
||||
|
||||
}
|
||||
str_key = "";
|
||||
str_val = "";
|
||||
in_token = false;
|
||||
continue;
|
||||
}
|
||||
if (has_key == false) {
|
||||
str_key += json[i];
|
||||
} else {
|
||||
str_val += json[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void gpt_split_words(std::string str, std::vector<std::string>& words) {
|
||||
const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
|
||||
const std::regex re(pattern);
|
||||
std::smatch m;
|
||||
|
||||
while (std::regex_search(str, m, re)) {
|
||||
for (auto x : m) {
|
||||
words.push_back(x);
|
||||
}
|
||||
str = m.suffix();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
|
||||
std::vector<std::string> words;
|
||||
|
||||
// first split the text into words
|
||||
{
|
||||
std::string str = text;
|
||||
|
||||
// Generate the subpattern from the special_tokens vector if it's not empty
|
||||
if (!vocab.special_tokens.empty()) {
|
||||
const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])");
|
||||
std::string special_tokens_subpattern;
|
||||
for (const auto & token : vocab.special_tokens) {
|
||||
if (!special_tokens_subpattern.empty()) {
|
||||
special_tokens_subpattern += "|";
|
||||
}
|
||||
special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)");
|
||||
}
|
||||
|
||||
std::regex re(special_tokens_subpattern);
|
||||
std::smatch m;
|
||||
// Split the text by special tokens.
|
||||
while (std::regex_search(str, m, re)) {
|
||||
// Split the substrings in-between special tokens into words.
|
||||
gpt_split_words(m.prefix(), words);
|
||||
// Add matched special tokens as words.
|
||||
for (auto x : m) {
|
||||
words.push_back(x);
|
||||
}
|
||||
str = m.suffix();
|
||||
}
|
||||
// Remaining text without special tokens will be handled below.
|
||||
}
|
||||
|
||||
gpt_split_words(str, words);
|
||||
}
|
||||
|
||||
// find the longest token that forms each word in words:
|
||||
std::vector<gpt_vocab::id> tokens;
|
||||
for (const auto & word : words) {
|
||||
for (int i = 0; i < (int) word.size(); ){
|
||||
for (int j = word.size() - 1; j >= i; j--){
|
||||
auto cand = word.substr(i, j-i+1);
|
||||
auto it = vocab.token_to_id.find(cand);
|
||||
if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
|
||||
tokens.push_back(it->second);
|
||||
i = j + 1;
|
||||
break;
|
||||
}
|
||||
else if (j == i){ // word.substr(i, 1) has no matching
|
||||
fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
static std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {
|
||||
std::vector<gpt_vocab::id> output;
|
||||
std::stringstream ss(input);
|
||||
std::string token;
|
||||
|
||||
while (std::getline(ss, token, delimiter)) {
|
||||
output.push_back(std::stoi(token));
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){
|
||||
if (fpath_test.empty()){
|
||||
fprintf(stderr, "%s : No test file found.\n", __func__);
|
||||
return std::map<std::string, std::vector<gpt_vocab::id>>();
|
||||
}
|
||||
|
||||
std::map<std::string, std::vector<gpt_vocab::id>> tests;
|
||||
|
||||
auto fin = std::ifstream(fpath_test, std::ios_base::in);
|
||||
const char * delimeter = " => ";
|
||||
const char del_tok = ',';
|
||||
std::string line;
|
||||
while (std::getline(fin, line)) {
|
||||
size_t delimiterPos = line.find(delimeter);
|
||||
if (delimiterPos != std::string::npos) {
|
||||
std::string text = line.substr(0, delimiterPos);
|
||||
std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));
|
||||
tests[text] = parse_tokens_from_string(s_tokens, del_tok);
|
||||
}
|
||||
}
|
||||
return tests;
|
||||
}
|
||||
|
||||
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){
|
||||
std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);
|
||||
|
||||
size_t n_fails = 0;
|
||||
|
||||
for (const auto & test : tests) {
|
||||
std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);
|
||||
|
||||
if (tokens != test.second){
|
||||
n_fails++;
|
||||
|
||||
// print out failure cases
|
||||
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str());
|
||||
fprintf(stderr, "%s : tokens in hf: ", __func__);
|
||||
for (const auto & t : test.second) {
|
||||
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "%s : tokens in ggml: ", __func__);
|
||||
for (const auto & t : tokens) {
|
||||
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
|
||||
}
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size());
|
||||
}
|
||||
|
||||
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
|
||||
printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
vocab.token_to_id = ::json_parse(fname);
|
||||
|
||||
for (const auto & kv : vocab.token_to_id) {
|
||||
vocab.id_to_token[kv.second] = kv.first;
|
||||
}
|
||||
|
||||
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
|
||||
|
||||
// print the vocabulary
|
||||
//for (auto kv : vocab.token_to_id) {
|
||||
// printf("'%s' -> %d\n", kv.first.data(), kv.second);
|
||||
//}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
gpt_vocab::id gpt_sample_top_k_top_p(
|
||||
const gpt_vocab & vocab,
|
||||
const float * logits,
|
||||
int top_k,
|
||||
double top_p,
|
||||
double temp,
|
||||
std::mt19937 & rng) {
|
||||
int n_logits = vocab.id_to_token.size();
|
||||
|
||||
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
|
||||
logits_id.reserve(n_logits);
|
||||
|
||||
{
|
||||
const double scale = 1.0/temp;
|
||||
for (int i = 0; i < n_logits; ++i) {
|
||||
logits_id.push_back(std::make_pair(logits[i]*scale, i));
|
||||
}
|
||||
}
|
||||
|
||||
// find the top K tokens
|
||||
std::partial_sort(
|
||||
logits_id.begin(),
|
||||
logits_id.begin() + top_k, logits_id.end(),
|
||||
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
|
||||
return a.first > b.first;
|
||||
});
|
||||
|
||||
logits_id.resize(top_k);
|
||||
|
||||
double maxl = -INFINITY;
|
||||
for (const auto & kv : logits_id) {
|
||||
maxl = std::max(maxl, kv.first);
|
||||
}
|
||||
|
||||
// compute probs for the top K tokens
|
||||
std::vector<double> probs;
|
||||
probs.reserve(logits_id.size());
|
||||
|
||||
double sum = 0.0;
|
||||
for (const auto & kv : logits_id) {
|
||||
double p = exp(kv.first - maxl);
|
||||
probs.push_back(p);
|
||||
sum += p;
|
||||
}
|
||||
|
||||
// normalize the probs
|
||||
for (auto & p : probs) {
|
||||
p /= sum;
|
||||
}
|
||||
|
||||
if (top_p < 1.0f) {
|
||||
double cumsum = 0.0f;
|
||||
for (int i = 0; i < top_k; i++) {
|
||||
cumsum += probs[i];
|
||||
if (cumsum >= top_p) {
|
||||
top_k = i + 1;
|
||||
probs.resize(top_k);
|
||||
logits_id.resize(top_k);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cumsum = 1.0/cumsum;
|
||||
for (int i = 0; i < (int) probs.size(); i++) {
|
||||
probs[i] *= cumsum;
|
||||
}
|
||||
}
|
||||
|
||||
//printf("\n");
|
||||
//for (int i = 0; i < (int) probs.size(); i++) {
|
||||
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
|
||||
//}
|
||||
//exit(0);
|
||||
|
||||
std::discrete_distribution<> dist(probs.begin(), probs.end());
|
||||
int idx = dist(rng);
|
||||
|
||||
return logits_id[idx].second;
|
||||
}
|
||||
|
||||
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
|
||||
const gpt_vocab & vocab,
|
||||
const float * logits,
|
||||
const int32_t * last_n_tokens_data,
|
||||
size_t last_n_tokens_data_size,
|
||||
int top_k,
|
||||
double top_p,
|
||||
double temp,
|
||||
int repeat_last_n,
|
||||
float repeat_penalty,
|
||||
std::mt19937 & rng) {
|
||||
|
||||
int n_logits = vocab.id_to_token.size();
|
||||
|
||||
const auto * plogits = logits;
|
||||
|
||||
const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);
|
||||
|
||||
if (temp <= 0) {
|
||||
// select the token with the highest logit directly
|
||||
float max_logit = plogits[0];
|
||||
gpt_vocab::id max_id = 0;
|
||||
|
||||
for (int i = 1; i < n_logits; ++i) {
|
||||
if (plogits[i] > max_logit) {
|
||||
max_logit = plogits[i];
|
||||
max_id = i;
|
||||
}
|
||||
}
|
||||
return max_id;
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
|
||||
logits_id.reserve(n_logits);
|
||||
|
||||
{
|
||||
const float scale = 1.0f/temp;
|
||||
for (int i = 0; i < n_logits; ++i) {
|
||||
// repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
|
||||
// credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
|
||||
if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {
|
||||
// if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
|
||||
if (plogits[i] < 0.0f) {
|
||||
logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
|
||||
} else {
|
||||
logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
|
||||
}
|
||||
} else {
|
||||
logits_id.push_back(std::make_pair(plogits[i]*scale, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// find the top K tokens
|
||||
std::partial_sort(
|
||||
logits_id.begin(),
|
||||
logits_id.begin() + top_k, logits_id.end(),
|
||||
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
|
||||
return a.first > b.first;
|
||||
});
|
||||
|
||||
logits_id.resize(top_k);
|
||||
|
||||
double maxl = -INFINITY;
|
||||
for (const auto & kv : logits_id) {
|
||||
maxl = std::max(maxl, kv.first);
|
||||
}
|
||||
|
||||
// compute probs for the top K tokens
|
||||
std::vector<double> probs;
|
||||
probs.reserve(logits_id.size());
|
||||
|
||||
double sum = 0.0;
|
||||
for (const auto & kv : logits_id) {
|
||||
double p = exp(kv.first - maxl);
|
||||
probs.push_back(p);
|
||||
sum += p;
|
||||
}
|
||||
|
||||
// normalize the probs
|
||||
for (auto & p : probs) {
|
||||
p /= sum;
|
||||
}
|
||||
|
||||
if (top_p < 1.0f) {
|
||||
double cumsum = 0.0f;
|
||||
for (int i = 0; i < top_k; i++) {
|
||||
cumsum += probs[i];
|
||||
if (cumsum >= top_p) {
|
||||
top_k = i + 1;
|
||||
probs.resize(top_k);
|
||||
logits_id.resize(top_k);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
cumsum = 1.0/cumsum;
|
||||
for (int i = 0; i < (int) probs.size(); i++) {
|
||||
probs[i] *= cumsum;
|
||||
}
|
||||
}
|
||||
|
||||
// printf("\n");
|
||||
// for (int i = 0; i < (int) probs.size(); i++) {
|
||||
// for (int i = 0; i < 10; i++) {
|
||||
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
|
||||
// }
|
||||
|
||||
std::discrete_distribution<> dist(probs.begin(), probs.end());
|
||||
int idx = dist(rng);
|
||||
|
||||
return logits_id[idx].second;
|
||||
|
||||
}
|
||||
|
||||
void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
|
||||
const float rc = 1.0f / (2.0f * M_PI * cutoff);
|
||||
const float dt = 1.0f / sample_rate;
|
||||
const float alpha = dt / (rc + dt);
|
||||
|
||||
float y = data[0];
|
||||
|
||||
for (size_t i = 1; i < data.size(); i++) {
|
||||
y = alpha * (y + data[i] - data[i - 1]);
|
||||
data[i] = y;
|
||||
}
|
||||
}
|
||||
|
||||
bool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {
|
||||
const int n_samples = pcmf32.size();
|
||||
const int n_samples_last = (sample_rate * last_ms) / 1000;
|
||||
|
||||
if (n_samples_last >= n_samples) {
|
||||
// not enough samples - assume no speech
|
||||
return false;
|
||||
}
|
||||
|
||||
if (freq_thold > 0.0f) {
|
||||
high_pass_filter(pcmf32, freq_thold, sample_rate);
|
||||
}
|
||||
|
||||
float energy_all = 0.0f;
|
||||
float energy_last = 0.0f;
|
||||
|
||||
for (int i = 0; i < n_samples; i++) {
|
||||
energy_all += fabsf(pcmf32[i]);
|
||||
|
||||
if (i >= n_samples - n_samples_last) {
|
||||
energy_last += fabsf(pcmf32[i]);
|
||||
}
|
||||
}
|
||||
|
||||
energy_all /= n_samples;
|
||||
energy_last /= n_samples_last;
|
||||
|
||||
if (verbose) {
|
||||
fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold);
|
||||
}
|
||||
|
||||
if (energy_last > vad_thold*energy_all) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
float similarity(const std::string & s0, const std::string & s1) {
|
||||
const size_t len0 = s0.size() + 1;
|
||||
const size_t len1 = s1.size() + 1;
|
||||
|
||||
std::vector<int> col(len1, 0);
|
||||
std::vector<int> prevCol(len1, 0);
|
||||
|
||||
for (size_t i = 0; i < len1; i++) {
|
||||
prevCol[i] = i;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < len0; i++) {
|
||||
col[0] = i;
|
||||
for (size_t j = 1; j < len1; j++) {
|
||||
col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));
|
||||
}
|
||||
col.swap(prevCol);
|
||||
}
|
||||
|
||||
const float dist = prevCol[len1 - 1];
|
||||
|
||||
return 1.0f - (dist / std::max(s0.size(), s1.size()));
|
||||
}
|
||||
|
||||
bool is_file_exist(const char * filename) {
|
||||
std::ifstream infile(filename);
|
||||
return infile.good();
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
// Various helper functions and utilities
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <random>
|
||||
#include <thread>
|
||||
#include <ctime>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
//
|
||||
// GPT CLI argument parsing
|
||||
//
|
||||
|
||||
struct gpt_params {
|
||||
int32_t seed = -1; // RNG seed
|
||||
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
|
||||
int32_t n_predict = 200; // new tokens to predict
|
||||
int32_t n_parallel = 1; // number of parallel streams
|
||||
int32_t n_batch = 32; // batch size for prompt processing
|
||||
int32_t n_ctx = 2048; // context size (this is the KV cache max size)
|
||||
int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU
|
||||
|
||||
bool ignore_eos = false; // ignore EOS token when generating text
|
||||
|
||||
// sampling parameters
|
||||
int32_t top_k = 40;
|
||||
float top_p = 0.9f;
|
||||
float temp = 0.9f;
|
||||
int32_t repeat_last_n = 64;
|
||||
float repeat_penalty = 1.00f;
|
||||
|
||||
std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
|
||||
std::string prompt = "";
|
||||
std::string token_test = "";
|
||||
|
||||
bool interactive = false;
|
||||
int32_t interactive_port = -1;
|
||||
};
|
||||
|
||||
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
|
||||
|
||||
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
|
||||
|
||||
std::string gpt_random_prompt(std::mt19937 & rng);
|
||||
|
||||
//
|
||||
// Vocab utils
|
||||
//
|
||||
|
||||
std::string trim(const std::string & s);
|
||||
|
||||
std::string replace(
|
||||
const std::string & s,
|
||||
const std::string & from,
|
||||
const std::string & to);
|
||||
|
||||
struct gpt_vocab {
|
||||
using id = int32_t;
|
||||
using token = std::string;
|
||||
|
||||
std::map<token, id> token_to_id;
|
||||
std::map<id, token> id_to_token;
|
||||
std::vector<std::string> special_tokens;
|
||||
|
||||
void add_special_token(const std::string & token);
|
||||
};
|
||||
|
||||
// poor-man's JSON parsing
|
||||
std::map<std::string, int32_t> json_parse(const std::string & fname);
|
||||
|
||||
std::string convert_to_utf8(const std::wstring & input);
|
||||
|
||||
std::wstring convert_to_wstring(const std::string & input);
|
||||
|
||||
void gpt_split_words(std::string str, std::vector<std::string>& words);
|
||||
|
||||
// split text into tokens
|
||||
//
|
||||
// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
|
||||
//
|
||||
// Regex (Python):
|
||||
// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
|
||||
//
|
||||
// Regex (C++):
|
||||
// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
|
||||
//
|
||||
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
|
||||
|
||||
// test outputs of gpt_tokenize
|
||||
//
|
||||
// - compare with tokens generated by the huggingface tokenizer
|
||||
// - test cases are chosen based on the model's main language (under 'prompt' directory)
|
||||
// - if all sentences are tokenized identically, print 'All tests passed.'
|
||||
// - otherwise, print sentence, huggingface tokens, ggml tokens
|
||||
//
|
||||
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);
|
||||
|
||||
// load the tokens from encoder.json
|
||||
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
|
||||
|
||||
// sample next token given probabilities for each embedding
|
||||
//
|
||||
// - consider only the top K tokens
|
||||
// - from them, consider only the top tokens with cumulative probability > P
|
||||
//
|
||||
// TODO: not sure if this implementation is correct
|
||||
// TODO: temperature is not implemented
|
||||
//
|
||||
gpt_vocab::id gpt_sample_top_k_top_p(
|
||||
const gpt_vocab & vocab,
|
||||
const float * logits,
|
||||
int top_k,
|
||||
double top_p,
|
||||
double temp,
|
||||
std::mt19937 & rng);
|
||||
|
||||
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
|
||||
const gpt_vocab & vocab,
|
||||
const float * logits,
|
||||
const int32_t * last_n_tokens_data,
|
||||
size_t last_n_tokens_data_size,
|
||||
int top_k,
|
||||
double top_p,
|
||||
double temp,
|
||||
int repeat_last_n,
|
||||
float repeat_penalty,
|
||||
std::mt19937 & rng);
|
||||
|
||||
//
|
||||
// Audio utils
|
||||
//
|
||||
|
||||
// Write PCM data into WAV audio file
|
||||
class wav_writer {
|
||||
private:
|
||||
std::ofstream file;
|
||||
uint32_t dataSize = 0;
|
||||
std::string wav_filename;
|
||||
|
||||
bool write_header(const uint32_t sample_rate,
|
||||
const uint16_t bits_per_sample,
|
||||
const uint16_t channels) {
|
||||
|
||||
file.write("RIFF", 4);
|
||||
file.write("\0\0\0\0", 4); // Placeholder for file size
|
||||
file.write("WAVE", 4);
|
||||
file.write("fmt ", 4);
|
||||
|
||||
const uint32_t sub_chunk_size = 16;
|
||||
const uint16_t audio_format = 1; // PCM format
|
||||
const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;
|
||||
const uint16_t block_align = channels * bits_per_sample / 8;
|
||||
|
||||
file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);
|
||||
file.write(reinterpret_cast<const char *>(&audio_format), 2);
|
||||
file.write(reinterpret_cast<const char *>(&channels), 2);
|
||||
file.write(reinterpret_cast<const char *>(&sample_rate), 4);
|
||||
file.write(reinterpret_cast<const char *>(&byte_rate), 4);
|
||||
file.write(reinterpret_cast<const char *>(&block_align), 2);
|
||||
file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);
|
||||
file.write("data", 4);
|
||||
file.write("\0\0\0\0", 4); // Placeholder for data size
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// It is assumed that PCM data is normalized to a range from -1 to 1
|
||||
bool write_audio(const float * data, size_t length) {
|
||||
for (size_t i = 0; i < length; ++i) {
|
||||
const int16_t intSample = int16_t(data[i] * 32767);
|
||||
file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));
|
||||
dataSize += sizeof(int16_t);
|
||||
}
|
||||
if (file.is_open()) {
|
||||
file.seekp(4, std::ios::beg);
|
||||
uint32_t fileSize = 36 + dataSize;
|
||||
file.write(reinterpret_cast<char *>(&fileSize), 4);
|
||||
file.seekp(40, std::ios::beg);
|
||||
file.write(reinterpret_cast<char *>(&dataSize), 4);
|
||||
file.seekp(0, std::ios::end);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool open_wav(const std::string & filename) {
|
||||
if (filename != wav_filename) {
|
||||
if (file.is_open()) {
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
if (!file.is_open()) {
|
||||
file.open(filename, std::ios::binary);
|
||||
wav_filename = filename;
|
||||
dataSize = 0;
|
||||
}
|
||||
return file.is_open();
|
||||
}
|
||||
|
||||
public:
|
||||
bool open(const std::string & filename,
|
||||
const uint32_t sample_rate,
|
||||
const uint16_t bits_per_sample,
|
||||
const uint16_t channels) {
|
||||
|
||||
if (open_wav(filename)) {
|
||||
write_header(sample_rate, bits_per_sample, channels);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool close() {
|
||||
file.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool write(const float * data, size_t length) {
|
||||
return write_audio(data, length);
|
||||
}
|
||||
|
||||
~wav_writer() {
|
||||
if (file.is_open()) {
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Apply a high-pass frequency filter to PCM audio
|
||||
// Suppresses frequencies below cutoff Hz
|
||||
void high_pass_filter(
|
||||
std::vector<float> & data,
|
||||
float cutoff,
|
||||
float sample_rate);
|
||||
|
||||
// Basic voice activity detection (VAD) using audio energy adaptive threshold
|
||||
bool vad_simple(
|
||||
std::vector<float> & pcmf32,
|
||||
int sample_rate,
|
||||
int last_ms,
|
||||
float vad_thold,
|
||||
float freq_thold,
|
||||
bool verbose);
|
||||
|
||||
// compute similarity between two strings using Levenshtein distance
|
||||
float similarity(const std::string & s0, const std::string & s1);
|
||||
|
||||
//
|
||||
// Terminal utils
|
||||
//
|
||||
|
||||
#define SQR(X) ((X) * (X))
|
||||
#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
|
||||
|
||||
/**
|
||||
* Quantizes 24-bit RGB to xterm256 code range [16,256).
|
||||
*/
|
||||
static int rgb2xterm256(int r, int g, int b) {
|
||||
unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
|
||||
int av, ir, ig, ib, il, qr, qg, qb, ql;
|
||||
av = r * .299 + g * .587 + b * .114 + .5;
|
||||
ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
|
||||
qr = cube[(ir = UNCUBE(r))];
|
||||
qg = cube[(ig = UNCUBE(g))];
|
||||
qb = cube[(ib = UNCUBE(b))];
|
||||
if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
|
||||
SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
|
||||
return ir * 36 + ig * 6 + ib + 020;
|
||||
return il + 0350;
|
||||
}
|
||||
|
||||
static std::string set_xterm256_foreground(int r, int g, int b) {
|
||||
int x = rgb2xterm256(r, g, b);
|
||||
std::ostringstream oss;
|
||||
oss << "\033[38;5;" << x << "m";
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
// Lowest is red, middle is yellow, highest is green. Color scheme from
|
||||
// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault
|
||||
const std::vector<std::string> k_colors = {
|
||||
set_xterm256_foreground(220, 5, 12),
|
||||
set_xterm256_foreground(232, 96, 28),
|
||||
set_xterm256_foreground(241, 147, 45),
|
||||
set_xterm256_foreground(246, 193, 65),
|
||||
set_xterm256_foreground(247, 240, 86),
|
||||
set_xterm256_foreground(144, 201, 135),
|
||||
set_xterm256_foreground( 78, 178, 101),
|
||||
};
|
||||
|
||||
// ANSI formatting codes
|
||||
static std::string set_inverse() {
|
||||
return "\033[7m";
|
||||
}
|
||||
|
||||
static std::string set_underline() {
|
||||
return "\033[4m";
|
||||
}
|
||||
|
||||
static std::string set_dim() {
|
||||
return "\033[2m";
|
||||
}
|
||||
|
||||
// Style scheme for different confidence levels
|
||||
const std::vector<std::string> k_styles = {
|
||||
set_inverse(), // Low confidence - inverse (highlighted)
|
||||
set_underline(), // Medium confidence - underlined
|
||||
set_dim(), // High confidence - dim
|
||||
};
|
||||
|
||||
//
|
||||
// Other utils
|
||||
//
|
||||
|
||||
// check if file exists using ifstream
|
||||
bool is_file_exist(const char * filename);
|
||||
@@ -0,0 +1,32 @@
|
||||
#
|
||||
# gpt-2
|
||||
|
||||
set(TEST_TARGET gpt-2-ctx)
|
||||
add_executable(${TEST_TARGET} main-ctx.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
set(TEST_TARGET gpt-2-alloc)
|
||||
add_executable(${TEST_TARGET} main-alloc.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
set(TEST_TARGET gpt-2-backend)
|
||||
add_executable(${TEST_TARGET} main-backend.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
set(TEST_TARGET gpt-2-sched)
|
||||
add_executable(${TEST_TARGET} main-sched.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
#
|
||||
# gpt-2-quantize
|
||||
|
||||
set(TEST_TARGET gpt-2-quantize)
|
||||
add_executable(${TEST_TARGET} quantize.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
#
|
||||
# gpt-2-batched
|
||||
|
||||
set(TEST_TARGET gpt-2-batched)
|
||||
add_executable(${TEST_TARGET} main-batched.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
@@ -0,0 +1,225 @@
|
||||
# gpt-2
|
||||
|
||||
This is a C++ example running GPT-2 inference using the [ggml](https://github.com/ggerganov/ggml) library.
|
||||
|
||||
The program runs on the CPU - no video card is required.
|
||||
|
||||
The [Cerebras-GPT](https://huggingface.co/cerebras) models are also supported.
|
||||
|
||||
The example supports the following GPT-2 models:
|
||||
|
||||
| Model | Description | Disk Size |
|
||||
| --- | --- | --- |
|
||||
| 117M | Small model | 240 MB |
|
||||
| 345M | Medium model | 680 MB |
|
||||
| 774M | Large model | 1.5 GB |
|
||||
| 1558M | XL model | 3.0 GB |
|
||||
|
||||
Sample performance on MacBook M1 Pro:
|
||||
|
||||
| Model | Size | Time / Token |
|
||||
| --- | --- | --- |
|
||||
| GPT-2 | 117M | 5 ms |
|
||||
| GPT-2 | 345M | 12 ms |
|
||||
| GPT-2 | 774M | 23 ms |
|
||||
| GPT-2 | 1558M | 42 ms |
|
||||
|
||||
*TODO: add tables for Cerebras-GPT models*
|
||||
|
||||
Sample output:
|
||||
|
||||
```bash
|
||||
$ ./bin/gpt-2 -h
|
||||
usage: ./bin/gpt-2 [options]
|
||||
|
||||
options:
|
||||
-h, --help show this help message and exit
|
||||
-s SEED, --seed SEED RNG seed (default: -1)
|
||||
-t N, --threads N number of threads to use during computation (default: 8)
|
||||
-p PROMPT, --prompt PROMPT
|
||||
prompt to start generation with (default: random)
|
||||
-n N, --n_predict N number of tokens to predict (default: 200)
|
||||
--top_k N top-k sampling (default: 40)
|
||||
--top_p N top-p sampling (default: 0.9)
|
||||
--temp N temperature (default: 1.0)
|
||||
-b N, --batch_size N batch size for prompt processing (default: 8)
|
||||
-m FNAME, --model FNAME
|
||||
model path (default: models/gpt-2-117M/ggml-model.bin)
|
||||
|
||||
$ ./bin/gpt-2
|
||||
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
|
||||
gpt2_model_load: n_vocab = 50257
|
||||
gpt2_model_load: n_ctx = 1024
|
||||
gpt2_model_load: n_embd = 768
|
||||
gpt2_model_load: n_head = 12
|
||||
gpt2_model_load: n_layer = 12
|
||||
gpt2_model_load: f16 = 1
|
||||
gpt2_model_load: ggml ctx size = 311.12 MB
|
||||
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
|
||||
gpt2_model_load: model size = 239.08 MB
|
||||
main: number of tokens in prompt = 1
|
||||
|
||||
So this is going to be the end of the line for us.
|
||||
|
||||
If the Dolphins continue to do their business, it's possible that the team could make a bid to bring in new defensive coordinator Scott Linehan.
|
||||
|
||||
Linehan's job is a little daunting, but he's a great coach and an excellent coach. I don't believe we're going to make the playoffs.
|
||||
|
||||
We're going to have to work hard to keep our heads down and get ready to go.<|endoftext|>
|
||||
|
||||
main: mem per token = 2048612 bytes
|
||||
main: load time = 106.32 ms
|
||||
main: sample time = 7.10 ms
|
||||
main: predict time = 506.40 ms / 5.06 ms per token
|
||||
main: total time = 629.84 ms
|
||||
```
|
||||
|
||||
## Downloading and converting the original models (GPT-2)
|
||||
|
||||
You can download the original model files using the [download-model.sh](download-model.sh) Bash script. The models are
|
||||
in Tensorflow format, so in order to use them with ggml, you need to convert them to appropriate format. This is done
|
||||
via the [convert-ckpt-to-ggml.py](convert-ckpt-to-ggml.py) python script.
|
||||
|
||||
Here is the entire process for the GPT-2 117M model (download from official site + conversion):
|
||||
|
||||
```bash
|
||||
cd ggml/build
|
||||
../examples/gpt-2/download-model.sh 117M
|
||||
|
||||
Downloading model 117M ...
|
||||
models/gpt-2-117M/checkpoint 100%[=============================>] 77 --.-KB/s in 0s
|
||||
models/gpt-2-117M/encoder.json 100%[=============================>] 1018K 1.20MB/s in 0.8s
|
||||
models/gpt-2-117M/hparams.json 100%[=============================>] 90 --.-KB/s in 0s
|
||||
models/gpt-2-117M/model.ckpt.data-00000-of-00001 100%[=============================>] 474.70M 1.21MB/s in 8m 39s
|
||||
models/gpt-2-117M/model.ckpt.index 100%[=============================>] 5.09K --.-KB/s in 0s
|
||||
models/gpt-2-117M/model.ckpt.meta 100%[=============================>] 460.11K 806KB/s in 0.6s
|
||||
models/gpt-2-117M/vocab.bpe 100%[=============================>] 445.62K 799KB/s in 0.6s
|
||||
Done! Model '117M' saved in 'models/gpt-2-117M/'
|
||||
|
||||
Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.
|
||||
|
||||
python /Users/john/ggml/examples/gpt-2/convert-ckpt-to-ggml.py models/gpt-2-117M/ 1
|
||||
|
||||
```
|
||||
|
||||
This conversion requires that you have python and Tensorflow installed on your computer. Still, if you want to avoid
|
||||
this, you can download the already converted ggml models as described below.
|
||||
|
||||
## Downloading and converting the original models (Cerebras-GPT)
|
||||
|
||||
Clone the respective repository from here: https://huggingface.co/cerebras
|
||||
|
||||
Use the [convert-cerebras-to-ggml.py](convert-cerebras-to-ggml.py) script to convert the model to `ggml` format:
|
||||
|
||||
```bash
|
||||
cd ggml/build
|
||||
git clone https://huggingface.co/cerebras/Cerebras-GPT-111M models/
|
||||
python ../examples/gpt-2/convert-cerebras-to-ggml.py models/Cerebras-GPT-111M/
|
||||
|
||||
```
|
||||
|
||||
## Downloading the ggml model directly (GPT-2)
|
||||
|
||||
For convenience, I will be hosting the converted ggml model files in order to make it easier to run the examples. This
|
||||
way, you can directly download a single binary file and start using it. No python or Tensorflow is required.
|
||||
|
||||
Here is how to get the 117M ggml model:
|
||||
|
||||
```bash
|
||||
cd ggml/build
|
||||
../examples/gpt-2/download-ggml-model.sh 117M
|
||||
|
||||
Downloading ggml model 117M ...
|
||||
models/gpt-2-117M/ggml-model.bin 100%[===============================>] 239.58M 8.52MB/s in 28s
|
||||
Done! Model '117M' saved in 'models/gpt-2-117M/ggml-model.bin'
|
||||
You can now use it like this:
|
||||
|
||||
$ ./bin/gpt-2 -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
|
||||
|
||||
```
|
||||
|
||||
At some point, I might decide to stop hosting these models. So in that case, simply revert to the manual process above.
|
||||
|
||||
## Quantizing the models
|
||||
|
||||
You can also try to quantize the `ggml` models via 4-bit integer quantization.
|
||||
Keep in mind that for smaller models, this will render them completely useless.
|
||||
You generally want to quantize larger models.
|
||||
|
||||
```bash
|
||||
# quantize GPT-2 F16 to Q4_0 (faster but less precise)
|
||||
./bin/gpt-2-quantize models/gpt-2-1558M/ggml-model-f16.bin models/gpt-2-1558M/ggml-model-q4_0.bin 2
|
||||
./bin/gpt-2 -m models/gpt-2-1558M/ggml-model-q4_0.bin -p "This is an example"
|
||||
|
||||
# quantize Cerebras F16 to Q4_1 (slower but more precise)
|
||||
./bin/gpt-2-quantize models/Cerebras-GPT-6.7B/ggml-model-f16.bin models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin 3
|
||||
./bin/gpt-2 -m models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin -p "This is an example"
|
||||
|
||||
```
|
||||
|
||||
## Batched generation example
|
||||
|
||||
You can try the batched generation from a given prompt using the gpt-2-batched binary.
|
||||
|
||||
Sample output:
|
||||
|
||||
```bash
|
||||
$ gpt-2-batched -np 5 -m models/gpt-2-117M/ggml-model.bin -p "Hello my name is" -n 50
|
||||
|
||||
main: seed = 1697037431
|
||||
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
|
||||
gpt2_model_load: n_vocab = 50257
|
||||
gpt2_model_load: n_ctx = 1024
|
||||
gpt2_model_load: n_embd = 768
|
||||
gpt2_model_load: n_head = 12
|
||||
gpt2_model_load: n_layer = 12
|
||||
gpt2_model_load: ftype = 1
|
||||
gpt2_model_load: qntvr = 0
|
||||
gpt2_model_load: ggml tensor size = 320 bytes
|
||||
gpt2_model_load: backend buffer size = 312.72 MB
|
||||
ggml_init_cublas: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce GTX 1660, compute capability 7.5
|
||||
gpt2_model_load: using CPU backend
|
||||
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
|
||||
gpt2_model_load: model size = 239.08 MB
|
||||
extract_tests_from_file : No test file found.
|
||||
test_gpt_tokenizer : 0 tests failed out of 0 tests.
|
||||
main: compute buffer size: 3.26 MB
|
||||
|
||||
|
||||
main: generating 5 sequences ...
|
||||
main: prompt: 'Hello my name is'
|
||||
main: number of tokens in prompt = 4, first 8 tokens: 15496 616 1438 318
|
||||
|
||||
|
||||
sequence 0:
|
||||
|
||||
Hello my name is John. You can call me any way you want, if you want, but for my very first date, I will be on the phone with you. We're both in our early 20s, but I feel like it's all
|
||||
|
||||
sequence 1:
|
||||
|
||||
Hello my name is Robert, and I want to say that we're proud to have your company here on the world's largest platform for sharing your stories with us. This is a huge opportunity for our community. We have hundreds of people on this team and
|
||||
|
||||
sequence 2:
|
||||
|
||||
Hello my name is Jack. I'm the one who created you.
|
||||
|
||||
Jack is a boy with a big smile and a big heart. He is a handsome guy. He loves the outdoors and loves the people he meets. He wants to be a
|
||||
|
||||
sequence 3:
|
||||
|
||||
Hello my name is John. I am a Canadian citizen with a large number of family in Quebec and I am interested in studying. My aim is to take up a post in the Journal of the International Academy of Sciences of Canada which I am currently finishing.
|
||||
|
||||
sequence 4:
|
||||
|
||||
Hello my name is Dan. I am an entrepreneur. I am a great father. I am a great husband. I am a great husband. I am a great dad. And I am a great husband.
|
||||
|
||||
I love my life. I love
|
||||
|
||||
|
||||
|
||||
main: load time = 880.80 ms
|
||||
main: sample time = 91.43 ms
|
||||
main: predict time = 2518.29 ms
|
||||
main: total time = 3544.32 ms
|
||||
```
|
||||
@@ -0,0 +1,183 @@
|
||||
# Convert Cerebras models to ggml format
|
||||
#
|
||||
# ref: https://www.cerebras.net/blog/cerebras-gpt-a-family-of-open-compute-efficient-large-language-models/
|
||||
#
|
||||
|
||||
import sys
|
||||
import struct
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
import re
|
||||
|
||||
from transformers import AutoModelForCausalLM
|
||||
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8+n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: convert-cerebras-to-ggml.py dir-model [use-f32]\n")
|
||||
sys.exit(1)
|
||||
|
||||
# output in the same directory as the model
|
||||
dir_model = sys.argv[1]
|
||||
fname_out = sys.argv[1] + "/ggml-model-f16.bin"
|
||||
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
encoder = json.load(f)
|
||||
|
||||
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
|
||||
hparams = json.load(f)
|
||||
|
||||
# use 16-bit or 32-bit floats
|
||||
use_f16 = True
|
||||
if len(sys.argv) > 2:
|
||||
use_f16 = False
|
||||
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
|
||||
#print (model)
|
||||
|
||||
list_vars = model.state_dict()
|
||||
#print (list_vars)
|
||||
|
||||
print(hparams)
|
||||
|
||||
fout = open(fname_out, "wb")
|
||||
|
||||
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
|
||||
fout.write(struct.pack("i", hparams["vocab_size"]))
|
||||
fout.write(struct.pack("i", hparams["n_positions"]))
|
||||
fout.write(struct.pack("i", hparams["n_embd"]))
|
||||
fout.write(struct.pack("i", hparams["n_head"]))
|
||||
fout.write(struct.pack("i", hparams["n_layer"]))
|
||||
fout.write(struct.pack("i", use_f16))
|
||||
|
||||
byte_encoder = bytes_to_unicode()
|
||||
byte_decoder = {v:k for k, v in byte_encoder.items()}
|
||||
|
||||
fout.write(struct.pack("i", len(encoder)))
|
||||
|
||||
for key in encoder:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for name in list_vars.keys():
|
||||
data = list_vars[name].squeeze().numpy()
|
||||
print("Processing variable: " + name + " with shape: ", data.shape)
|
||||
|
||||
# rename headers to keep compatibility
|
||||
if name == "transformer.ln_f.weight":
|
||||
name = "model/ln_f/g"
|
||||
elif name == "transformer.ln_f.bias":
|
||||
name = "model/ln_f/b"
|
||||
elif name == "transformer.wte.weight":
|
||||
name = "model/wte"
|
||||
elif name == "transformer.wpe.weight":
|
||||
name = "model/wpe"
|
||||
elif name == "lm_head.weight":
|
||||
name = "model/lm_head"
|
||||
elif re.match(r"transformer.h\.\d+\.ln_1\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_1/g"
|
||||
elif re.match(r"transformer.h\.\d+\.ln_1\.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_1/b"
|
||||
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_attn/w"
|
||||
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_attn/b"
|
||||
elif re.match(r"transformer.h\.\d+\.attn\.c_proj\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_proj/w"
|
||||
elif re.match(r"transformer.h.\d+.attn.c_proj.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_proj/b"
|
||||
elif re.match(r"transformer.h.\d+.ln_2.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_2/g"
|
||||
elif re.match(r"transformer.h.\d+.ln_2.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_2/b"
|
||||
elif re.match(r"transformer.h.\d+.mlp.c_fc.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_fc/w"
|
||||
elif re.match(r"transformer.h.\d+.mlp.c_fc.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_fc/b"
|
||||
elif re.match(r"transformer.h.\d+.mlp.c_proj.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_proj/w"
|
||||
elif re.match(r"transformer.h.\d+.mlp.c_proj.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_proj/b"
|
||||
else:
|
||||
print("Unrecognized variable name. %s", name)
|
||||
|
||||
# we don't need these
|
||||
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
|
||||
print(" Skipping variable: " + name)
|
||||
continue
|
||||
|
||||
n_dims = len(data.shape);
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype = 0;
|
||||
if use_f16:
|
||||
if (name == "model/wte" or name == "model/lm_head" or name[-2:] == "/g" or name[-2:] == "/w") and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype = 0
|
||||
|
||||
# for efficiency - transpose the projection matrices
|
||||
# "model/h.*/attn/c_attn/w"
|
||||
# "model/h.*/attn/c_proj/w"
|
||||
# "model/h.*/mlp/c_fc/w"
|
||||
# "model/h.*/mlp/c_proj/w"
|
||||
if name[-14:] == "/attn/c_attn/w" or \
|
||||
name[-14:] == "/attn/c_proj/w" or \
|
||||
name[-11:] == "/mlp/c_fc/w" or \
|
||||
name[-13:] == "/mlp/c_proj/w":
|
||||
print(" Transposing")
|
||||
data = data.transpose()
|
||||
|
||||
# header
|
||||
str = name.encode('utf-8')
|
||||
fout.write(struct.pack("iii", n_dims, len(str), ftype))
|
||||
for i in range(n_dims):
|
||||
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
|
||||
fout.write(str);
|
||||
|
||||
# data
|
||||
data.tofile(fout)
|
||||
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
print("")
|
||||
@@ -0,0 +1,159 @@
|
||||
# Convert a model checkpoint to a ggml compatible file
|
||||
#
|
||||
# Load the model using TensorFlow.
|
||||
# Iterate over all variables and write them to a binary file.
|
||||
#
|
||||
# For each variable, write the following:
|
||||
# - Number of dimensions (int)
|
||||
# - Name length (int)
|
||||
# - Dimensions (int[n_dims])
|
||||
# - Name (char[name_length])
|
||||
# - Data (float[n_dims])
|
||||
#
|
||||
# By default, the bigger matrices are converted to 16-bit floats.
|
||||
# This can be disabled by adding the "use-f32" CLI argument.
|
||||
#
|
||||
# At the start of the ggml file we write the model parameters
|
||||
# and vocabulary.
|
||||
#
|
||||
|
||||
import sys
|
||||
import json
|
||||
import struct
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8+n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
# helper method to convert a numpy array to different float types
|
||||
def convert_to_ftype(data, ftype):
|
||||
# fp16
|
||||
if ftype == 1:
|
||||
return data.astype(np.float16)
|
||||
|
||||
assert False, "Invalid ftype: " + str(ftype)
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: convert-ckpt-to-ggml.py dir-model ftype\n")
|
||||
print(" ftype == 0 -> float32")
|
||||
print(" ftype == 1 -> float16")
|
||||
sys.exit(1)
|
||||
|
||||
# output in the same directory as the model
|
||||
dir_model = sys.argv[1]
|
||||
fname_out = sys.argv[1] + "/ggml-model.bin"
|
||||
|
||||
with open(dir_model + "/encoder.json", "r", encoding="utf-8") as f:
|
||||
encoder = json.load(f)
|
||||
|
||||
with open(dir_model + "/hparams.json", "r", encoding="utf-8") as f:
|
||||
hparams = json.load(f)
|
||||
|
||||
# possible data types
|
||||
# ftype == 0 -> float32
|
||||
# ftype == 1 -> float16
|
||||
#
|
||||
# map from ftype to string
|
||||
ftype_str = ["f32", "f16"]
|
||||
|
||||
ftype = 1
|
||||
if len(sys.argv) > 2:
|
||||
ftype = int(sys.argv[2])
|
||||
if ftype < 0 or ftype > 1:
|
||||
print("Invalid ftype: " + str(ftype))
|
||||
sys.exit(1)
|
||||
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
|
||||
|
||||
list_vars = tf.train.list_variables(dir_model)
|
||||
|
||||
fout = open(fname_out, "wb")
|
||||
|
||||
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
|
||||
fout.write(struct.pack("i", hparams["n_vocab"]))
|
||||
fout.write(struct.pack("i", hparams["n_ctx"]))
|
||||
fout.write(struct.pack("i", hparams["n_embd"]))
|
||||
fout.write(struct.pack("i", hparams["n_head"]))
|
||||
fout.write(struct.pack("i", hparams["n_layer"]))
|
||||
fout.write(struct.pack("i", ftype))
|
||||
|
||||
byte_encoder = bytes_to_unicode()
|
||||
byte_decoder = {v:k for k, v in byte_encoder.items()}
|
||||
|
||||
fout.write(struct.pack("i", len(encoder)))
|
||||
|
||||
for key in encoder:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for name, shape in list_vars:
|
||||
print("Processing variable: " + name + " with shape: ", shape)
|
||||
|
||||
data = tf.train.load_variable(dir_model, name).squeeze()
|
||||
n_dims = len(data.shape);
|
||||
|
||||
# for efficiency - transpose the projection matrices
|
||||
# "model/h.*/attn/c_attn/w"
|
||||
# "model/h.*/attn/c_proj/w"
|
||||
# "model/h.*/mlp/c_fc/w"
|
||||
# "model/h.*/mlp/c_proj/w"
|
||||
if name[-14:] == "/attn/c_attn/w" or \
|
||||
name[-14:] == "/attn/c_proj/w" or \
|
||||
name[-11:] == "/mlp/c_fc/w" or \
|
||||
name[-13:] == "/mlp/c_proj/w":
|
||||
print(" Transposing")
|
||||
data = data.transpose()
|
||||
|
||||
dshape = data.shape
|
||||
|
||||
ftype_cur = 0
|
||||
if ftype != 0:
|
||||
# match name:
|
||||
# "model/wte"
|
||||
# "model/h.*/attn/c_attn/w"
|
||||
# "model/h.*/attn/c_proj/w"
|
||||
# "model/h.*/mlp/c_fc/w"
|
||||
# "model/h.*/mlp/c_proj/w"
|
||||
if name == "model/wte" or name[-2:] == "/w":
|
||||
print(" Converting to " + ftype_str[ftype])
|
||||
data = convert_to_ftype(data, ftype)
|
||||
ftype_cur = ftype
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
|
||||
# header
|
||||
str = name.encode('utf-8')
|
||||
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
|
||||
for i in range(n_dims):
|
||||
fout.write(struct.pack("i", dshape[n_dims - 1 - i]))
|
||||
fout.write(str);
|
||||
|
||||
# data
|
||||
data.tofile(fout)
|
||||
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
print("")
|
||||
@@ -0,0 +1,195 @@
|
||||
# Convert GPT-2 h5 transformer model to ggml format
|
||||
#
|
||||
# Load the model using GPT2Model.
|
||||
# Iterate over all variables and write them to a binary file.
|
||||
#
|
||||
# For each variable, write the following:
|
||||
# - Number of dimensions (int)
|
||||
# - Name length (int)
|
||||
# - Dimensions (int[n_dims])
|
||||
# - Name (char[name_length])
|
||||
# - Data (float[n_dims])
|
||||
#
|
||||
# By default, the bigger matrices are converted to 16-bit floats.
|
||||
# This can be disabled by adding the "use-f32" CLI argument.
|
||||
#
|
||||
# At the start of the ggml file we write the model parameters
|
||||
# and vocabulary.
|
||||
#
|
||||
|
||||
import sys
|
||||
import struct
|
||||
import json
|
||||
import numpy as np
|
||||
import re
|
||||
|
||||
from transformers import GPT2Model
|
||||
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8+n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
|
||||
sys.exit(1)
|
||||
|
||||
# output in the same directory as the model
|
||||
dir_model = sys.argv[1]
|
||||
fname_out = sys.argv[1] + "/ggml-model.bin"
|
||||
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
encoder = json.load(f)
|
||||
|
||||
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
|
||||
encoder_added = json.load(f)
|
||||
|
||||
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
|
||||
hparams = json.load(f)
|
||||
|
||||
# use 16-bit or 32-bit floats
|
||||
use_f16 = True
|
||||
if len(sys.argv) > 2:
|
||||
use_f16 = False
|
||||
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
|
||||
|
||||
model = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True)
|
||||
#print (model)
|
||||
|
||||
list_vars = model.state_dict()
|
||||
#print (list_vars)
|
||||
|
||||
fout = open(fname_out, "wb")
|
||||
|
||||
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
|
||||
fout.write(struct.pack("i", hparams["vocab_size"]))
|
||||
fout.write(struct.pack("i", hparams["n_positions"]))
|
||||
fout.write(struct.pack("i", hparams["n_embd"]))
|
||||
fout.write(struct.pack("i", hparams["n_head"]))
|
||||
fout.write(struct.pack("i", hparams["n_layer"]))
|
||||
#fout.write(struct.pack("i", hparams["rotary_dim"]))
|
||||
fout.write(struct.pack("i", use_f16))
|
||||
|
||||
byte_encoder = bytes_to_unicode()
|
||||
byte_decoder = {v:k for k, v in byte_encoder.items()}
|
||||
|
||||
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
|
||||
|
||||
for key in encoder:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for key in encoder_added:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for name in list_vars.keys():
|
||||
data = list_vars[name].squeeze().numpy()
|
||||
print("Processing variable: " + name + " with shape: ", data.shape)
|
||||
|
||||
# we don't need these
|
||||
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
|
||||
print(" Skipping variable: " + name)
|
||||
continue
|
||||
|
||||
n_dims = len(data.shape);
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype = 0;
|
||||
if use_f16:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype = 0
|
||||
|
||||
# for efficiency - transpose these matrices:
|
||||
# "transformer.h.*.mlp.c_proj.weight
|
||||
if name.endswith(".mlp.c_proj.weight"):
|
||||
print(" Transposing")
|
||||
data = data.transpose()
|
||||
|
||||
# rename headers to keep compatibility
|
||||
if name == "ln_f.weight":
|
||||
name = "model/ln_f/g"
|
||||
elif name == "ln_f.bias":
|
||||
name = "model/ln_f/b"
|
||||
elif name == "wte.weight":
|
||||
name = "model/wte"
|
||||
elif name == "wpe.weight":
|
||||
name = "model/wpe"
|
||||
elif re.match(r"h\.\d+\.ln_1\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_1/g"
|
||||
elif re.match(r"h\.\d+\.ln_1\.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_1/b"
|
||||
elif re.match(r"h\.\d+\.attn\.c_attn\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_attn/w"
|
||||
elif re.match(r"h\.\d+\.attn\.c_attn\.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_attn/b"
|
||||
elif re.match(r"h\.\d+\.attn\.c_proj\.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_proj/w"
|
||||
elif re.match(r"h.\d+.attn.c_proj.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/attn/c_proj/b"
|
||||
elif re.match(r"h.\d+.ln_2.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_2/g"
|
||||
elif re.match(r"h.\d+.ln_2.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/ln_2/b"
|
||||
elif re.match(r"h.\d+.mlp.c_fc.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_fc/w"
|
||||
elif re.match(r"h.\d+.mlp.c_fc.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_fc/b"
|
||||
elif re.match(r"h.\d+.mlp.c_proj.weight", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_proj/w"
|
||||
elif re.match(r"h.\d+.mlp.c_proj.bias", name):
|
||||
i = re.findall("\d+", name)[0]
|
||||
name = f"model/h{i}/mlp/c_proj/b"
|
||||
else:
|
||||
print("Unrecognized variable name. %s", name)
|
||||
|
||||
str = name.encode('utf-8')
|
||||
|
||||
fout.write(struct.pack("iii", n_dims, len(str), ftype))
|
||||
for i in range(n_dims):
|
||||
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
|
||||
fout.write(str);
|
||||
|
||||
# data
|
||||
data.tofile(fout)
|
||||
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
print("")
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script downloads GPT-2 model files that have already been converted to ggml format.
|
||||
# This way you don't have to convert them yourself.
|
||||
#
|
||||
# If you want to download the original GPT-2 model files, use the "download-model.sh" script instead.
|
||||
|
||||
#src="https://ggml.ggerganov.com"
|
||||
#pfx="ggml-model-gpt-2"
|
||||
|
||||
src="https://huggingface.co/ggerganov/ggml"
|
||||
pfx="resolve/main/ggml-model-gpt-2"
|
||||
|
||||
ggml_path=$(dirname $(realpath $0))
|
||||
|
||||
# GPT-2 models
|
||||
models=( "117M" "345M" "774M" "1558M" )
|
||||
|
||||
# list available models
|
||||
function list_models {
|
||||
printf "\n"
|
||||
printf " Available models:"
|
||||
for model in "${models[@]}"; do
|
||||
printf " $model"
|
||||
done
|
||||
printf "\n\n"
|
||||
}
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
printf "Usage: $0 <model>\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
model=$1
|
||||
|
||||
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
|
||||
printf "Invalid model: $model\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# download ggml model
|
||||
|
||||
printf "Downloading ggml model $model ...\n"
|
||||
|
||||
mkdir -p models/gpt-2-$model
|
||||
|
||||
if [ -x "$(command -v wget)" ]; then
|
||||
wget --quiet --show-progress -O models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
|
||||
elif [ -x "$(command -v curl)" ]; then
|
||||
curl -L --output models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
|
||||
else
|
||||
printf "Either wget or curl is required to download models.\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
printf "Failed to download ggml model $model \n"
|
||||
printf "Please try again later or download the original GPT-2 model files and convert them yourself.\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "Done! Model '$model' saved in 'models/gpt-2-$model/ggml-model.bin'\n"
|
||||
printf "You can now use it like this:\n\n"
|
||||
printf " $ ./bin/gpt-2 -m models/gpt-2-$model/ggml-model.bin -p \"This is an example\"\n"
|
||||
printf "\n"
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
|
||||
ggml_path=$(dirname $(realpath $0))
|
||||
|
||||
# GPT-2 models
|
||||
models=( "117M" "345M" "774M" "1558M" )
|
||||
|
||||
# list available models
|
||||
function list_models {
|
||||
printf "\n"
|
||||
printf " Available models:"
|
||||
for model in "${models[@]}"; do
|
||||
printf " $model"
|
||||
done
|
||||
printf "\n\n"
|
||||
}
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
printf "Usage: $0 <model>\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
model=$1
|
||||
|
||||
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
|
||||
printf "Invalid model: $model\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# download model
|
||||
|
||||
printf "Downloading model $model ...\n"
|
||||
|
||||
mkdir -p models/gpt-2-$model
|
||||
|
||||
for file in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do
|
||||
wget --quiet --show-progress -O models/gpt-2-$model/$file https://openaipublic.blob.core.windows.net/gpt-2/models/$model/$file
|
||||
done
|
||||
|
||||
printf "Done! Model '$model' saved in 'models/gpt-2-$model/'\n\n"
|
||||
printf "Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.\n"
|
||||
printf "\n"
|
||||
printf " python $ggml_path/convert-ckpt-to-ggml.py models/gpt-2-$model/\n"
|
||||
printf "\n"
|
||||
@@ -0,0 +1,880 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
// default hparams (GPT-2 117M)
|
||||
struct gpt2_hparams {
|
||||
int32_t n_vocab = 50257;
|
||||
int32_t n_ctx = 1024;
|
||||
int32_t n_embd = 768;
|
||||
int32_t n_head = 12;
|
||||
int32_t n_layer = 12;
|
||||
int32_t ftype = 1;
|
||||
float eps = 1e-5f;
|
||||
};
|
||||
|
||||
struct gpt2_layer {
|
||||
// normalization
|
||||
struct ggml_tensor * ln_1_g;
|
||||
struct ggml_tensor * ln_1_b;
|
||||
|
||||
struct ggml_tensor * ln_2_g;
|
||||
struct ggml_tensor * ln_2_b;
|
||||
|
||||
// attention
|
||||
struct ggml_tensor * c_attn_attn_w;
|
||||
struct ggml_tensor * c_attn_attn_b;
|
||||
|
||||
struct ggml_tensor * c_attn_proj_w;
|
||||
struct ggml_tensor * c_attn_proj_b;
|
||||
|
||||
// mlp
|
||||
struct ggml_tensor * c_mlp_fc_w;
|
||||
struct ggml_tensor * c_mlp_fc_b;
|
||||
|
||||
struct ggml_tensor * c_mlp_proj_w;
|
||||
struct ggml_tensor * c_mlp_proj_b;
|
||||
};
|
||||
|
||||
struct gpt2_model {
|
||||
gpt2_hparams hparams;
|
||||
|
||||
// normalization
|
||||
struct ggml_tensor * ln_f_g;
|
||||
struct ggml_tensor * ln_f_b;
|
||||
|
||||
struct ggml_tensor * wte; // token embedding
|
||||
struct ggml_tensor * wpe; // position embedding
|
||||
struct ggml_tensor * lm_head; // language model head
|
||||
|
||||
std::vector<gpt2_layer> layers;
|
||||
|
||||
// key + value memory
|
||||
struct ggml_tensor * memory_k;
|
||||
struct ggml_tensor * memory_v;
|
||||
|
||||
//
|
||||
struct ggml_context * ctx_w;
|
||||
std::map<std::string, struct ggml_tensor *> tensors;
|
||||
};
|
||||
|
||||
// load the model's weights from a file
|
||||
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
|
||||
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
fin.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// load hparams
|
||||
{
|
||||
auto & hparams = model.hparams;
|
||||
|
||||
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: ftype = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr = %d\n", __func__, qntvr);
|
||||
|
||||
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
fin.read((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != model.hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
std::vector<char> buf(128);
|
||||
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
fin.read((char *) &len, sizeof(len));
|
||||
|
||||
buf.resize(len);
|
||||
fin.read((char *) buf.data(), len);
|
||||
word.assign(buf.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
|
||||
// in order to save memory and also to speed up the computation
|
||||
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
|
||||
if (wtype == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
|
||||
__func__, fname.c_str(), model.hparams.ftype);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto & ctx = model.ctx_w;
|
||||
|
||||
size_t ctx_size = 0;
|
||||
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
|
||||
|
||||
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32 , n_ctx*n_embd); // wpe
|
||||
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
|
||||
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
|
||||
|
||||
ctx_size += (6 + 12*n_layer)*512; // object overhead
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
|
||||
}
|
||||
|
||||
// create the ggml context
|
||||
{
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ctx_size,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
model.ctx_w = ggml_init(params);
|
||||
if (!model.ctx_w) {
|
||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// prepare memory for the weights
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
model.layers.resize(n_layer);
|
||||
|
||||
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
|
||||
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/ln_f/g"] = model.ln_f_g;
|
||||
model.tensors["model/ln_f/b"] = model.ln_f_b;
|
||||
|
||||
model.tensors["model/wte"] = model.wte;
|
||||
model.tensors["model/wpe"] = model.wpe;
|
||||
model.tensors["model/lm_head"] = model.lm_head;
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = model.layers[i];
|
||||
|
||||
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
|
||||
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
|
||||
|
||||
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
|
||||
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
|
||||
|
||||
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
|
||||
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
|
||||
}
|
||||
}
|
||||
|
||||
// key + value memory
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
|
||||
const int n_mem = n_layer*n_ctx;
|
||||
const int n_elements = n_embd*n_mem;
|
||||
|
||||
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
|
||||
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
|
||||
|
||||
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
|
||||
}
|
||||
|
||||
// load weights
|
||||
{
|
||||
size_t total_size = 0;
|
||||
|
||||
bool has_lm_head = false;
|
||||
|
||||
while (true) {
|
||||
int32_t n_dims;
|
||||
int32_t length;
|
||||
int32_t ttype;
|
||||
|
||||
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
|
||||
if (fin.eof()) {
|
||||
break;
|
||||
}
|
||||
|
||||
int32_t nelements = 1;
|
||||
int32_t ne[2] = { 1, 1 };
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
nelements *= ne[i];
|
||||
}
|
||||
|
||||
std::string name(length, 0);
|
||||
fin.read(&name[0], length);
|
||||
|
||||
if (model.tensors.find(name) == model.tensors.end()) {
|
||||
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tensor = model.tensors[name];
|
||||
if (ggml_nelements(tensor) != nelements) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
|
||||
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
|
||||
return false;
|
||||
}
|
||||
|
||||
// for debugging
|
||||
if (0) {
|
||||
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
const size_t bpe = ggml_type_size(ggml_type(ttype));
|
||||
|
||||
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
|
||||
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
|
||||
return false;
|
||||
}
|
||||
|
||||
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
|
||||
|
||||
// GPT-2 models share the WTE tensor as the LM head
|
||||
if (name == "model/wte" && has_lm_head == false) {
|
||||
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
if (name == "model/lm_head") {
|
||||
has_lm_head = true;
|
||||
}
|
||||
|
||||
total_size += ggml_nbytes(tensor);
|
||||
}
|
||||
|
||||
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// build the computation graph
|
||||
struct ggml_cgraph * gpt2_graph(
|
||||
const gpt2_model & model,
|
||||
const int n_past,
|
||||
const int n_tokens) {
|
||||
const int N = n_tokens;
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_head = hparams.n_head;
|
||||
|
||||
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
|
||||
// at this point, the tensor data is not allocated yet and cannot be set
|
||||
// we will find the tensor after the graph is allocated by its name, and set the data then
|
||||
ggml_set_name(embd, "embd");
|
||||
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
|
||||
// this is important to ensure that the input tensors are not overwritten before they are used
|
||||
ggml_set_input(embd);
|
||||
|
||||
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
|
||||
ggml_set_name(position, "position");
|
||||
ggml_set_input(position);
|
||||
|
||||
// wte + wpe
|
||||
struct ggml_tensor * inpL =
|
||||
ggml_add(ctx,
|
||||
ggml_get_rows(ctx, model.wte, embd),
|
||||
ggml_get_rows(ctx, model.wpe, position));
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
struct ggml_tensor * cur;
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
cur = ggml_norm(ctx, inpL, hparams.eps);
|
||||
|
||||
// cur = ln_1_g*cur + ln_1_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].ln_1_g, cur),
|
||||
cur),
|
||||
ggml_repeat(ctx, model.layers[il].ln_1_b, cur));
|
||||
}
|
||||
|
||||
// attn
|
||||
// [2304, 768] - model.layers[il].c_attn_attn_w
|
||||
// [2304, 1] - model.layers[il].c_attn_attn_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [2304, N] - cur (out)
|
||||
//
|
||||
// cur = attn_w*cur + attn_b
|
||||
// [2304, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_attn_attn_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].c_attn_attn_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// self-attention
|
||||
{
|
||||
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
|
||||
|
||||
// store key and value to memory
|
||||
if (N >= 1) {
|
||||
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
|
||||
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
|
||||
}
|
||||
|
||||
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * Q =
|
||||
ggml_permute(ctx,
|
||||
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
|
||||
// [64, n_past + N, 12]
|
||||
struct ggml_tensor * K =
|
||||
ggml_permute(ctx,
|
||||
ggml_reshape_3d(ctx,
|
||||
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// GG: flash attention
|
||||
//struct ggml_tensor * V =
|
||||
// ggml_cpy(ctx0,
|
||||
// ggml_permute(ctx0,
|
||||
// ggml_reshape_3d(ctx0,
|
||||
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
// n_embd/n_head, n_head, n_past + N),
|
||||
// 1, 2, 0, 3),
|
||||
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
|
||||
|
||||
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
|
||||
|
||||
// K * Q
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
|
||||
|
||||
// KQ_scaled = KQ / sqrt(n_embd/n_head)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_scaled =
|
||||
ggml_scale(ctx,
|
||||
KQ,
|
||||
1.0f/sqrtf(float(n_embd)/n_head));
|
||||
|
||||
// KQ_masked = mask_past(KQ_scaled)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
|
||||
|
||||
// KQ = soft_max(KQ_masked)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
|
||||
|
||||
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
|
||||
// [n_past + N, 64, 12]
|
||||
struct ggml_tensor * V_trans =
|
||||
ggml_cont_3d(ctx,
|
||||
ggml_permute(ctx,
|
||||
ggml_reshape_3d(ctx,
|
||||
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
1, 2, 0, 3),
|
||||
n_past + N, n_embd/n_head, n_head);
|
||||
|
||||
// KQV = transpose(V) * KQ_soft_max
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
|
||||
|
||||
// KQV_merged = KQV.permute(0, 2, 1, 3)
|
||||
// [64, 12, N]
|
||||
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
|
||||
|
||||
// cur = KQV_merged.contiguous().view(n_embd, N)
|
||||
// [768, N]
|
||||
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
|
||||
}
|
||||
|
||||
// projection
|
||||
// [ 768, 768] - model.layers[il].c_attn_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_attn_proj_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_attn_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].c_attn_proj_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// add the input
|
||||
cur = ggml_add(ctx, cur, inpL);
|
||||
|
||||
struct ggml_tensor * inpFF = cur;
|
||||
|
||||
// feed-forward network
|
||||
{
|
||||
// norm
|
||||
{
|
||||
cur = ggml_norm(ctx, inpFF, hparams.eps);
|
||||
|
||||
// cur = ln_2_g*cur + ln_2_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].ln_2_g, cur),
|
||||
cur),
|
||||
ggml_repeat(ctx, model.layers[il].ln_2_b, cur));
|
||||
}
|
||||
|
||||
// fully connected
|
||||
// [3072, 768] - model.layers[il].c_mlp_fc_w
|
||||
// [3072, 1] - model.layers[il].c_mlp_fc_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [3072, N] - cur (out)
|
||||
//
|
||||
// cur = fc_w*cur + fc_b
|
||||
// [3072, N]
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_mlp_fc_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].c_mlp_fc_b, cur),
|
||||
cur);
|
||||
|
||||
// GELU activation
|
||||
// [3072, N]
|
||||
cur = ggml_gelu(ctx, cur);
|
||||
|
||||
// projection
|
||||
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_mlp_proj_b
|
||||
// [3072, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_mlp_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
ggml_repeat(ctx, model.layers[il].c_mlp_proj_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// input for next layer
|
||||
inpL = ggml_add(ctx, cur, inpFF);
|
||||
}
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
inpL = ggml_norm(ctx, inpL, hparams.eps);
|
||||
|
||||
// inpL = ln_f_g*inpL + ln_f_b
|
||||
// [ 768, N]
|
||||
inpL = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
ggml_repeat(ctx, model.ln_f_g, inpL),
|
||||
inpL),
|
||||
ggml_repeat(ctx, model.ln_f_b, inpL));
|
||||
}
|
||||
|
||||
// inpL = WTE * inpL
|
||||
// [ 768, 50257] - model.lm_head
|
||||
// [ 768, N] - inpL
|
||||
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
|
||||
ggml_set_name(inpL, "logits");
|
||||
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
|
||||
ggml_set_output(inpL);
|
||||
|
||||
// logits -> probs
|
||||
//inpL = ggml_soft_max(ctx0, inpL);
|
||||
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
|
||||
ggml_free(ctx);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
// - model: the model
|
||||
// - allocr: ggml_gallocr to use to allocate the compute buffer
|
||||
// - n_threads: number of threads to use
|
||||
// - n_past: the context size so far
|
||||
// - embd_inp: the embeddings of the tokens in the context
|
||||
// - embd_w: the predicted logits for the next token
|
||||
//
|
||||
bool gpt2_eval(
|
||||
const gpt2_model & model,
|
||||
ggml_gallocr_t allocr,
|
||||
const int n_threads,
|
||||
const int n_past,
|
||||
const std::vector<gpt_vocab::id> & embd_inp,
|
||||
std::vector<float> & embd_w) {
|
||||
const int N = embd_inp.size();
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
|
||||
|
||||
// allocate the graph tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
|
||||
// set the graph inputs
|
||||
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
|
||||
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
|
||||
|
||||
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
|
||||
for (int i = 0; i < N; ++i) {
|
||||
((int32_t *) position->data)[i] = n_past + i;
|
||||
}
|
||||
|
||||
// run the computation
|
||||
struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr);
|
||||
static std::vector<uint8_t> work_buffer;
|
||||
work_buffer.resize(plan.work_size);
|
||||
plan.work_data = work_buffer.data();
|
||||
ggml_graph_compute(gf, &plan);
|
||||
|
||||
//if (n_past%100 == 0) {
|
||||
// ggml_graph_print (&gf);
|
||||
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
|
||||
//}
|
||||
|
||||
// get the graph outputs
|
||||
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
|
||||
|
||||
//embd_w.resize(n_vocab*N);
|
||||
//memcpy(embd_w.data(), ggml_get_data(logits), sizeof(float)*n_vocab*N);
|
||||
|
||||
// return result just for the last token
|
||||
embd_w.resize(n_vocab);
|
||||
memcpy(embd_w.data(), (float *) ggml_get_data(logits) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_time_init();
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
gpt_params params;
|
||||
params.model = "models/gpt-2-117M/ggml-model.bin";
|
||||
|
||||
if (gpt_params_parse(argc, argv, params) == false) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.seed < 0) {
|
||||
params.seed = time(NULL);
|
||||
}
|
||||
|
||||
printf("%s: seed = %d\n", __func__, params.seed);
|
||||
|
||||
std::mt19937 rng(params.seed);
|
||||
if (params.prompt.empty()) {
|
||||
params.prompt = gpt_random_prompt(rng);
|
||||
}
|
||||
|
||||
int64_t t_load_us = 0;
|
||||
|
||||
gpt_vocab vocab;
|
||||
gpt2_model model;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_model_load(params.model, model, vocab)) {
|
||||
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_load_us = ggml_time_us() - t_start_us;
|
||||
|
||||
test_gpt_tokenizer(vocab, params.token_test);
|
||||
}
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
// allocate the compute buffer
|
||||
{
|
||||
allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());
|
||||
|
||||
// create the worst case graph for memory usage estimation
|
||||
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
|
||||
int n_past = model.hparams.n_ctx - n_tokens;
|
||||
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
|
||||
|
||||
// pre-allocate the compute buffer for the worst case (optional)
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
|
||||
int64_t t_sample_us = 0;
|
||||
int64_t t_predict_us = 0;
|
||||
|
||||
std::vector<float> logits;
|
||||
|
||||
// tokenize the prompt
|
||||
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
|
||||
|
||||
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
|
||||
|
||||
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
|
||||
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
|
||||
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
|
||||
printf("%d ", embd_inp[i]);
|
||||
}
|
||||
printf("\n\n");
|
||||
|
||||
// submit the input prompt token-by-token
|
||||
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
|
||||
std::vector<gpt_vocab::id> embd;
|
||||
|
||||
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
|
||||
// predict
|
||||
if (embd.size() > 0) {
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
|
||||
printf("Failed to predict\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_predict_us += ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
n_past += embd.size();
|
||||
embd.clear();
|
||||
|
||||
if (i >= embd_inp.size()) {
|
||||
// sample next token
|
||||
const int top_k = params.top_k;
|
||||
const float top_p = params.top_p;
|
||||
const float temp = params.temp;
|
||||
|
||||
const int n_vocab = model.hparams.n_vocab;
|
||||
|
||||
gpt_vocab::id id = 0;
|
||||
|
||||
{
|
||||
const int64_t t_start_sample_us = ggml_time_us();
|
||||
|
||||
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
|
||||
|
||||
t_sample_us += ggml_time_us() - t_start_sample_us;
|
||||
}
|
||||
|
||||
// add it to the context
|
||||
embd.push_back(id);
|
||||
} else {
|
||||
// if here, it means we are still processing the input prompt
|
||||
for (size_t k = i; k < embd_inp.size(); k++) {
|
||||
embd.push_back(embd_inp[k]);
|
||||
if (int32_t(embd.size()) >= params.n_batch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
i += embd.size() - 1;
|
||||
}
|
||||
|
||||
// display text
|
||||
for (auto id : embd) {
|
||||
printf("%s", vocab.id_to_token[id].c_str());
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
// end of text token
|
||||
if (embd.back() == 50256) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n\n");
|
||||
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
|
||||
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
|
||||
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
ggml_free(model.ctx_w);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,946 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
#include "ggml-cuda.h"
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
#include "ggml-metal.h"
|
||||
#endif
|
||||
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
#define GPT2_MAX_NODES 4096
|
||||
|
||||
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
|
||||
(void) level;
|
||||
(void) user_data;
|
||||
fputs(text, stderr);
|
||||
fflush(stderr);
|
||||
}
|
||||
|
||||
// default hparams (GPT-2 117M)
|
||||
struct gpt2_hparams {
|
||||
int32_t n_vocab = 50257;
|
||||
int32_t n_ctx = 1024;
|
||||
int32_t n_embd = 768;
|
||||
int32_t n_head = 12;
|
||||
int32_t n_layer = 12;
|
||||
int32_t ftype = 1;
|
||||
float eps = 1e-5f;
|
||||
};
|
||||
|
||||
struct gpt2_layer {
|
||||
// normalization
|
||||
struct ggml_tensor * ln_1_g;
|
||||
struct ggml_tensor * ln_1_b;
|
||||
|
||||
struct ggml_tensor * ln_2_g;
|
||||
struct ggml_tensor * ln_2_b;
|
||||
|
||||
// attention
|
||||
struct ggml_tensor * c_attn_attn_w;
|
||||
struct ggml_tensor * c_attn_attn_b;
|
||||
|
||||
struct ggml_tensor * c_attn_proj_w;
|
||||
struct ggml_tensor * c_attn_proj_b;
|
||||
|
||||
// mlp
|
||||
struct ggml_tensor * c_mlp_fc_w;
|
||||
struct ggml_tensor * c_mlp_fc_b;
|
||||
|
||||
struct ggml_tensor * c_mlp_proj_w;
|
||||
struct ggml_tensor * c_mlp_proj_b;
|
||||
};
|
||||
|
||||
struct gpt2_model {
|
||||
gpt2_hparams hparams;
|
||||
|
||||
// normalization
|
||||
struct ggml_tensor * ln_f_g;
|
||||
struct ggml_tensor * ln_f_b;
|
||||
|
||||
struct ggml_tensor * wte; // token embedding
|
||||
struct ggml_tensor * wpe; // position embedding
|
||||
struct ggml_tensor * lm_head; // language model head
|
||||
|
||||
std::vector<gpt2_layer> layers;
|
||||
|
||||
// key + value memory
|
||||
struct ggml_tensor * memory_k;
|
||||
struct ggml_tensor * memory_v;
|
||||
|
||||
//
|
||||
struct ggml_context * ctx_w;
|
||||
struct ggml_context * ctx_kv;
|
||||
|
||||
ggml_backend_t backend = NULL;
|
||||
|
||||
ggml_backend_buffer_t buffer_w;
|
||||
ggml_backend_buffer_t buffer_kv;
|
||||
|
||||
std::map<std::string, struct ggml_tensor *> tensors;
|
||||
};
|
||||
|
||||
// load the model's weights from a file
|
||||
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) {
|
||||
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
fin.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// load hparams
|
||||
{
|
||||
auto & hparams = model.hparams;
|
||||
|
||||
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: ftype = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr = %d\n", __func__, qntvr);
|
||||
|
||||
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
fin.read((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != model.hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
std::vector<char> buf(128);
|
||||
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
fin.read((char *) &len, sizeof(len));
|
||||
|
||||
buf.resize(len);
|
||||
fin.read((char *) buf.data(), len);
|
||||
word.assign(buf.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
|
||||
// in order to save memory and also to speed up the computation
|
||||
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
|
||||
if (wtype == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
|
||||
__func__, fname.c_str(), model.hparams.ftype);
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_log_set(ggml_log_callback_default, nullptr);
|
||||
|
||||
auto & ctx = model.ctx_w;
|
||||
|
||||
// create the ggml context
|
||||
{
|
||||
size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer;
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
ctx = ggml_init(params);
|
||||
if (!ctx) {
|
||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// initialize the backend
|
||||
#ifdef GGML_USE_CUDA
|
||||
if (n_gpu_layers > 0) {
|
||||
fprintf(stderr, "%s: using CUDA backend\n", __func__);
|
||||
model.backend = ggml_backend_cuda_init(0);
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_METAL
|
||||
if (n_gpu_layers > 0) {
|
||||
fprintf(stderr, "%s: using Metal backend\n", __func__);
|
||||
model.backend = ggml_backend_metal_init();
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (!model.backend) {
|
||||
// fallback to CPU backend
|
||||
fprintf(stderr, "%s: using CPU backend\n", __func__);
|
||||
model.backend = ggml_backend_cpu_init();
|
||||
}
|
||||
|
||||
if (!model.backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
// create the tensors for the model
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
model.layers.resize(n_layer);
|
||||
|
||||
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
|
||||
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/ln_f/g"] = model.ln_f_g;
|
||||
model.tensors["model/ln_f/b"] = model.ln_f_b;
|
||||
|
||||
model.tensors["model/wte"] = model.wte;
|
||||
model.tensors["model/wpe"] = model.wpe;
|
||||
model.tensors["model/lm_head"] = model.lm_head;
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = model.layers[i];
|
||||
|
||||
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
|
||||
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
|
||||
|
||||
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
|
||||
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
|
||||
|
||||
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
|
||||
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
|
||||
}
|
||||
}
|
||||
|
||||
// allocate the model tensors in a backend buffer
|
||||
model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: backend buffer size = %6.2f MB\n", __func__, ggml_backend_buffer_get_size(model.buffer_w)/(1024.0*1024.0));
|
||||
|
||||
// override the default training context with the user-provided
|
||||
model.hparams.n_ctx = n_ctx;
|
||||
|
||||
// key + value memory
|
||||
{
|
||||
auto * ctx = model.ctx_kv;
|
||||
|
||||
// create the ggml context
|
||||
{
|
||||
size_t n_tensors = 2;
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
ctx = ggml_init(params);
|
||||
if (!ctx) {
|
||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
|
||||
const int n_mem = n_layer*n_ctx;
|
||||
const int n_elements = n_embd*n_mem;
|
||||
|
||||
// k and v here can also be GGML_TYPE_F16 to save memory and speed up the computation
|
||||
// if backend supports it
|
||||
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
|
||||
// allocate the KV memory in a backend buffer
|
||||
model.buffer_kv = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
|
||||
|
||||
const size_t memory_size = ggml_backend_buffer_get_size(model.buffer_kv);
|
||||
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
|
||||
}
|
||||
|
||||
// load weights
|
||||
{
|
||||
size_t total_size = 0;
|
||||
|
||||
bool has_lm_head = false;
|
||||
|
||||
std::vector<char> read_buf;
|
||||
|
||||
while (true) {
|
||||
int32_t n_dims;
|
||||
int32_t length;
|
||||
int32_t ttype;
|
||||
|
||||
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
|
||||
if (fin.eof()) {
|
||||
break;
|
||||
}
|
||||
|
||||
int32_t nelements = 1;
|
||||
int32_t ne[2] = { 1, 1 };
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
nelements *= ne[i];
|
||||
}
|
||||
|
||||
std::string name(length, 0);
|
||||
fin.read(&name[0], length);
|
||||
|
||||
if (model.tensors.find(name) == model.tensors.end()) {
|
||||
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tensor = model.tensors[name];
|
||||
ggml_set_name(tensor, name.c_str());
|
||||
if (ggml_nelements(tensor) != nelements) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
|
||||
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
|
||||
return false;
|
||||
}
|
||||
|
||||
// for debugging
|
||||
if (0) {
|
||||
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
const size_t bpe = ggml_type_size(ggml_type(ttype));
|
||||
|
||||
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
|
||||
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ggml_backend_buffer_is_host(model.buffer_w)) {
|
||||
// for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it
|
||||
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
|
||||
} else {
|
||||
// read into a temporary buffer first, then copy to device memory
|
||||
read_buf.resize(ggml_nbytes(tensor));
|
||||
fin.read(read_buf.data(), ggml_nbytes(tensor));
|
||||
ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
// GPT-2 models share the WTE tensor as the LM head
|
||||
if (name == "model/wte" && has_lm_head == false) {
|
||||
//ggml_backend_tensor_copy(tensor, model.lm_head);
|
||||
model.lm_head = tensor;
|
||||
}
|
||||
|
||||
if (name == "model/lm_head") {
|
||||
has_lm_head = true;
|
||||
}
|
||||
|
||||
total_size += ggml_nbytes(tensor);
|
||||
}
|
||||
|
||||
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// build the computation graph
|
||||
struct ggml_cgraph * gpt2_graph(
|
||||
const gpt2_model & model,
|
||||
const int n_past,
|
||||
const int n_tokens) {
|
||||
const int N = n_tokens;
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_head = hparams.n_head;
|
||||
|
||||
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
|
||||
static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);
|
||||
|
||||
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
|
||||
// at this point, the tensor data is not allocated yet and cannot be set
|
||||
// we will find the tensor after the graph is allocated by its name, and set the data then
|
||||
ggml_set_name(embd, "embd");
|
||||
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
|
||||
// this is important to ensure that the input tensors are not overwritten before they are used
|
||||
ggml_set_input(embd);
|
||||
|
||||
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
|
||||
ggml_set_name(position, "position");
|
||||
ggml_set_input(position);
|
||||
|
||||
// wte + wpe
|
||||
struct ggml_tensor * inpL =
|
||||
ggml_add(ctx,
|
||||
ggml_get_rows(ctx, model.wte, embd),
|
||||
ggml_get_rows(ctx, model.wpe, position));
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
struct ggml_tensor * cur;
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
cur = ggml_norm(ctx, inpL, hparams.eps);
|
||||
|
||||
// cur = ln_1_g*cur + ln_1_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
cur,
|
||||
model.layers[il].ln_1_g),
|
||||
model.layers[il].ln_1_b);
|
||||
}
|
||||
|
||||
// attn
|
||||
// [2304, 768] - model.layers[il].c_attn_attn_w
|
||||
// [2304, 1] - model.layers[il].c_attn_attn_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [2304, N] - cur (out)
|
||||
//
|
||||
// cur = attn_w*cur + attn_b
|
||||
// [2304, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_attn_attn_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
cur,
|
||||
model.layers[il].c_attn_attn_b);
|
||||
}
|
||||
|
||||
// self-attention
|
||||
{
|
||||
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
|
||||
|
||||
// store key and value to memory
|
||||
if (N >= 1) {
|
||||
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
|
||||
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
|
||||
}
|
||||
|
||||
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * Q =
|
||||
ggml_permute(ctx,
|
||||
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
|
||||
// [64, n_past + N, 12]
|
||||
struct ggml_tensor * K =
|
||||
ggml_permute(ctx,
|
||||
ggml_reshape_3d(ctx,
|
||||
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// GG: flash attention
|
||||
//struct ggml_tensor * V =
|
||||
// ggml_cpy(ctx0,
|
||||
// ggml_permute(ctx0,
|
||||
// ggml_reshape_3d(ctx0,
|
||||
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
// n_embd/n_head, n_head, n_past + N),
|
||||
// 1, 2, 0, 3),
|
||||
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
|
||||
|
||||
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
|
||||
|
||||
// K * Q
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
|
||||
|
||||
// KQ_scaled = KQ / sqrt(n_embd/n_head)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_scaled =
|
||||
ggml_scale(ctx,
|
||||
KQ,
|
||||
1.0f/sqrtf(float(n_embd)/n_head));
|
||||
|
||||
// KQ_masked = mask_past(KQ_scaled)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
|
||||
|
||||
// KQ = soft_max(KQ_masked)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
|
||||
|
||||
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
|
||||
// [n_past + N, 64, 12]
|
||||
struct ggml_tensor * V_trans =
|
||||
ggml_cont_3d(ctx,
|
||||
ggml_permute(ctx,
|
||||
ggml_reshape_3d(ctx,
|
||||
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
1, 2, 0, 3),
|
||||
n_past + N, n_embd/n_head, n_head);
|
||||
|
||||
// KQV = transpose(V) * KQ_soft_max
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
|
||||
|
||||
// KQV_merged = KQV.permute(0, 2, 1, 3)
|
||||
// [64, 12, N]
|
||||
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
|
||||
|
||||
// cur = KQV_merged.contiguous().view(n_embd, N)
|
||||
// [768, N]
|
||||
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
|
||||
}
|
||||
|
||||
// projection
|
||||
// [ 768, 768] - model.layers[il].c_attn_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_attn_proj_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_attn_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
cur,
|
||||
model.layers[il].c_attn_proj_b);
|
||||
}
|
||||
|
||||
// add the input
|
||||
cur = ggml_add(ctx, cur, inpL);
|
||||
|
||||
struct ggml_tensor * inpFF = cur;
|
||||
|
||||
// feed-forward network
|
||||
{
|
||||
// norm
|
||||
{
|
||||
cur = ggml_norm(ctx, inpFF, hparams.eps);
|
||||
|
||||
// cur = ln_2_g*cur + ln_2_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
cur,
|
||||
model.layers[il].ln_2_g),
|
||||
model.layers[il].ln_2_b);
|
||||
}
|
||||
|
||||
// fully connected
|
||||
// [3072, 768] - model.layers[il].c_mlp_fc_w
|
||||
// [3072, 1] - model.layers[il].c_mlp_fc_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [3072, N] - cur (out)
|
||||
//
|
||||
// cur = fc_w*cur + fc_b
|
||||
// [3072, N]
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_mlp_fc_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
cur,
|
||||
model.layers[il].c_mlp_fc_b);
|
||||
|
||||
// GELU activation
|
||||
// [3072, N]
|
||||
cur = ggml_gelu(ctx, cur);
|
||||
|
||||
// projection
|
||||
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_mlp_proj_b
|
||||
// [3072, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
cur = ggml_mul_mat(ctx,
|
||||
model.layers[il].c_mlp_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx,
|
||||
cur,
|
||||
model.layers[il].c_mlp_proj_b);
|
||||
}
|
||||
|
||||
// input for next layer
|
||||
inpL = ggml_add(ctx, cur, inpFF);
|
||||
}
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
inpL = ggml_norm(ctx, inpL, hparams.eps);
|
||||
|
||||
// inpL = ln_f_g*inpL + ln_f_b
|
||||
// [ 768, N]
|
||||
inpL = ggml_add(ctx,
|
||||
ggml_mul(ctx,
|
||||
inpL,
|
||||
model.ln_f_g),
|
||||
model.ln_f_b);
|
||||
}
|
||||
|
||||
// inpL = WTE * inpL
|
||||
// [ 768, 50257] - model.lm_head
|
||||
// [ 768, N] - inpL
|
||||
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
|
||||
ggml_set_name(inpL, "logits");
|
||||
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
|
||||
ggml_set_output(inpL);
|
||||
|
||||
// logits -> probs
|
||||
//inpL = ggml_soft_max(ctx0, inpL);
|
||||
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
|
||||
ggml_free(ctx);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
// - model: the model
|
||||
// - allocr: ggml_gallocr to use to allocate the compute buffer
|
||||
// - n_threads: number of threads to use
|
||||
// - n_past: the context size so far
|
||||
// - embd_inp: the embeddings of the tokens in the context
|
||||
// - embd_w: the predicted logits for the next token
|
||||
//
|
||||
bool gpt2_eval(
|
||||
const gpt2_model & model,
|
||||
ggml_gallocr_t allocr,
|
||||
const int n_threads,
|
||||
const int n_past,
|
||||
const std::vector<gpt_vocab::id> & embd_inp,
|
||||
std::vector<float> & embd_w) {
|
||||
const int N = embd_inp.size();
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
|
||||
|
||||
// allocate the graph tensors
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
|
||||
// set the graph inputs
|
||||
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
|
||||
ggml_backend_tensor_set(embd, embd_inp.data(), 0, N*ggml_element_size(embd));
|
||||
|
||||
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
|
||||
for (int i = 0; i < N; ++i) {
|
||||
int32_t v = n_past + i;
|
||||
ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v));
|
||||
}
|
||||
|
||||
// set backend options
|
||||
if (ggml_backend_is_cpu(model.backend)) {
|
||||
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
|
||||
}
|
||||
|
||||
// run the computation
|
||||
ggml_backend_graph_compute(model.backend, gf);
|
||||
|
||||
//if (n_past%100 == 0) {
|
||||
// ggml_graph_print (&gf);
|
||||
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
|
||||
//}
|
||||
|
||||
// get the graph outputs
|
||||
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
|
||||
|
||||
//embd_w.resize(n_vocab*N);
|
||||
//ggml_backend_tensor_get(logits, embd_w.data(), 0, sizeof(float)*n_vocab*N);
|
||||
|
||||
// return result just for the last token
|
||||
embd_w.resize(n_vocab);
|
||||
ggml_backend_tensor_get(logits, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_time_init();
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
gpt_params params;
|
||||
params.model = "models/gpt-2-117M/ggml-model.bin";
|
||||
|
||||
if (gpt_params_parse(argc, argv, params) == false) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.seed < 0) {
|
||||
params.seed = time(NULL);
|
||||
}
|
||||
|
||||
printf("%s: seed = %d\n", __func__, params.seed);
|
||||
|
||||
std::mt19937 rng(params.seed);
|
||||
if (params.prompt.empty()) {
|
||||
params.prompt = gpt_random_prompt(rng);
|
||||
}
|
||||
|
||||
int64_t t_load_us = 0;
|
||||
|
||||
gpt_vocab vocab;
|
||||
gpt2_model model;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) {
|
||||
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_load_us = ggml_time_us() - t_start_us;
|
||||
|
||||
test_gpt_tokenizer(vocab, params.token_test);
|
||||
}
|
||||
|
||||
ggml_gallocr_t allocr = NULL;
|
||||
// allocate the compute buffer
|
||||
{
|
||||
// create a graph allocator with the backend's default buffer type
|
||||
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
// create the worst case graph for memory usage estimation
|
||||
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
|
||||
int n_past = model.hparams.n_ctx - n_tokens;
|
||||
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
|
||||
|
||||
// pre-allocate the compute buffer for the worst case (optional)
|
||||
ggml_gallocr_reserve(allocr, gf);
|
||||
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
|
||||
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
|
||||
int64_t t_sample_us = 0;
|
||||
int64_t t_predict_us = 0;
|
||||
|
||||
std::vector<float> logits;
|
||||
|
||||
// tokenize the prompt
|
||||
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
|
||||
|
||||
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
|
||||
|
||||
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
|
||||
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
|
||||
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
|
||||
printf("%d ", embd_inp[i]);
|
||||
}
|
||||
printf("\n\n");
|
||||
|
||||
// submit the input prompt token-by-token
|
||||
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
|
||||
std::vector<gpt_vocab::id> embd;
|
||||
|
||||
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
|
||||
// predict
|
||||
if (embd.size() > 0) {
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
|
||||
printf("Failed to predict\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_predict_us += ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
n_past += embd.size();
|
||||
embd.clear();
|
||||
|
||||
if (i >= embd_inp.size()) {
|
||||
// sample next token
|
||||
const int top_k = params.top_k;
|
||||
const float top_p = params.top_p;
|
||||
const float temp = params.temp;
|
||||
|
||||
const int n_vocab = model.hparams.n_vocab;
|
||||
|
||||
gpt_vocab::id id = 0;
|
||||
|
||||
{
|
||||
const int64_t t_start_sample_us = ggml_time_us();
|
||||
|
||||
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
|
||||
|
||||
t_sample_us += ggml_time_us() - t_start_sample_us;
|
||||
}
|
||||
|
||||
// add it to the context
|
||||
embd.push_back(id);
|
||||
} else {
|
||||
// if here, it means we are still processing the input prompt
|
||||
for (size_t k = i; k < embd_inp.size(); k++) {
|
||||
embd.push_back(embd_inp[k]);
|
||||
if (int32_t(embd.size()) >= params.n_batch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
i += embd.size() - 1;
|
||||
}
|
||||
|
||||
// display text
|
||||
for (auto id : embd) {
|
||||
printf("%s", vocab.id_to_token[id].c_str());
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
// end of text token
|
||||
if (!params.ignore_eos && embd.back() == 50256) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n\n");
|
||||
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
|
||||
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
|
||||
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
ggml_free(model.ctx_w);
|
||||
|
||||
ggml_gallocr_free(allocr);
|
||||
ggml_backend_buffer_free(model.buffer_w);
|
||||
ggml_backend_buffer_free(model.buffer_kv);
|
||||
ggml_backend_free(model.backend);
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,840 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
// default hparams (GPT-2 117M)
|
||||
struct gpt2_hparams {
|
||||
int32_t n_vocab = 50257;
|
||||
int32_t n_ctx = 1024;
|
||||
int32_t n_embd = 768;
|
||||
int32_t n_head = 12;
|
||||
int32_t n_layer = 12;
|
||||
int32_t ftype = 1;
|
||||
float eps = 1e-5f;
|
||||
};
|
||||
|
||||
struct gpt2_layer {
|
||||
// normalization
|
||||
struct ggml_tensor * ln_1_g;
|
||||
struct ggml_tensor * ln_1_b;
|
||||
|
||||
struct ggml_tensor * ln_2_g;
|
||||
struct ggml_tensor * ln_2_b;
|
||||
|
||||
// attention
|
||||
struct ggml_tensor * c_attn_attn_w;
|
||||
struct ggml_tensor * c_attn_attn_b;
|
||||
|
||||
struct ggml_tensor * c_attn_proj_w;
|
||||
struct ggml_tensor * c_attn_proj_b;
|
||||
|
||||
// mlp
|
||||
struct ggml_tensor * c_mlp_fc_w;
|
||||
struct ggml_tensor * c_mlp_fc_b;
|
||||
|
||||
struct ggml_tensor * c_mlp_proj_w;
|
||||
struct ggml_tensor * c_mlp_proj_b;
|
||||
};
|
||||
|
||||
struct gpt2_model {
|
||||
gpt2_hparams hparams;
|
||||
|
||||
// normalization
|
||||
struct ggml_tensor * ln_f_g;
|
||||
struct ggml_tensor * ln_f_b;
|
||||
|
||||
struct ggml_tensor * wte; // token embedding
|
||||
struct ggml_tensor * wpe; // position embedding
|
||||
struct ggml_tensor * lm_head; // language model head
|
||||
|
||||
std::vector<gpt2_layer> layers;
|
||||
|
||||
// key + value memory
|
||||
struct ggml_tensor * memory_k;
|
||||
struct ggml_tensor * memory_v;
|
||||
|
||||
//
|
||||
struct ggml_context * ctx_w;
|
||||
std::map<std::string, struct ggml_tensor *> tensors;
|
||||
};
|
||||
|
||||
// load the model's weights from a file
|
||||
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
|
||||
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
fin.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// load hparams
|
||||
{
|
||||
auto & hparams = model.hparams;
|
||||
|
||||
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: ftype = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr = %d\n", __func__, qntvr);
|
||||
|
||||
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
fin.read((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != model.hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
std::vector<char> buf(128);
|
||||
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
fin.read((char *) &len, sizeof(len));
|
||||
|
||||
buf.resize(len);
|
||||
fin.read((char *) buf.data(), len);
|
||||
word.assign(buf.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
|
||||
// in order to save memory and also to speed up the computation
|
||||
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
|
||||
if (wtype == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
|
||||
__func__, fname.c_str(), model.hparams.ftype);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto & ctx = model.ctx_w;
|
||||
|
||||
size_t ctx_size = 0;
|
||||
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
|
||||
|
||||
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe
|
||||
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
|
||||
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
|
||||
|
||||
ctx_size += (6 + 12*n_layer)*512; // object overhead
|
||||
|
||||
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
|
||||
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
|
||||
}
|
||||
|
||||
// create the ggml context
|
||||
{
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ctx_size,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
model.ctx_w = ggml_init(params);
|
||||
if (!model.ctx_w) {
|
||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// prepare memory for the weights
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
model.layers.resize(n_layer);
|
||||
|
||||
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
|
||||
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/ln_f/g"] = model.ln_f_g;
|
||||
model.tensors["model/ln_f/b"] = model.ln_f_b;
|
||||
|
||||
model.tensors["model/wte"] = model.wte;
|
||||
model.tensors["model/wpe"] = model.wpe;
|
||||
model.tensors["model/lm_head"] = model.lm_head;
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = model.layers[i];
|
||||
|
||||
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
|
||||
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
|
||||
|
||||
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
|
||||
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
|
||||
|
||||
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
|
||||
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
// map by name
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
|
||||
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
|
||||
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
|
||||
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
|
||||
}
|
||||
}
|
||||
|
||||
// key + value memory
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
|
||||
const int n_mem = n_layer*n_ctx;
|
||||
const int n_elements = n_embd*n_mem;
|
||||
|
||||
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
|
||||
|
||||
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
|
||||
|
||||
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
|
||||
}
|
||||
|
||||
// load weights
|
||||
{
|
||||
size_t total_size = 0;
|
||||
|
||||
bool has_lm_head = false;
|
||||
|
||||
while (true) {
|
||||
int32_t n_dims;
|
||||
int32_t length;
|
||||
int32_t ttype;
|
||||
|
||||
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
|
||||
if (fin.eof()) {
|
||||
break;
|
||||
}
|
||||
|
||||
int32_t nelements = 1;
|
||||
int32_t ne[2] = { 1, 1 };
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
nelements *= ne[i];
|
||||
}
|
||||
|
||||
std::string name(length, 0);
|
||||
fin.read(&name[0], length);
|
||||
|
||||
if (model.tensors.find(name) == model.tensors.end()) {
|
||||
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tensor = model.tensors[name];
|
||||
if (ggml_nelements(tensor) != nelements) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
|
||||
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
|
||||
return false;
|
||||
}
|
||||
|
||||
// for debugging
|
||||
if (0) {
|
||||
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
const size_t bpe = ggml_type_size(ggml_type(ttype));
|
||||
|
||||
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
|
||||
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
|
||||
return false;
|
||||
}
|
||||
|
||||
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
|
||||
|
||||
// GPT-2 models share the WTE tensor as the LM head
|
||||
if (name == "model/wte" && has_lm_head == false) {
|
||||
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
if (name == "model/lm_head") {
|
||||
has_lm_head = true;
|
||||
}
|
||||
|
||||
total_size += ggml_nbytes(tensor);
|
||||
}
|
||||
|
||||
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
// - model: the model
|
||||
// - n_threads: number of threads to use
|
||||
// - n_past: the context size so far
|
||||
// - embd_inp: the embeddings of the tokens in the context
|
||||
// - embd_w: the predicted logits for the next token
|
||||
//
|
||||
bool gpt2_eval(
|
||||
const gpt2_model & model,
|
||||
const int n_threads,
|
||||
const int n_past,
|
||||
const std::vector<gpt_vocab::id> & embd_inp,
|
||||
std::vector<float> & embd_w,
|
||||
size_t & mem_per_token) {
|
||||
const int N = embd_inp.size();
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_head = hparams.n_head;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
static size_t buf_size = 256u*1024*1024;
|
||||
static void * buf = malloc(buf_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
|
||||
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
||||
|
||||
// reallocate
|
||||
buf_size = buf_size_new;
|
||||
buf = realloc(buf, buf_size);
|
||||
if (buf == nullptr) {
|
||||
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx0 = ggml_init(params);
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
|
||||
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
|
||||
|
||||
struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
|
||||
for (int i = 0; i < N; ++i) {
|
||||
((int32_t *) position->data)[i] = n_past + i;
|
||||
}
|
||||
|
||||
// wte + wpe
|
||||
struct ggml_tensor * inpL =
|
||||
ggml_add(ctx0,
|
||||
ggml_get_rows(ctx0, model.wte, embd),
|
||||
ggml_get_rows(ctx0, model.wpe, position));
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
struct ggml_tensor * cur;
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
cur = ggml_norm(ctx0, inpL, hparams.eps);
|
||||
|
||||
// cur = ln_1_g*cur + ln_1_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_mul(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
|
||||
cur),
|
||||
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
|
||||
}
|
||||
|
||||
// attn
|
||||
// [2304, 768] - model.layers[il].c_attn_attn_w
|
||||
// [2304, 1] - model.layers[il].c_attn_attn_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [2304, N] - cur (out)
|
||||
//
|
||||
// cur = attn_w*cur + attn_b
|
||||
// [2304, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_attn_attn_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// self-attention
|
||||
{
|
||||
struct ggml_tensor * Qcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Kcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
|
||||
struct ggml_tensor * Vcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
|
||||
|
||||
// store key and value to memory
|
||||
if (N >= 1) {
|
||||
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
|
||||
struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
|
||||
}
|
||||
|
||||
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * Q =
|
||||
ggml_permute(ctx0,
|
||||
ggml_cpy(ctx0,
|
||||
Qcur,
|
||||
ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
|
||||
// [64, n_past + N, 12]
|
||||
struct ggml_tensor * K =
|
||||
ggml_permute(ctx0,
|
||||
ggml_reshape_3d(ctx0,
|
||||
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// GG: flash attention
|
||||
//struct ggml_tensor * V =
|
||||
// ggml_cpy(ctx0,
|
||||
// ggml_permute(ctx0,
|
||||
// ggml_reshape_3d(ctx0,
|
||||
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
// n_embd/n_head, n_head, n_past + N),
|
||||
// 1, 2, 0, 3),
|
||||
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
|
||||
|
||||
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
|
||||
|
||||
// K * Q
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
|
||||
|
||||
// KQ_scaled = KQ / sqrt(n_embd/n_head)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
|
||||
|
||||
// KQ_masked = mask_past(KQ_scaled)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
|
||||
|
||||
// KQ = soft_max(KQ_masked)
|
||||
// [n_past + N, N, 12]
|
||||
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
|
||||
|
||||
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
|
||||
// [n_past + N, 64, 12]
|
||||
struct ggml_tensor * V_trans =
|
||||
ggml_cpy(ctx0,
|
||||
ggml_permute(ctx0,
|
||||
ggml_reshape_3d(ctx0,
|
||||
ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
1, 2, 0, 3),
|
||||
ggml_new_tensor_3d(ctx0, model.memory_v->type, n_past + N, n_embd/n_head, n_head));
|
||||
|
||||
// KQV = transpose(V) * KQ_soft_max
|
||||
// [64, N, 12]
|
||||
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
|
||||
|
||||
// KQV_merged = KQV.permute(0, 2, 1, 3)
|
||||
// [64, 12, N]
|
||||
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
|
||||
|
||||
// cur = KQV_merged.contiguous().view(n_embd, N)
|
||||
// [768, N]
|
||||
cur = ggml_cpy(ctx0,
|
||||
KQV_merged,
|
||||
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
|
||||
}
|
||||
|
||||
// projection
|
||||
// [ 768, 768] - model.layers[il].c_attn_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_attn_proj_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
{
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_attn_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// add the input
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
|
||||
struct ggml_tensor * inpFF = cur;
|
||||
|
||||
// feed-forward network
|
||||
{
|
||||
// norm
|
||||
{
|
||||
cur = ggml_norm(ctx0, inpFF, hparams.eps);
|
||||
|
||||
// cur = ln_2_g*cur + ln_2_b
|
||||
// [ 768, N]
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_mul(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].ln_2_g, cur),
|
||||
cur),
|
||||
ggml_repeat(ctx0, model.layers[il].ln_2_b, cur));
|
||||
}
|
||||
|
||||
// fully connected
|
||||
// [3072, 768] - model.layers[il].c_mlp_fc_w
|
||||
// [3072, 1] - model.layers[il].c_mlp_fc_b
|
||||
// [ 768, N] - cur (in)
|
||||
// [3072, N] - cur (out)
|
||||
//
|
||||
// cur = fc_w*cur + fc_b
|
||||
// [3072, N]
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_mlp_fc_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
|
||||
cur);
|
||||
|
||||
// GELU activation
|
||||
// [3072, N]
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
|
||||
// projection
|
||||
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
|
||||
// [ 768, 1] - model.layers[il].c_mlp_proj_b
|
||||
// [3072, N] - cur (in)
|
||||
// [ 768, N] - cur (out)
|
||||
//
|
||||
// cur = proj_w*cur + proj_b
|
||||
// [768, N]
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_mlp_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// input for next layer
|
||||
inpL = ggml_add(ctx0, cur, inpFF);
|
||||
}
|
||||
|
||||
// norm
|
||||
{
|
||||
// [ 768, N]
|
||||
inpL = ggml_norm(ctx0, inpL, hparams.eps);
|
||||
|
||||
// inpL = ln_f_g*inpL + ln_f_b
|
||||
// [ 768, N]
|
||||
inpL = ggml_add(ctx0,
|
||||
ggml_mul(ctx0,
|
||||
ggml_repeat(ctx0, model.ln_f_g, inpL),
|
||||
inpL),
|
||||
ggml_repeat(ctx0, model.ln_f_b, inpL));
|
||||
}
|
||||
|
||||
// inpL = WTE * inpL
|
||||
// [ 768, 50257] - model.lm_head
|
||||
// [ 768, N] - inpL
|
||||
inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
|
||||
|
||||
// logits -> probs
|
||||
//inpL = ggml_soft_max_inplace(ctx0, inpL);
|
||||
|
||||
// run the computation
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
|
||||
|
||||
//if (n_past%100 == 0) {
|
||||
// ggml_graph_print (&gf);
|
||||
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
|
||||
//}
|
||||
|
||||
//embd_w.resize(n_vocab*N);
|
||||
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
|
||||
|
||||
// return result just for the last token
|
||||
embd_w.resize(n_vocab);
|
||||
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
|
||||
|
||||
if (mem_per_token == 0) {
|
||||
mem_per_token = ggml_used_mem(ctx0)/N;
|
||||
}
|
||||
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
|
||||
|
||||
ggml_free(ctx0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_time_init();
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
gpt_params params;
|
||||
params.model = "models/gpt-2-117M/ggml-model.bin";
|
||||
|
||||
if (gpt_params_parse(argc, argv, params) == false) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.seed < 0) {
|
||||
params.seed = time(NULL);
|
||||
}
|
||||
|
||||
printf("%s: seed = %d\n", __func__, params.seed);
|
||||
|
||||
std::mt19937 rng(params.seed);
|
||||
if (params.prompt.empty()) {
|
||||
params.prompt = gpt_random_prompt(rng);
|
||||
}
|
||||
|
||||
int64_t t_load_us = 0;
|
||||
|
||||
gpt_vocab vocab;
|
||||
gpt2_model model;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_model_load(params.model, model, vocab)) {
|
||||
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_load_us = ggml_time_us() - t_start_us;
|
||||
|
||||
test_gpt_tokenizer(vocab, params.token_test);
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
|
||||
int64_t t_sample_us = 0;
|
||||
int64_t t_predict_us = 0;
|
||||
|
||||
std::vector<float> logits;
|
||||
|
||||
// tokenize the prompt
|
||||
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
|
||||
|
||||
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
|
||||
|
||||
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
|
||||
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
|
||||
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
|
||||
printf("%d ", embd_inp[i]);
|
||||
}
|
||||
printf("\n\n");
|
||||
|
||||
// submit the input prompt token-by-token
|
||||
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
|
||||
std::vector<gpt_vocab::id> embd;
|
||||
|
||||
// determine the required inference memory per token:
|
||||
size_t mem_per_token = 0;
|
||||
gpt2_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
|
||||
|
||||
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
|
||||
// predict
|
||||
if (embd.size() > 0) {
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
|
||||
printf("Failed to predict\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_predict_us += ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
n_past += embd.size();
|
||||
embd.clear();
|
||||
|
||||
if (i >= embd_inp.size()) {
|
||||
// sample next token
|
||||
const int top_k = params.top_k;
|
||||
const float top_p = params.top_p;
|
||||
const float temp = params.temp;
|
||||
|
||||
const int n_vocab = model.hparams.n_vocab;
|
||||
|
||||
gpt_vocab::id id = 0;
|
||||
|
||||
{
|
||||
const int64_t t_start_sample_us = ggml_time_us();
|
||||
|
||||
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
|
||||
|
||||
t_sample_us += ggml_time_us() - t_start_sample_us;
|
||||
}
|
||||
|
||||
// add it to the context
|
||||
embd.push_back(id);
|
||||
} else {
|
||||
// if here, it means we are still processing the input prompt
|
||||
for (size_t k = i; k < embd_inp.size(); k++) {
|
||||
embd.push_back(embd_inp[k]);
|
||||
if (int32_t(embd.size()) >= params.n_batch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
i += embd.size() - 1;
|
||||
}
|
||||
|
||||
// display text
|
||||
for (auto id : embd) {
|
||||
printf("%s", vocab.id_to_token[id].c_str());
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
// end of text token
|
||||
if (embd.back() == 50256) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n\n");
|
||||
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
|
||||
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
|
||||
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
|
||||
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
ggml_free(model.ctx_w);
|
||||
|
||||
return 0;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,184 @@
|
||||
#include "ggml.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <regex>
|
||||
|
||||
// default hparams (GPT-2 117M)
|
||||
struct gpt2_hparams {
|
||||
int32_t n_vocab = 50257;
|
||||
int32_t n_ctx = 1024;
|
||||
int32_t n_embd = 768;
|
||||
int32_t n_head = 12;
|
||||
int32_t n_layer = 12;
|
||||
int32_t ftype = 1;
|
||||
};
|
||||
|
||||
// quantize a model
|
||||
bool gpt2_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
|
||||
gpt_vocab vocab;
|
||||
|
||||
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
|
||||
|
||||
auto finp = std::ifstream(fname_inp, std::ios::binary);
|
||||
if (!finp) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto fout = std::ofstream(fname_out, std::ios::binary);
|
||||
if (!fout) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
finp.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
fout.write((char *) &magic, sizeof(magic));
|
||||
}
|
||||
|
||||
gpt2_hparams hparams;
|
||||
|
||||
// load hparams
|
||||
{
|
||||
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
|
||||
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
|
||||
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
|
||||
|
||||
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
finp.read ((char *) &n_vocab, sizeof(n_vocab));
|
||||
fout.write((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
finp.read ((char *) &len, sizeof(len));
|
||||
fout.write((char *) &len, sizeof(len));
|
||||
|
||||
word.resize(len);
|
||||
finp.read ((char *) word.data(), len);
|
||||
fout.write((char *) word.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// regexes of tensor names to be quantized
|
||||
const std::vector<std::string> to_quant = {
|
||||
"model/wte",
|
||||
"model/lm_head",
|
||||
"model/h.*/attn/c_attn/w",
|
||||
"model/h.*/attn/c_proj/w",
|
||||
"model/h.*/mlp/c_fc/w",
|
||||
"model/h.*/mlp/c_proj/w",
|
||||
};
|
||||
|
||||
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
|
||||
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
finp.close();
|
||||
fout.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// usage:
|
||||
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
|
||||
//
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc != 4) {
|
||||
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
|
||||
ggml_print_ftypes(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// needed to initialize f16 tables
|
||||
{
|
||||
struct ggml_init_params params = { 0, NULL, false };
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
const std::string fname_inp = argv[1];
|
||||
const std::string fname_out = argv[2];
|
||||
|
||||
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
int64_t t_quantize_us = 0;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gpt2_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
|
||||
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_quantize_us = ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n");
|
||||
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#
|
||||
# gpt-j
|
||||
|
||||
set(TEST_TARGET gpt-j)
|
||||
add_executable(${TEST_TARGET} main.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
#
|
||||
# gpt-j-quantize
|
||||
|
||||
set(TEST_TARGET gpt-j-quantize)
|
||||
add_executable(${TEST_TARGET} quantize.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
@@ -0,0 +1,239 @@
|
||||
# gpt-j
|
||||
|
||||
Local GPT-J inference on your computer using C/C++
|
||||
|
||||
No video card required. You just need to have 16 GB of RAM.
|
||||
|
||||
## Motivation
|
||||
|
||||
The GPT-J 6B model is the open-source alternative to OpenAI's GPT-3. It's basically a neural network that allows you to
|
||||
generate coherent, human-like text given a certain context (prompt).
|
||||
|
||||
The GPT-J model is quite big - the compact version of the model uses 16-bit floating point representation of the weights
|
||||
and is still 12 GB big. This means that in order to run inference on your computer, you would need to have a video card
|
||||
with at least 12 GB of video RAM. Alternatively, you can try to run the python implementations on the CPU, but that
|
||||
would probably not be very efficient as they are primarily optimized for running on a GPU (or at least this is my guess -
|
||||
I don't have much experience with python).
|
||||
|
||||
I wanted to try and run the model on my MacBook, so I decided to implement the model inference from scratch using my own
|
||||
custom build tensor library. The tensor library (called [ggml](https://github.com/ggerganov/ggml), written in C) is in
|
||||
early development stage, but it already allows me to run the GPT-J model.
|
||||
|
||||
On my 32GB MacBook M1 Pro, I achieve an inference speed of about `125 ms/token` or about ~6 words per second (1 word
|
||||
typically consists of 1 or 2 tokens).
|
||||
|
||||
Here is a sample run with prompt `int main(int argc, char ** argv) {`:
|
||||
|
||||
```bash
|
||||
$ time ./bin/gpt-j -p "int main(int argc, char ** argv) {"
|
||||
|
||||
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
|
||||
gptj_model_load: n_vocab = 50400
|
||||
gptj_model_load: n_ctx = 2048
|
||||
gptj_model_load: n_embd = 4096
|
||||
gptj_model_load: n_head = 16
|
||||
gptj_model_load: n_layer = 28
|
||||
gptj_model_load: n_rot = 64
|
||||
gptj_model_load: f16 = 1
|
||||
gptj_model_load: ggml ctx size = 13334.86 MB
|
||||
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
|
||||
gptj_model_load: ................................... done
|
||||
gptj_model_load: model size = 11542.79 MB / num tensors = 285
|
||||
main: number of tokens in prompt = 13
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
|
||||
{
|
||||
struct sockaddr_in addr;
|
||||
int addrlen;
|
||||
char * ip = "192.168.1.4";
|
||||
int i;
|
||||
|
||||
if ( (addrlen = sizeof(addr)) == -1 )
|
||||
return -1;
|
||||
|
||||
for (i = 0; i < 10; ++i) {
|
||||
addr.sin_family = AF_INET;
|
||||
addr.sin_addr.s_addr = inet_addr(ip);
|
||||
|
||||
main: mem per token = 16430420 bytes
|
||||
main: load time = 6211.48 ms
|
||||
main: sample time = 13.74 ms
|
||||
main: predict time = 26420.34 ms / 124.62 ms per token
|
||||
main: total time = 33035.37 ms
|
||||
|
||||
real 0m33.171s
|
||||
user 3m32.269s
|
||||
sys 0m3.686s
|
||||
|
||||
$
|
||||
```
|
||||
|
||||
It took ~6.2 seconds to load the model to memory. After that, it took ~26.4 seconds to generate 200 tokens of what
|
||||
looks like to be the beginning of a networking program in C. Pretty cool!
|
||||
|
||||
Here is another run, just for fun:
|
||||
|
||||
```bash
|
||||
time ./bin/gpt-j -n 500 -t 8 -p "Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
|
||||
"
|
||||
|
||||
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
|
||||
gptj_model_load: n_vocab = 50400
|
||||
gptj_model_load: n_ctx = 2048
|
||||
gptj_model_load: n_embd = 4096
|
||||
gptj_model_load: n_head = 16
|
||||
gptj_model_load: n_layer = 28
|
||||
gptj_model_load: n_rot = 64
|
||||
gptj_model_load: f16 = 1
|
||||
gptj_model_load: ggml ctx size = 13334.86 MB
|
||||
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
|
||||
gptj_model_load: ................................... done
|
||||
gptj_model_load: model size = 11542.79 MB / num tensors = 285
|
||||
main: number of tokens in prompt = 24
|
||||
|
||||
Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
|
||||
|
||||
I've inherited a team with some very strange and un-documented practices, one of them is that they use an old custom
|
||||
application with a very slow tech stack written in Python that the team doesn't want to touch but also doesn't want to
|
||||
throw away as it has some "legacy" code in it.
|
||||
|
||||
The problem is, the tech stack is very very slow.
|
||||
|
||||
They have a single web server on a VM that is slow.
|
||||
The server is a little bit busy (not very busy though) and they have a lot of processes (30+ that are constantly being
|
||||
spawned by the application)
|
||||
They have an application that is single threaded and was written in Python and the team don't want to touch this, and
|
||||
the application is very slow.
|
||||
|
||||
My task as a new member of the team is to fix this.
|
||||
|
||||
I'm a senior dev on the team (3 years on the project) and have been told that I will take the lead on this task. I know
|
||||
next to nothing about Python. So here is what I have so far.
|
||||
|
||||
What I have done is I've been trying to debug the processes with the "ps" command. This way I can see what is running
|
||||
and where. From what I see, the application spawns 10 processes a minute and some of them are used for nothing.
|
||||
|
||||
I have also started to look for the code. The application source is not in GitHub or any other repository, it is only on
|
||||
our internal GitLab.
|
||||
|
||||
What I've found so far:
|
||||
|
||||
The application uses a custom SQLAlchemy implementation to interact with the data. I've looked at the source, it looks
|
||||
like an object cache or something like that. But from what I've seen, the cache gets full every 20 minutes and then gets
|
||||
cleared with a special command.
|
||||
|
||||
Another strange thing is that the application creates a file for every entry in the database (even if the entry already
|
||||
exists). I've looked at the file to see if it contains something, but it seems to be a JSON file with lots of records.
|
||||
|
||||
The other strange thing is that I can only find the database tables in the GitLab repository and not the code. So I
|
||||
can't really understand how the application is supposed to interact with the database.
|
||||
|
||||
I also found a "log" directory, but the code is encrypted with AES. From what I've found, it is in
|
||||
|
||||
main: mem per token = 16430420 bytes
|
||||
main: load time = 3900.10 ms
|
||||
main: sample time = 32.58 ms
|
||||
main: predict time = 68049.91 ms / 130.11 ms per token
|
||||
main: total time = 73020.05 ms
|
||||
|
||||
real 1m13.156s
|
||||
user 9m1.328s
|
||||
sys. 0m7.103s
|
||||
```
|
||||
|
||||
## Implementation details
|
||||
|
||||
The high level implementation of the model is contained in the [main.cpp](main.cpp) file. The core computations are
|
||||
performed by the [ggml](https://github.com/ggerganov/ggml/blob/master/include/ggml.h) library.
|
||||
|
||||
|
||||
#### Matrix multiplication
|
||||
|
||||
The most performance critical part of the implementation is of course the matrix multiplication routine. 99% of the time
|
||||
is spent here, so it was important to optimize this as much as possible.
|
||||
|
||||
On Arm64, I utilize the 128-bit NEON intrinsics for 16-bit floating point operations:
|
||||
|
||||
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/src/ggml.c#L187-L243
|
||||
|
||||
These instructions allow each core to operate simultaneously on 64 16-bit floats. I'm no expert in SIMD, but after quite
|
||||
some trials this was the most efficient code for dot product of a row and column that I could come up with. Combined
|
||||
with the parallel computation on 8 CPU threads, I believe I'm close to the maximum performance that one could possibly
|
||||
get on the M1 CPU. Still, I'm curious to know if there is a more efficient way to implement this.
|
||||
|
||||
|
||||
#### Attempt to use the M1 GPU
|
||||
|
||||
One interesting property of the GPT-J transformer architecture is that it allows you to perform part of the inference in
|
||||
parallel - i.e. the Feed-forward network can be computed in parallel to the Self-attention layer:
|
||||
|
||||
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/examples/gpt-j/main.cpp#L507-L531
|
||||
|
||||
So I thought why not try and bring in the M1 GPU to compute half of the neural network in parallel to the CPU and
|
||||
potentially gain some extra performance. Thanks to the M1's shared memory model, it was relatively easy to offload part
|
||||
of the computation to the GPU using Apple's [Metal Performance
|
||||
Shaders](https://developer.apple.com/documentation/metalperformanceshaders). The GPU shares the host memory, so there is
|
||||
no need to copy the data back and forth as you would normally do with Cuda or OpenCL. The weight matrices are directly
|
||||
available to be used by the GPU.
|
||||
|
||||
However, to my surprise, using MPS together with the CPU did not lead to any performance improvement at all. My
|
||||
conclusion was that the 8-thread NEON CPU computation is already saturating the memory bandwidth of the M1 and since
|
||||
the CPU and the GPU on the MacBook are sharing that bandwidth, it does not help to offload the computation to the GPU.
|
||||
Another observation was that the MPS GPU matrix multiplication using 16-bit floats had the same performance as the
|
||||
8-thread NEON CPU implementation. Again, I explain this with a saturated memory channel. But of course, my explanation
|
||||
could be totally wrong and somehow the implementation wasn't utilizing the resources correctly.
|
||||
|
||||
In the end, I decided to not use MPS or the GPU all together.
|
||||
|
||||
### Zero memory allocations
|
||||
|
||||
Another property of my implementation is that it does not perform any memory allocations once the model is loaded into
|
||||
memory. All required memory is allocated at the start of the program with a single `malloc` (technically 2 calls, but
|
||||
that is not important).
|
||||
|
||||
## Usage
|
||||
|
||||
If you want to give this a try and you are on Linux or Mac OS, simply follow these instructions:
|
||||
|
||||
```bash
|
||||
# Download the ggml-compatible GPT-J 6B model (requires 12GB disk space)
|
||||
../examples/gpt-j/download-ggml-model.sh 6B
|
||||
|
||||
# Run the inference (requires 16GB of CPU RAM)
|
||||
./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin -p "This is an example"
|
||||
|
||||
# Input prompt through pipe and run the inference.
|
||||
echo "This is an example" > prompt.txt
|
||||
cat prompt.txt | ./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin
|
||||
```
|
||||
|
||||
To run the `gpt-j` tool, you need the 12GB `ggml-model.bin` file which contains the GPT-J model in
|
||||
[ggml](https://github.com/ggerganov/ggml) compatible format. In the instructions above, the binary file
|
||||
is downloaded from my repository on Hugging Face using the [download-ggml-model.sh](download-ggml-model.sh) script.
|
||||
You can also, download the file manually from this link:
|
||||
|
||||
https://huggingface.co/ggerganov/ggml/tree/main
|
||||
|
||||
---
|
||||
|
||||
Alternatively, if you don't want to download the 12GB ggml model file, you can perform the conversion yourself using
|
||||
python.
|
||||
|
||||
First, you need to download the full GPT-J model from here: https://huggingface.co/EleutherAI/gpt-j-6B
|
||||
|
||||
Note that the full model is quite big - about 72 GB. After you download it, you need to convert it to ggml format using
|
||||
the [convert-h5-to-ggml.py](convert-h5-to-ggml.py) script. This will generate the `ggml-model.bin` file, which you can
|
||||
then use with the `gpt-j` program.
|
||||
|
||||
|
||||
## GPT-2
|
||||
|
||||
I also implemented a tool for CPU inference using the smaller GPT-2 models. They have worse quality compared to GPT-J,
|
||||
but are much faster to execute.
|
||||
|
||||
For example, the Small GPT-2 model is only 240 MB big and the inference speed on my MacBook is about 200 tokens/sec.
|
||||
|
||||
For more details, checkout the GPT-2 example here: [gpt-2](https://github.com/ggerganov/ggml/tree/master/examples/gpt-2)
|
||||
@@ -0,0 +1,173 @@
|
||||
# Convert GPT-J-6B h5 transformer model to ggml format
|
||||
#
|
||||
# Load the model using GPTJForCausalLM.
|
||||
# Iterate over all variables and write them to a binary file.
|
||||
#
|
||||
# For each variable, write the following:
|
||||
# - Number of dimensions (int)
|
||||
# - Name length (int)
|
||||
# - Dimensions (int[n_dims])
|
||||
# - Name (char[name_length])
|
||||
# - Data (float[n_dims])
|
||||
#
|
||||
# By default, the bigger matrices are converted to 16-bit floats.
|
||||
# This can be disabled by adding the "use-f32" CLI argument.
|
||||
#
|
||||
# At the start of the ggml file we write the model parameters
|
||||
# and vocabulary.
|
||||
#
|
||||
|
||||
import sys
|
||||
import struct
|
||||
import json
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
from transformers import GPTJForCausalLM
|
||||
|
||||
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
|
||||
def bytes_to_unicode():
|
||||
"""
|
||||
Returns list of utf-8 byte and a corresponding list of unicode strings.
|
||||
The reversible bpe codes work on unicode strings.
|
||||
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
|
||||
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
|
||||
This is a signficant percentage of your normal, say, 32K bpe vocab.
|
||||
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
|
||||
And avoids mapping to whitespace/control characters the bpe code barfs on.
|
||||
"""
|
||||
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
|
||||
cs = bs[:]
|
||||
n = 0
|
||||
for b in range(2**8):
|
||||
if b not in bs:
|
||||
bs.append(b)
|
||||
cs.append(2**8+n)
|
||||
n += 1
|
||||
cs = [chr(n) for n in cs]
|
||||
return dict(zip(bs, cs))
|
||||
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
|
||||
print(" ftype == 0 -> float32")
|
||||
print(" ftype == 1 -> float16")
|
||||
sys.exit(1)
|
||||
|
||||
# output in the same directory as the model
|
||||
dir_model = sys.argv[1]
|
||||
fname_out = sys.argv[1] + "/ggml-model.bin"
|
||||
|
||||
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
|
||||
encoder = json.load(f)
|
||||
|
||||
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
|
||||
encoder_added = json.load(f)
|
||||
|
||||
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
|
||||
hparams = json.load(f)
|
||||
|
||||
# possible data types
|
||||
# ftype == 0 -> float32
|
||||
# ftype == 1 -> float16
|
||||
#
|
||||
# map from ftype to string
|
||||
ftype_str = ["f32", "f16"]
|
||||
|
||||
ftype = 1
|
||||
if len(sys.argv) > 2:
|
||||
ftype = int(sys.argv[2])
|
||||
if ftype < 0 or ftype > 1:
|
||||
print("Invalid ftype: " + str(ftype))
|
||||
sys.exit(1)
|
||||
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
|
||||
|
||||
|
||||
model = GPTJForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
|
||||
#print (model)
|
||||
|
||||
list_vars = model.state_dict()
|
||||
#print (list_vars)
|
||||
|
||||
fout = open(fname_out, "wb")
|
||||
|
||||
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
|
||||
fout.write(struct.pack("i", hparams["vocab_size"]))
|
||||
fout.write(struct.pack("i", hparams["n_positions"]))
|
||||
fout.write(struct.pack("i", hparams["n_embd"]))
|
||||
fout.write(struct.pack("i", hparams["n_head"]))
|
||||
fout.write(struct.pack("i", hparams["n_layer"]))
|
||||
fout.write(struct.pack("i", hparams["rotary_dim"]))
|
||||
fout.write(struct.pack("i", ftype))
|
||||
|
||||
byte_encoder = bytes_to_unicode()
|
||||
byte_decoder = {v:k for k, v in byte_encoder.items()}
|
||||
|
||||
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
|
||||
|
||||
for key in encoder:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for key in encoder_added:
|
||||
text = bytearray([byte_decoder[c] for c in key])
|
||||
fout.write(struct.pack("i", len(text)))
|
||||
fout.write(text)
|
||||
|
||||
for name in list_vars.keys():
|
||||
data = list_vars[name].squeeze().numpy()
|
||||
print("Processing variable: " + name + " with shape: ", data.shape)
|
||||
|
||||
# we don't need these
|
||||
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
|
||||
print(" Skipping variable: " + name)
|
||||
continue
|
||||
|
||||
n_dims = len(data.shape);
|
||||
|
||||
# ftype == 0 -> float32, ftype == 1 -> float16
|
||||
ftype_cur = 0;
|
||||
if ftype != 0:
|
||||
if name[-7:] == ".weight" and n_dims == 2:
|
||||
print(" Converting to float16")
|
||||
data = data.astype(np.float16)
|
||||
ftype_cur = 1
|
||||
else:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
else:
|
||||
if data.dtype != np.float32:
|
||||
print(" Converting to float32")
|
||||
data = data.astype(np.float32)
|
||||
ftype_cur = 0
|
||||
|
||||
# for efficiency - transpose these matrices:
|
||||
# (note - with latest ggml this is no longer more efficient, so disabling it)
|
||||
# "transformer.h.*.mlp.fc_in.weight"
|
||||
# "transformer.h.*.attn.out_proj.weight"
|
||||
# "transformer.h.*.attn.q_proj.weight"
|
||||
# "transformer.h.*.attn.k_proj.weight"
|
||||
# "transformer.h.*.attn.v_proj.weight"
|
||||
#if name.endswith(".mlp.fc_in.weight") or \
|
||||
# name.endswith(".attn.out_proj.weight") or \
|
||||
# name.endswith(".attn.q_proj.weight") or \
|
||||
# name.endswith(".attn.k_proj.weight") or \
|
||||
# name.endswith(".attn.v_proj.weight"):
|
||||
# print(" Transposing")
|
||||
# data = data.transpose()
|
||||
|
||||
# header
|
||||
str = name.encode('utf-8')
|
||||
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
|
||||
for i in range(n_dims):
|
||||
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
|
||||
fout.write(str);
|
||||
|
||||
# data
|
||||
data.tofile(fout)
|
||||
|
||||
fout.close()
|
||||
|
||||
print("Done. Output file: " + fname_out)
|
||||
print("")
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script downloads GPT-J model files that have already been converted to ggml format.
|
||||
# This way you don't have to convert them yourself.
|
||||
#
|
||||
# If you want to download the original GPT-J model files, use the "download-model.sh" script instead.
|
||||
|
||||
#src="https://ggml.ggerganov.com"
|
||||
#pfx="ggml-model-gpt-j"
|
||||
|
||||
src="https://huggingface.co/ggerganov/ggml"
|
||||
pfx="resolve/main/ggml-model-gpt-j"
|
||||
|
||||
ggml_path=$(dirname $(realpath $0))
|
||||
|
||||
# GPT-J models
|
||||
models=( "6B" )
|
||||
|
||||
# list available models
|
||||
function list_models {
|
||||
printf "\n"
|
||||
printf " Available models:"
|
||||
for model in "${models[@]}"; do
|
||||
printf " $model"
|
||||
done
|
||||
printf "\n\n"
|
||||
}
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
printf "Usage: $0 <model>\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
model=$1
|
||||
|
||||
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
|
||||
printf "Invalid model: $model\n"
|
||||
list_models
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# download ggml model
|
||||
|
||||
printf "Downloading ggml model $model ...\n"
|
||||
|
||||
mkdir -p models/gpt-j-$model
|
||||
|
||||
if [ -x "$(command -v wget)" ]; then
|
||||
wget --quiet --show-progress -O models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
|
||||
elif [ -x "$(command -v curl)" ]; then
|
||||
curl -L --output models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
|
||||
else
|
||||
printf "Either wget or curl is required to download models.\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
printf "Failed to download ggml model $model \n"
|
||||
printf "Please try again later or download the original GPT-J model files and convert them yourself.\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "Done! Model '$model' saved in 'models/gpt-j-$model/ggml-model.bin'\n"
|
||||
printf "You can now use it like this:\n\n"
|
||||
printf " $ ./bin/gpt-j -m models/gpt-j-$model/ggml-model.bin -p \"This is an example\"\n"
|
||||
printf "\n"
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
printf "To obtain the GPT-J 6B model files, please visit: https://huggingface.co/EleutherAI/gpt-j-6B\n\n"
|
||||
|
||||
printf "The model is very big. For example, the reposirory above is 72GB in size.\n"
|
||||
printf "If you are sure that you want to clone it, simply run the following command:\n\n"
|
||||
|
||||
printf " $ git clone https://huggingface.co/EleutherAI/gpt-j-6B models/gpt-j-6B\n\n"
|
||||
|
||||
printf "Alternatively, use the 'download-ggml-model.sh' script to download a 12GB ggml version of the model.\n"
|
||||
printf "This version is enough to run inference using the ggml library.\n\n"
|
||||
@@ -0,0 +1,755 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
|
||||
// default hparams (GPT-J 6B)
|
||||
struct gptj_hparams {
|
||||
int32_t n_vocab = 50400;
|
||||
int32_t n_ctx = 2048;
|
||||
int32_t n_embd = 4096;
|
||||
int32_t n_head = 16;
|
||||
int32_t n_layer = 28;
|
||||
int32_t n_rot = 64;
|
||||
int32_t ftype = 1;
|
||||
float eps = 1e-5f;
|
||||
};
|
||||
|
||||
struct gptj_layer {
|
||||
// normalization
|
||||
struct ggml_tensor * ln_1_g;
|
||||
struct ggml_tensor * ln_1_b;
|
||||
|
||||
// attention
|
||||
struct ggml_tensor * c_attn_q_proj_w;
|
||||
struct ggml_tensor * c_attn_k_proj_w;
|
||||
struct ggml_tensor * c_attn_v_proj_w;
|
||||
|
||||
struct ggml_tensor * c_attn_proj_w;
|
||||
|
||||
// ff
|
||||
struct ggml_tensor * c_mlp_fc_w;
|
||||
struct ggml_tensor * c_mlp_fc_b;
|
||||
|
||||
struct ggml_tensor * c_mlp_proj_w;
|
||||
struct ggml_tensor * c_mlp_proj_b;
|
||||
};
|
||||
|
||||
struct gptj_model {
|
||||
gptj_hparams hparams;
|
||||
|
||||
// normalization
|
||||
struct ggml_tensor * ln_f_g;
|
||||
struct ggml_tensor * ln_f_b;
|
||||
|
||||
struct ggml_tensor * wte; // token embedding
|
||||
|
||||
struct ggml_tensor * lmh_g; // language model head
|
||||
struct ggml_tensor * lmh_b; // language model bias
|
||||
|
||||
std::vector<gptj_layer> layers;
|
||||
|
||||
// key + value memory
|
||||
struct ggml_tensor * memory_k;
|
||||
struct ggml_tensor * memory_v;
|
||||
|
||||
//
|
||||
struct ggml_context * ctx;
|
||||
std::map<std::string, struct ggml_tensor *> tensors;
|
||||
};
|
||||
|
||||
// load the model's weights from a file
|
||||
bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
|
||||
printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
|
||||
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
fin.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// load hparams
|
||||
{
|
||||
auto & hparams = model.hparams;
|
||||
|
||||
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
|
||||
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: n_rot = %d\n", __func__, hparams.n_rot);
|
||||
printf("%s: ftype = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr = %d\n", __func__, qntvr);
|
||||
|
||||
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
fin.read((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != model.hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
std::vector<char> buf(128);
|
||||
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
fin.read((char *) &len, sizeof(len));
|
||||
|
||||
buf.resize(len);
|
||||
fin.read((char *) buf.data(), len);
|
||||
word.assign(buf.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
|
||||
// in order to save memory and also to speed up the computation
|
||||
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
|
||||
if (wtype == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
|
||||
__func__, fname.c_str(), model.hparams.ftype);
|
||||
return false;
|
||||
}
|
||||
|
||||
auto & ctx = model.ctx;
|
||||
|
||||
size_t ctx_size = 0;
|
||||
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
|
||||
|
||||
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // wte
|
||||
|
||||
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // lmh_g
|
||||
ctx_size += ggml_row_size(GGML_TYPE_F32, n_vocab); // lmh_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_q_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_k_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_v_proj_w
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
|
||||
|
||||
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
|
||||
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_mlp_proj_b
|
||||
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_k
|
||||
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_v
|
||||
|
||||
ctx_size += (5 + 10*n_layer)*512; // object overhead
|
||||
|
||||
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
|
||||
}
|
||||
|
||||
// create the ggml context
|
||||
{
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ctx_size,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
model.ctx = ggml_init(params);
|
||||
if (!model.ctx) {
|
||||
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// prepare memory for the weights
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
|
||||
model.layers.resize(n_layer);
|
||||
|
||||
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
|
||||
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
model.lmh_g = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
|
||||
model.lmh_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
|
||||
|
||||
// map by name
|
||||
model.tensors["transformer.wte.weight"] = model.wte;
|
||||
|
||||
model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
|
||||
model.tensors["transformer.ln_f.bias"] = model.ln_f_b;
|
||||
|
||||
model.tensors["lm_head.weight"] = model.lmh_g;
|
||||
model.tensors["lm_head.bias"] = model.lmh_b;
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & layer = model.layers[i];
|
||||
|
||||
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
|
||||
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
|
||||
|
||||
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
|
||||
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
|
||||
|
||||
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
|
||||
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
|
||||
|
||||
// map by name
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"] = layer.ln_1_g;
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"] = layer.ln_1_b;
|
||||
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"] = layer.c_attn_q_proj_w;
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"] = layer.c_attn_k_proj_w;
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"] = layer.c_attn_v_proj_w;
|
||||
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
|
||||
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"] = layer.c_mlp_fc_w;
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"] = layer.c_mlp_fc_b;
|
||||
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"] = layer.c_mlp_proj_w;
|
||||
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"] = layer.c_mlp_proj_b;
|
||||
}
|
||||
}
|
||||
|
||||
// key + value memory
|
||||
{
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
|
||||
const int n_mem = n_layer*n_ctx;
|
||||
const int n_elements = n_embd*n_mem;
|
||||
|
||||
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
|
||||
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
|
||||
|
||||
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
|
||||
|
||||
printf("%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
|
||||
}
|
||||
|
||||
// load weights
|
||||
{
|
||||
int n_tensors = 0;
|
||||
size_t total_size = 0;
|
||||
|
||||
printf("%s: ", __func__);
|
||||
|
||||
while (true) {
|
||||
int32_t n_dims;
|
||||
int32_t length;
|
||||
int32_t ttype;
|
||||
|
||||
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
|
||||
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
|
||||
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
|
||||
|
||||
if (fin.eof()) {
|
||||
break;
|
||||
}
|
||||
|
||||
int32_t nelements = 1;
|
||||
int32_t ne[2] = { 1, 1 };
|
||||
for (int i = 0; i < n_dims; ++i) {
|
||||
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
|
||||
nelements *= ne[i];
|
||||
}
|
||||
|
||||
std::string name(length, 0);
|
||||
fin.read(&name[0], length);
|
||||
|
||||
if (model.tensors.find(name) == model.tensors.end()) {
|
||||
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto tensor = model.tensors[name];
|
||||
if (ggml_nelements(tensor) != nelements) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
|
||||
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
|
||||
return false;
|
||||
}
|
||||
|
||||
// for debugging
|
||||
if (0) {
|
||||
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
|
||||
}
|
||||
|
||||
const size_t bpe = ggml_type_size(ggml_type(ttype));
|
||||
|
||||
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
|
||||
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
|
||||
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
|
||||
return false;
|
||||
}
|
||||
|
||||
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
|
||||
|
||||
//printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.c_str(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
|
||||
total_size += ggml_nbytes(tensor);
|
||||
if (++n_tensors % 8 == 0) {
|
||||
printf(".");
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
printf(" done\n");
|
||||
|
||||
printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
|
||||
}
|
||||
|
||||
fin.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// evaluate the transformer
|
||||
//
|
||||
// - model: the model
|
||||
// - n_threads: number of threads to use
|
||||
// - n_past: the context size so far
|
||||
// - embd_inp: the embeddings of the tokens in the context
|
||||
// - embd_w: the predicted logits for the next token
|
||||
//
|
||||
// The GPT-J model requires about 16MB of memory per input token.
|
||||
//
|
||||
bool gptj_eval(
|
||||
const gptj_model & model,
|
||||
const int n_threads,
|
||||
const int n_past,
|
||||
const std::vector<gpt_vocab::id> & embd_inp,
|
||||
std::vector<float> & embd_w,
|
||||
size_t & mem_per_token) {
|
||||
const int N = embd_inp.size();
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
const int n_embd = hparams.n_embd;
|
||||
const int n_layer = hparams.n_layer;
|
||||
const int n_ctx = hparams.n_ctx;
|
||||
const int n_head = hparams.n_head;
|
||||
const int n_vocab = hparams.n_vocab;
|
||||
const int n_rot = hparams.n_rot;
|
||||
|
||||
static size_t buf_size = 256u*1024*1024;
|
||||
static void * buf = malloc(buf_size);
|
||||
|
||||
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
|
||||
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
||||
|
||||
// reallocate
|
||||
buf_size = buf_size_new;
|
||||
buf = realloc(buf, buf_size);
|
||||
if (buf == nullptr) {
|
||||
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx0 = ggml_init(params);
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
|
||||
|
||||
// KQ_pos - contains the positions
|
||||
struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
|
||||
int * data = (int *) KQ_pos->data;
|
||||
for (int i = 0; i < N; ++i) {
|
||||
data[i] = n_past + i;
|
||||
}
|
||||
|
||||
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
|
||||
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
|
||||
|
||||
// wte
|
||||
struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
struct ggml_tensor * cur;
|
||||
|
||||
// norm
|
||||
{
|
||||
cur = ggml_norm(ctx0, inpL, hparams.eps);
|
||||
|
||||
// cur = ln_1_g*cur + ln_1_b
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_mul(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
|
||||
cur),
|
||||
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
|
||||
}
|
||||
|
||||
struct ggml_tensor * inpSA = cur;
|
||||
|
||||
// self-attention
|
||||
{
|
||||
struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
|
||||
struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
|
||||
|
||||
// store key and value to memory
|
||||
{
|
||||
struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
|
||||
|
||||
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
|
||||
struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd,
|
||||
( n_ctx)*ggml_element_size(model.memory_v),
|
||||
(il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v));
|
||||
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
|
||||
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
|
||||
}
|
||||
|
||||
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
|
||||
struct ggml_tensor * Q =
|
||||
ggml_permute(ctx0,
|
||||
Qcur,
|
||||
0, 2, 1, 3);
|
||||
|
||||
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
|
||||
struct ggml_tensor * K =
|
||||
ggml_permute(ctx0,
|
||||
ggml_reshape_3d(ctx0,
|
||||
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
|
||||
n_embd/n_head, n_head, n_past + N),
|
||||
0, 2, 1, 3);
|
||||
|
||||
// K * Q
|
||||
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
|
||||
|
||||
// KQ_scaled = KQ / sqrt(n_embd/n_head)
|
||||
struct ggml_tensor * KQ_scaled =
|
||||
ggml_scale_inplace(ctx0,
|
||||
KQ,
|
||||
1.0f/sqrt(float(n_embd)/n_head));
|
||||
|
||||
// KQ_masked = mask_past(KQ_scaled)
|
||||
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
|
||||
|
||||
// KQ = soft_max(KQ_masked)
|
||||
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
|
||||
|
||||
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
|
||||
struct ggml_tensor * V =
|
||||
ggml_view_3d(ctx0, model.memory_v,
|
||||
n_past + N, n_embd/n_head, n_head,
|
||||
n_ctx*ggml_element_size(model.memory_v),
|
||||
n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head,
|
||||
il*n_ctx*ggml_element_size(model.memory_v)*n_embd);
|
||||
|
||||
// KQV = transpose(V) * KQ_soft_max
|
||||
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
|
||||
|
||||
// KQV_merged = KQV.permute(0, 2, 1, 3)
|
||||
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
|
||||
|
||||
// cur = KQV_merged.contiguous().view(n_embd, N)
|
||||
cur = ggml_cpy(ctx0,
|
||||
KQV_merged,
|
||||
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
|
||||
|
||||
// projection (no bias)
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_attn_proj_w,
|
||||
cur);
|
||||
}
|
||||
|
||||
struct ggml_tensor * inpFF = cur;
|
||||
|
||||
// feed-forward network
|
||||
// this is independent of the self-attention result, so it could be done in parallel to the self-attention
|
||||
{
|
||||
// note here we pass inpSA instead of cur
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_mlp_fc_w,
|
||||
inpSA);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
|
||||
cur);
|
||||
|
||||
// GELU activation
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
|
||||
// projection
|
||||
// cur = proj_w*cur + proj_b
|
||||
cur = ggml_mul_mat(ctx0,
|
||||
model.layers[il].c_mlp_proj_w,
|
||||
cur);
|
||||
|
||||
cur = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
|
||||
cur);
|
||||
}
|
||||
|
||||
// self-attention + FF
|
||||
cur = ggml_add(ctx0, cur, inpFF);
|
||||
|
||||
// input for next layer
|
||||
inpL = ggml_add(ctx0, cur, inpL);
|
||||
}
|
||||
|
||||
// norm
|
||||
{
|
||||
inpL = ggml_norm(ctx0, inpL, hparams.eps);
|
||||
|
||||
// inpL = ln_f_g*inpL + ln_f_b
|
||||
inpL = ggml_add(ctx0,
|
||||
ggml_mul(ctx0,
|
||||
ggml_repeat(ctx0, model.ln_f_g, inpL),
|
||||
inpL),
|
||||
ggml_repeat(ctx0, model.ln_f_b, inpL));
|
||||
}
|
||||
|
||||
// lm_head
|
||||
{
|
||||
inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
|
||||
|
||||
inpL = ggml_add(ctx0,
|
||||
ggml_repeat(ctx0, model.lmh_b, inpL),
|
||||
inpL);
|
||||
}
|
||||
|
||||
// logits -> probs
|
||||
//inpL = ggml_soft_max_inplace(ctx0, inpL);
|
||||
|
||||
// run the computation
|
||||
ggml_build_forward_expand(gf, inpL);
|
||||
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
|
||||
|
||||
//if (n_past%100 == 0) {
|
||||
// ggml_graph_print (&gf);
|
||||
// ggml_graph_dump_dot(&gf, NULL, "gpt-j.dot");
|
||||
//}
|
||||
|
||||
//embd_w.resize(n_vocab*N);
|
||||
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
|
||||
|
||||
// return result for just the last token
|
||||
embd_w.resize(n_vocab);
|
||||
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
|
||||
|
||||
if (mem_per_token == 0) {
|
||||
mem_per_token = ggml_used_mem(ctx0)/N;
|
||||
}
|
||||
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
|
||||
|
||||
ggml_free(ctx0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_time_init();
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
gpt_params params;
|
||||
params.model = "models/gpt-j-6B/ggml-model.bin";
|
||||
|
||||
if (gpt_params_parse(argc, argv, params) == false) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.seed < 0) {
|
||||
params.seed = time(NULL);
|
||||
}
|
||||
|
||||
printf("%s: seed = %d\n", __func__, params.seed);
|
||||
|
||||
std::mt19937 rng(params.seed);
|
||||
if (params.prompt.empty()) {
|
||||
params.prompt = gpt_random_prompt(rng);
|
||||
}
|
||||
|
||||
int64_t t_load_us = 0;
|
||||
|
||||
gpt_vocab vocab;
|
||||
gptj_model model;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gptj_model_load(params.model, model, vocab)) {
|
||||
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_load_us = ggml_time_us() - t_start_us;
|
||||
|
||||
test_gpt_tokenizer(vocab, params.token_test);
|
||||
}
|
||||
|
||||
int n_past = 0;
|
||||
|
||||
int64_t t_sample_us = 0;
|
||||
int64_t t_predict_us = 0;
|
||||
|
||||
std::vector<float> logits;
|
||||
|
||||
// tokenize the prompt
|
||||
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
|
||||
|
||||
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
|
||||
|
||||
printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
|
||||
printf("\n");
|
||||
|
||||
std::vector<gpt_vocab::id> embd;
|
||||
|
||||
// determine the required inference memory per token:
|
||||
size_t mem_per_token = 0;
|
||||
gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
|
||||
|
||||
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
|
||||
// predict
|
||||
if (embd.size() > 0) {
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
|
||||
printf("Failed to predict\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_predict_us += ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
n_past += embd.size();
|
||||
embd.clear();
|
||||
|
||||
if (i >= embd_inp.size()) {
|
||||
// sample next token
|
||||
const int top_k = params.top_k;
|
||||
const float top_p = params.top_p;
|
||||
const float temp = params.temp;
|
||||
|
||||
const int n_vocab = model.hparams.n_vocab;
|
||||
|
||||
gpt_vocab::id id = 0;
|
||||
|
||||
{
|
||||
const int64_t t_start_sample_us = ggml_time_us();
|
||||
|
||||
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
|
||||
|
||||
t_sample_us += ggml_time_us() - t_start_sample_us;
|
||||
}
|
||||
|
||||
// add it to the context
|
||||
embd.push_back(id);
|
||||
} else {
|
||||
// if here, it means we are still processing the input prompt
|
||||
for (size_t k = i; k < embd_inp.size(); k++) {
|
||||
embd.push_back(embd_inp[k]);
|
||||
if (int32_t(embd.size()) > params.n_batch) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
i += embd.size() - 1;
|
||||
}
|
||||
|
||||
// display text
|
||||
for (auto id : embd) {
|
||||
printf("%s", vocab.id_to_token[id].c_str());
|
||||
}
|
||||
fflush(stdout);
|
||||
|
||||
// end of text token
|
||||
if (embd.back() == 50256) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n\n");
|
||||
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
|
||||
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
|
||||
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
|
||||
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
ggml_free(model.ctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
#include "ggml.h"
|
||||
|
||||
#include "common.h"
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <regex>
|
||||
|
||||
// default hparams (GPT-J 6B)
|
||||
struct gptj_hparams {
|
||||
int32_t n_vocab = 50400;
|
||||
int32_t n_ctx = 2048;
|
||||
int32_t n_embd = 4096;
|
||||
int32_t n_head = 16;
|
||||
int32_t n_layer = 28;
|
||||
int32_t n_rot = 64;
|
||||
int32_t ftype = 1;
|
||||
};
|
||||
|
||||
// quantize a model
|
||||
bool gptj_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
|
||||
gpt_vocab vocab;
|
||||
|
||||
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
|
||||
|
||||
auto finp = std::ifstream(fname_inp, std::ios::binary);
|
||||
if (!finp) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto fout = std::ofstream(fname_out, std::ios::binary);
|
||||
if (!fout) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// verify magic
|
||||
{
|
||||
uint32_t magic;
|
||||
finp.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
fout.write((char *) &magic, sizeof(magic));
|
||||
}
|
||||
|
||||
gptj_hparams hparams;
|
||||
|
||||
// load hparams
|
||||
{
|
||||
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
finp.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
|
||||
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
|
||||
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
|
||||
|
||||
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
|
||||
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
|
||||
printf("%s: n_head = %d\n", __func__, hparams.n_head);
|
||||
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
|
||||
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
|
||||
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
|
||||
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
|
||||
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
|
||||
|
||||
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
|
||||
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
|
||||
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
|
||||
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
|
||||
fout.write((char *) &hparams.n_rot, sizeof(hparams.n_rot));
|
||||
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
|
||||
}
|
||||
|
||||
// load vocab
|
||||
{
|
||||
int32_t n_vocab = 0;
|
||||
finp.read ((char *) &n_vocab, sizeof(n_vocab));
|
||||
fout.write((char *) &n_vocab, sizeof(n_vocab));
|
||||
|
||||
if (n_vocab != hparams.n_vocab) {
|
||||
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
|
||||
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string word;
|
||||
for (int i = 0; i < n_vocab; i++) {
|
||||
uint32_t len;
|
||||
finp.read ((char *) &len, sizeof(len));
|
||||
fout.write((char *) &len, sizeof(len));
|
||||
|
||||
word.resize(len);
|
||||
finp.read ((char *) word.data(), len);
|
||||
fout.write((char *) word.data(), len);
|
||||
|
||||
vocab.token_to_id[word] = i;
|
||||
vocab.id_to_token[i] = word;
|
||||
}
|
||||
}
|
||||
|
||||
// regexes of tensor names to be quantized
|
||||
const std::vector<std::string> to_quant = {
|
||||
".*weight",
|
||||
};
|
||||
|
||||
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
|
||||
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
finp.close();
|
||||
fout.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// usage:
|
||||
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
|
||||
//
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc != 4) {
|
||||
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
|
||||
ggml_print_ftypes(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// needed to initialize f16 tables
|
||||
{
|
||||
struct ggml_init_params params = { 0, NULL, false };
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
const std::string fname_inp = argv[1];
|
||||
const std::string fname_out = argv[2];
|
||||
|
||||
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
|
||||
|
||||
const int64_t t_main_start_us = ggml_time_us();
|
||||
|
||||
int64_t t_quantize_us = 0;
|
||||
|
||||
// load the model
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!gptj_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
|
||||
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
t_quantize_us = ggml_time_us() - t_start_us;
|
||||
}
|
||||
|
||||
// report timing
|
||||
{
|
||||
const int64_t t_main_end_us = ggml_time_us();
|
||||
|
||||
printf("\n");
|
||||
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
|
||||
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
#
|
||||
# magika
|
||||
|
||||
set(TEST_TARGET magika)
|
||||
add_executable(${TEST_TARGET} main.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
|
||||
|
||||
#
|
||||
# For GPU offloading
|
||||
|
||||
if (GGML_CUDA)
|
||||
add_compile_definitions(GGML_USE_CUDA)
|
||||
endif()
|
||||
|
||||
if (GGML_METAL)
|
||||
add_compile_definitions(GGML_USE_METAL)
|
||||
endif()
|
||||
@@ -0,0 +1,23 @@
|
||||
# Google Magika inference
|
||||
|
||||
Simple example that shows how to use GGML for inference with the [Google Magika](https://github.com/google/magika) file type detection model.
|
||||
|
||||
### Usage
|
||||
|
||||
- Obtain the Magika model in H5 format
|
||||
- Pinned version: https://github.com/google/magika/blob/4460acb5d3f86807c3b53223229dee2afa50c025/assets_generation/models/standard_v1/model.h5
|
||||
- Use `convert.py` to convert the model to gguf format:
|
||||
```bash
|
||||
$ python examples/magika/convert.py /path/to/model.h5
|
||||
```
|
||||
- Invoke the program with the model file and a list of files to identify:
|
||||
```bash
|
||||
$ build/bin/magika model.h5.gguf examples/sam/example.jpg examples/magika/convert.py README.md src/ggml.c /bin/gcc write.exe jfk.wav
|
||||
examples/sam/example.jpg : jpeg (100.00%) pptx (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
|
||||
examples/magika/convert.py : python (99.99%) javascript (0.00%) txt (0.00%) asm (0.00%) scala (0.00%)
|
||||
README.md : markdown (100.00%) txt (0.00%) yaml (0.00%) ppt (0.00%) shell (0.00%)
|
||||
src/ggml.c : c (99.95%) txt (0.04%) asm (0.01%) yaml (0.00%) html (0.00%)
|
||||
/bin/gcc : elf (99.98%) odex (0.02%) pptx (0.00%) smali (0.00%) shell (0.00%)
|
||||
write.exe : pebin (100.00%) ppt (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
|
||||
jfk.wav : wav (100.00%) ppt (0.00%) shell (0.00%) sevenzip (0.00%) scala (0.00%)
|
||||
```
|
||||
@@ -0,0 +1,32 @@
|
||||
import sys
|
||||
from tensorflow import keras
|
||||
import gguf
|
||||
|
||||
def convert(model_name):
|
||||
model = keras.models.load_model(model_name, compile=False)
|
||||
gguf_model_name = model_name + ".gguf"
|
||||
gguf_writer = gguf.GGUFWriter(gguf_model_name, "magika")
|
||||
|
||||
for layer in model.layers:
|
||||
# export layers with weights
|
||||
if layer.weights:
|
||||
for weight in layer.weights:
|
||||
print(f" [{weight.name}] {weight.shape} {weight.dtype}")
|
||||
weight_data = weight.numpy()
|
||||
gguf_writer.add_tensor(weight.name, weight_data.T)
|
||||
|
||||
|
||||
gguf_writer.write_header_to_file()
|
||||
gguf_writer.write_kv_data_to_file()
|
||||
gguf_writer.write_tensors_to_file()
|
||||
gguf_writer.close()
|
||||
print("Model converted and saved to '{}'".format(gguf_model_name))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) > 1:
|
||||
model_file = sys.argv[1]
|
||||
else:
|
||||
model_file = "model.h5"
|
||||
|
||||
convert(model_file)
|
||||
@@ -0,0 +1,374 @@
|
||||
#include "ggml.h"
|
||||
#include "gguf.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static const char * magika_labels[] = {
|
||||
"ai", "apk", "appleplist", "asm", "asp",
|
||||
"batch", "bmp", "bzip", "c", "cab",
|
||||
"cat", "chm", "coff", "crx", "cs",
|
||||
"css", "csv", "deb", "dex", "dmg",
|
||||
"doc", "docx", "elf", "emf", "eml",
|
||||
"epub", "flac", "gif", "go", "gzip",
|
||||
"hlp", "html", "ico", "ini", "internetshortcut",
|
||||
"iso", "jar", "java", "javabytecode", "javascript",
|
||||
"jpeg", "json", "latex", "lisp", "lnk",
|
||||
"m3u", "macho", "makefile", "markdown", "mht",
|
||||
"mp3", "mp4", "mscompress", "msi", "mum",
|
||||
"odex", "odp", "ods", "odt", "ogg",
|
||||
"outlook", "pcap", "pdf", "pebin", "pem",
|
||||
"perl", "php", "png", "postscript", "powershell",
|
||||
"ppt", "pptx", "python", "pythonbytecode", "rar",
|
||||
"rdf", "rpm", "rst", "rtf", "ruby",
|
||||
"rust", "scala", "sevenzip", "shell", "smali",
|
||||
"sql", "squashfs", "svg", "swf", "symlinktext",
|
||||
"tar", "tga", "tiff", "torrent", "ttf",
|
||||
"txt", "unknown", "vba", "wav", "webm",
|
||||
"webp", "winregistry", "wmf", "xar", "xls",
|
||||
"xlsb", "xlsx", "xml", "xpi", "xz",
|
||||
"yaml", "zip", "zlibstream"
|
||||
};
|
||||
|
||||
struct magika_hparams {
|
||||
const int block_size = 4096;
|
||||
const int beg_size = 512;
|
||||
const int mid_size = 512;
|
||||
const int end_size = 512;
|
||||
const int min_file_size_for_dl = 16;
|
||||
const int n_label = 113;
|
||||
const float f_norm_eps = 0.001f;
|
||||
const int padding_token = 256;
|
||||
};
|
||||
|
||||
struct magika_model {
|
||||
~magika_model() {
|
||||
ggml_backend_buffer_free(buf_w);
|
||||
ggml_backend_free(backend);
|
||||
ggml_free(ctx_w);
|
||||
}
|
||||
|
||||
magika_hparams hparams;
|
||||
|
||||
struct ggml_tensor * dense_w;
|
||||
struct ggml_tensor * dense_b;
|
||||
|
||||
struct ggml_tensor * layer_norm_gamma;
|
||||
struct ggml_tensor * layer_norm_beta;
|
||||
|
||||
struct ggml_tensor * dense_1_w;
|
||||
struct ggml_tensor * dense_1_b;
|
||||
|
||||
struct ggml_tensor * dense_2_w;
|
||||
struct ggml_tensor * dense_2_b;
|
||||
|
||||
struct ggml_tensor * layer_norm_1_gamma;
|
||||
struct ggml_tensor * layer_norm_1_beta;
|
||||
|
||||
struct ggml_tensor * target_label_w;
|
||||
struct ggml_tensor * target_label_b;
|
||||
|
||||
ggml_backend_t backend = ggml_backend_cpu_init();
|
||||
ggml_backend_buffer_t buf_w = nullptr;
|
||||
struct ggml_context * ctx_w = nullptr;
|
||||
};
|
||||
|
||||
struct ggml_tensor * checked_get_tensor(struct ggml_context * ctx, const char * name) {
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
|
||||
if (!tensor) {
|
||||
fprintf(stderr, "%s: tensor '%s' not found\n", __func__, name);
|
||||
throw std::runtime_error("ggml_get_tensor() failed");
|
||||
}
|
||||
return tensor;
|
||||
}
|
||||
|
||||
bool magika_model_load(const std::string & fname, magika_model & model) {
|
||||
auto & ctx = model.ctx_w;
|
||||
|
||||
struct gguf_init_params params = {
|
||||
/*.no_alloc =*/ true,
|
||||
/*.ctx =*/ &ctx,
|
||||
};
|
||||
|
||||
struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), params);
|
||||
if (!ctx_gguf) {
|
||||
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
model.buf_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
|
||||
if (!model.buf_w) {
|
||||
fprintf(stderr, "%s: ggml_backend_alloc_ctx_tensors() failed\n", __func__);
|
||||
gguf_free(ctx_gguf);
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
model.dense_w = checked_get_tensor(ctx, "dense/kernel:0");
|
||||
model.dense_b = checked_get_tensor(ctx, "dense/bias:0");
|
||||
|
||||
model.layer_norm_gamma = checked_get_tensor(ctx, "layer_normalization/gamma:0");
|
||||
model.layer_norm_beta = checked_get_tensor(ctx, "layer_normalization/beta:0");
|
||||
|
||||
model.dense_1_w = checked_get_tensor(ctx, "dense_1/kernel:0");
|
||||
model.dense_1_b = checked_get_tensor(ctx, "dense_1/bias:0");
|
||||
|
||||
model.dense_2_w = checked_get_tensor(ctx, "dense_2/kernel:0");
|
||||
model.dense_2_b = checked_get_tensor(ctx, "dense_2/bias:0");
|
||||
|
||||
model.layer_norm_1_gamma = checked_get_tensor(ctx, "layer_normalization_1/gamma:0");
|
||||
model.layer_norm_1_beta = checked_get_tensor(ctx, "layer_normalization_1/beta:0");
|
||||
|
||||
model.target_label_w = checked_get_tensor(ctx, "target_label/kernel:0");
|
||||
model.target_label_b = checked_get_tensor(ctx, "target_label/bias:0");
|
||||
} catch (const std::exception & e) {
|
||||
fprintf(stderr, "%s: %s\n", __func__, e.what());
|
||||
gguf_free(ctx_gguf);
|
||||
return false;
|
||||
}
|
||||
|
||||
FILE * f = fopen(fname.c_str(), "rb");
|
||||
if (!f) {
|
||||
fprintf(stderr, "%s: fopen() failed\n", __func__);
|
||||
gguf_free(ctx_gguf);
|
||||
return false;
|
||||
}
|
||||
|
||||
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
|
||||
|
||||
for (int i = 0; i < n_tensors; i++) {
|
||||
const char * name = gguf_get_tensor_name(ctx_gguf, i);
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
|
||||
size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
|
||||
|
||||
//printf("%-30s: [%3ld, %3ld, %3ld, %3ld] %s\n",
|
||||
// name,
|
||||
// tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3],
|
||||
// ggml_type_name(tensor->type));
|
||||
|
||||
std::vector<uint8_t> buf(ggml_nbytes(tensor));
|
||||
if (fseek(f, offs, SEEK_SET) != 0) {
|
||||
fprintf(stderr, "%s: fseek() failed\n", __func__);
|
||||
gguf_free(ctx_gguf);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {
|
||||
fprintf(stderr, "%s: fread() failed\n", __func__);
|
||||
gguf_free(ctx_gguf);
|
||||
fclose(f);
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size());
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
gguf_free(ctx_gguf);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
struct ggml_cgraph * magika_graph(
|
||||
const magika_model & model,
|
||||
const int n_files) {
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
|
||||
static std::vector<uint8_t> buf(buf_size);
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ buf_size,
|
||||
/*.mem_buffer =*/ buf.data(),
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_cgraph * gf = ggml_new_graph(ctx);
|
||||
|
||||
struct ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 257, 1536, n_files); // one-hot
|
||||
ggml_set_name(input, "input");
|
||||
ggml_set_input(input);
|
||||
|
||||
struct ggml_tensor * cur;
|
||||
|
||||
// dense
|
||||
cur = ggml_mul_mat(ctx, model.dense_w, input);
|
||||
cur = ggml_add(ctx, cur, model.dense_b); // [128, 1536, n_files]
|
||||
cur = ggml_gelu(ctx, cur);
|
||||
|
||||
// reshape
|
||||
cur = ggml_reshape_3d(ctx, cur, 512, 384, n_files); // [384, 512, n_files]
|
||||
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
|
||||
|
||||
// layer normalization
|
||||
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
|
||||
cur = ggml_mul(ctx, cur, model.layer_norm_gamma); // [384, 512, n_files]
|
||||
cur = ggml_add(ctx, cur, model.layer_norm_beta); // [384, 512, n_files]
|
||||
|
||||
// dense_1
|
||||
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
|
||||
cur = ggml_mul_mat(ctx, model.dense_1_w, cur);
|
||||
cur = ggml_add(ctx, cur, model.dense_1_b); // [256, 384, n_files]
|
||||
cur = ggml_gelu(ctx, cur);
|
||||
|
||||
// dense_2
|
||||
cur = ggml_mul_mat(ctx, model.dense_2_w, cur);
|
||||
cur = ggml_add(ctx, cur, model.dense_2_b); // [256, 384, n_files]
|
||||
cur = ggml_gelu(ctx, cur);
|
||||
|
||||
// global_max_pooling1d
|
||||
cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); // [384, 256, n_files]
|
||||
cur = ggml_pool_1d(ctx, cur, GGML_OP_POOL_MAX, 384, 384, 0); // [1, 256, n_files]
|
||||
cur = ggml_reshape_2d(ctx, cur, 256, n_files); // [256, n_files]
|
||||
|
||||
// layer normalization 1
|
||||
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
|
||||
cur = ggml_mul(ctx, cur, model.layer_norm_1_gamma); // [256, n_files]
|
||||
cur = ggml_add(ctx, cur, model.layer_norm_1_beta); // [256, n_files]
|
||||
|
||||
// target_label
|
||||
cur = ggml_mul_mat(ctx, model.target_label_w, cur);
|
||||
cur = ggml_add(ctx, cur, model.target_label_b); // [n_label, n_files]
|
||||
cur = ggml_soft_max(ctx, cur); // [n_label, n_files]
|
||||
ggml_set_name(cur, "target_label_probs");
|
||||
ggml_set_output(cur);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
bool magika_eval(
|
||||
struct magika_model & model,
|
||||
const std::vector<std::string> & fnames) {
|
||||
|
||||
const auto & hparams = model.hparams;
|
||||
|
||||
static ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
|
||||
|
||||
struct ggml_cgraph * gf = magika_graph(model, fnames.size());
|
||||
|
||||
if (!ggml_gallocr_alloc_graph(alloc, gf)) {
|
||||
fprintf(stderr, "%s: ggml_gallocr_alloc_graph() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
struct ggml_tensor * input = ggml_graph_get_tensor(gf, "input");
|
||||
|
||||
for (size_t i = 0; i < fnames.size(); i++) {
|
||||
FILE * f = fopen(fnames[i].c_str(), "rb");
|
||||
if (!f) {
|
||||
fprintf(stderr, "%s: fopen() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long fsize = ftell(f);
|
||||
|
||||
// the buffer is padded with the padding_token if the file is smaller than the block size
|
||||
std::vector<int> buf(1536, hparams.padding_token);
|
||||
std::vector<uint8_t> read_buf(std::max(hparams.beg_size, std::max(hparams.mid_size, hparams.end_size)));
|
||||
|
||||
// read beg
|
||||
fseek(f, 0, SEEK_SET);
|
||||
int n_read = fread(read_buf.data(), 1, hparams.beg_size, f);
|
||||
for (int j = 0; j < n_read; j++) {
|
||||
// pad at the end
|
||||
buf[j] = read_buf[j];
|
||||
}
|
||||
|
||||
// read mid
|
||||
long mid_offs = std::max(0L, (fsize - hparams.mid_size) / 2);
|
||||
fseek(f, mid_offs, SEEK_SET);
|
||||
n_read = fread(read_buf.data(), 1, hparams.mid_size, f);
|
||||
for (int j = 0; j < n_read; j++) {
|
||||
// pad at both ends
|
||||
long mid_idx = hparams.beg_size + (hparams.mid_size / 2) - n_read / 2 + j;
|
||||
buf[mid_idx] = read_buf[j];
|
||||
}
|
||||
|
||||
// read end
|
||||
long end_offs = std::max(0L, fsize - hparams.end_size);
|
||||
fseek(f, end_offs, SEEK_SET);
|
||||
n_read = fread(read_buf.data(), 1, hparams.end_size, f);
|
||||
for (int j = 0; j < n_read; j++) {
|
||||
// pad at the beginning
|
||||
int end_idx = hparams.beg_size + hparams.mid_size + hparams.end_size - n_read + j;
|
||||
buf[end_idx] = read_buf[j];
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
|
||||
const size_t inp_bytes = hparams.beg_size + hparams.mid_size + hparams.end_size;
|
||||
|
||||
// convert to one-hot
|
||||
std::vector<float> one_hot(257*inp_bytes);
|
||||
for (size_t j = 0; j < inp_bytes; j++) {
|
||||
one_hot[257*j + buf[j]] = 1.0f;
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set(input, one_hot.data(), 257*inp_bytes*i*sizeof(float), 257*inp_bytes*sizeof(float));
|
||||
}
|
||||
|
||||
if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {
|
||||
fprintf(stderr, "%s: ggml_backend_graph_compute() failed\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
struct ggml_tensor * target_label_probs = ggml_graph_get_tensor(gf, "target_label_probs");
|
||||
|
||||
// print probabilities for the top labels of each file
|
||||
for (size_t i = 0; i < fnames.size(); i++) {
|
||||
std::vector<float> probs(hparams.n_label);
|
||||
ggml_backend_tensor_get(target_label_probs, probs.data(), hparams.n_label*i*sizeof(float), hparams.n_label*sizeof(float));
|
||||
|
||||
// sort the probabilities
|
||||
std::vector<int> idx(hparams.n_label);
|
||||
std::iota(idx.begin(), idx.end(), 0);
|
||||
std::sort(idx.begin(), idx.end(), [&probs](int i1, int i2) { return probs[i1] > probs[i2]; });
|
||||
|
||||
// print the top labels
|
||||
const int top_n = 5;
|
||||
printf("%-30s: ", fnames[i].c_str());
|
||||
for (int j = 0; j < top_n; j++) {
|
||||
printf("%s (%.2f%%) ", magika_labels[idx[j]], probs[idx[j]]*100);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, const char ** argv) {
|
||||
if (argc < 3) {
|
||||
fprintf(stderr, "usage: %s <model> <file1> [<file2> ...]\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const char * model_fname = argv[1];
|
||||
std::vector<std::string> fnames;
|
||||
for (int i = 2; i < argc; i++) {
|
||||
fnames.push_back(argv[i]);
|
||||
}
|
||||
|
||||
magika_model model;
|
||||
if (!magika_model_load(model_fname, model)) {
|
||||
fprintf(stderr, "magika_model_load() failed\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
magika_eval(model, fnames);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
data/
|
||||
*.gguf
|
||||
*.ggml
|
||||
@@ -0,0 +1,58 @@
|
||||
#
|
||||
# mnist-common
|
||||
|
||||
set(TEST_TARGET mnist-common)
|
||||
add_library(${TEST_TARGET} STATIC mnist-common.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
|
||||
|
||||
#
|
||||
# mnist-eval
|
||||
|
||||
set(TEST_TARGET mnist-eval)
|
||||
add_executable(${TEST_TARGET} mnist-eval.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
|
||||
|
||||
#
|
||||
# mnist-train
|
||||
|
||||
set(TEST_TARGET mnist-train)
|
||||
add_executable(${TEST_TARGET} mnist-train.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
|
||||
|
||||
|
||||
#
|
||||
# mnist-wasm
|
||||
if (EMSCRIPTEN)
|
||||
set(TARGET mnist)
|
||||
|
||||
add_executable(${TARGET} mnist-common.cpp)
|
||||
target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu)
|
||||
|
||||
set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
|
||||
--bind \
|
||||
-s FORCE_FILESYSTEM=1 \
|
||||
-s USE_PTHREADS=1 \
|
||||
-s PTHREAD_POOL_SIZE=10 \
|
||||
-s ASSERTIONS=1 \
|
||||
-s WASM=1 \
|
||||
-s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'setValue', 'getValue']\" \
|
||||
-s EXPORTED_FUNCTIONS=\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\" \
|
||||
-s ALLOW_MEMORY_GROWTH=1 \
|
||||
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \
|
||||
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \
|
||||
")
|
||||
|
||||
# Copy output to web directory
|
||||
add_custom_command(
|
||||
TARGET ${TARGET} POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.js
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.wasm
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${CMAKE_BINARY_DIR}/bin/mnist.worker.js
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js
|
||||
)
|
||||
endif()
|
||||
@@ -0,0 +1,206 @@
|
||||
# MNIST Examples for GGML
|
||||
|
||||
This directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/).
|
||||
All commands listed in this README assume the working directory to be `examples/mnist`.
|
||||
Please note that training in GGML is a work-in-progress and not production ready.
|
||||
|
||||
## Obtaining the data
|
||||
|
||||
A description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/).
|
||||
While it is also in principle possible to download the dataset from this website these downloads are frequently throttled and
|
||||
it is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead.
|
||||
The dataset will be downloaded automatically when running `mnist-train-fc.py`.
|
||||
|
||||
## Fully connected network
|
||||
|
||||
For our first example we will train a fully connected network.
|
||||
To train a fully connected model in PyTorch and save it as a GGUF file, run:
|
||||
|
||||
```bash
|
||||
$ python3 mnist-train-fc.py mnist-fc-f32.gguf
|
||||
|
||||
...
|
||||
|
||||
Test loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14%
|
||||
|
||||
Model tensors saved to mnist-fc-f32.gguf:
|
||||
fc1.weight (500, 784)
|
||||
fc1.bias (500,)
|
||||
fc2.weight (10, 500)
|
||||
fc2.bias (10,)
|
||||
```
|
||||
|
||||
The training script includes an evaluation of the model on the test set.
|
||||
To evaluate the model on the CPU using GGML, run:
|
||||
|
||||
```bash
|
||||
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
|
||||
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
__________________________________####__________________
|
||||
______________________________########__________________
|
||||
__________________________##########____________________
|
||||
______________________##############____________________
|
||||
____________________######________####__________________
|
||||
__________________________________####__________________
|
||||
__________________________________####__________________
|
||||
________________________________####____________________
|
||||
______________________________####______________________
|
||||
________________________##########______________________
|
||||
______________________########__####____________________
|
||||
________________________##__________##__________________
|
||||
____________________________________##__________________
|
||||
__________________________________##____________________
|
||||
__________________________________##____________________
|
||||
________________________________##______________________
|
||||
____________________________####________________________
|
||||
__________##____________######__________________________
|
||||
__________##############________________________________
|
||||
________________####____________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
|
||||
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
|
||||
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
|
||||
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
|
||||
mnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf'
|
||||
mnist_model_init_from_file: model arch is mnist-fc
|
||||
mnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf
|
||||
main: loaded model in 109.44 ms
|
||||
mnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image
|
||||
main: predicted digit is 3
|
||||
main: test_loss=0.066379+-0.009101
|
||||
main: test_acc=97.94+-0.14%
|
||||
```
|
||||
|
||||
In addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image.
|
||||
To train a fully connected model on the CPU using GGML run:
|
||||
|
||||
``` bash
|
||||
$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
|
||||
```
|
||||
|
||||
It can then be evaluated with the same binary as above.
|
||||
|
||||
## Convolutional network
|
||||
|
||||
To train a convolutional network using TensorFlow run:
|
||||
|
||||
```bash
|
||||
$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf
|
||||
|
||||
...
|
||||
|
||||
Test loss: 0.047947
|
||||
Test accuracy: 98.46%
|
||||
GGUF model saved to 'mnist-cnn-f32.gguf'
|
||||
```
|
||||
|
||||
The saved model can be evaluated on the CPU using the `mnist-eval` binary:
|
||||
|
||||
```bash
|
||||
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
|
||||
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
______________________________________##________________
|
||||
______________________________________##________________
|
||||
______________________________________##________________
|
||||
____________________________________##__________________
|
||||
__________________________________####__________________
|
||||
__________________________________##____________________
|
||||
________________________________##______________________
|
||||
______________________________##________________________
|
||||
____________________________####________________________
|
||||
____________________________##__________________________
|
||||
__________________________##____________________________
|
||||
________________________##______________________________
|
||||
______________________##________________________________
|
||||
____________________####________________________________
|
||||
____________________##__________________________________
|
||||
__________________##____________________________________
|
||||
________________##______________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
________________________________________________________
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
|
||||
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
|
||||
ggml_cuda_init: found 1 CUDA devices:
|
||||
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
|
||||
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
|
||||
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
|
||||
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
|
||||
mnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf'
|
||||
mnist_model_init_from_file: model arch is mnist-cnn
|
||||
mnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf
|
||||
main: loaded model in 91.99 ms
|
||||
mnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image
|
||||
main: predicted digit is 1
|
||||
main: test_loss=0.047955+-0.007029
|
||||
main: test_acc=98.46+-0.12%
|
||||
```
|
||||
|
||||
Like with the fully connected network the convolutional network can also be trained using GGML:
|
||||
|
||||
``` bash
|
||||
$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
|
||||
```
|
||||
|
||||
As always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`.
|
||||
|
||||
## Hardware Acceleration
|
||||
|
||||
Both the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations.
|
||||
A specific backend can be selected by appending the above commands with a backend name.
|
||||
The compute graphs then schedule the operations to preferentially use the specified backend.
|
||||
Note that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance.
|
||||
|
||||
## Web demo
|
||||
|
||||
The evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation).
|
||||
First, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`.
|
||||
Copy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`.
|
||||
Symlinking these files will *not* work!
|
||||
Compile the code like so:
|
||||
|
||||
```bash
|
||||
$ cd ../../
|
||||
$ mkdir -p build-em
|
||||
$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \
|
||||
-DCMAKE_C_FLAGS="-pthread -matomics -mbulk-memory" \
|
||||
-DCMAKE_CXX_FLAGS="-pthread -matomics -mbulk-memory"
|
||||
$ make mnist
|
||||
```
|
||||
|
||||
The compilation output is copied into `examples/mnist/web`.
|
||||
To run it, you need an HTTP server.
|
||||
For example:
|
||||
|
||||
``` bash
|
||||
$ python3 examples/mnist/server.py
|
||||
|
||||
Serving directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000
|
||||
Application context root: http://localhost:8000/
|
||||
```
|
||||
|
||||
The web demo can then be accessed via the link printed on the console.
|
||||
Simply draw a digit on the canvas and the model will try to predict what it's supposed to be.
|
||||
Alternatively, click the "Random" button to retrieve a random digit from the test set.
|
||||
Be aware that like all neural networks the one we trained is susceptible to distributional shift:
|
||||
if the numbers you draw look different than the ones in the training set
|
||||
(e.g. because they're not centered) the model will perform comparatively worse.
|
||||
An online demo can be accessed [here](https://mnist.ggerganov.com).
|
||||
@@ -0,0 +1,496 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) {
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open images file %s\n", fname.c_str());
|
||||
return false;
|
||||
}
|
||||
fin.seekg(16);
|
||||
|
||||
uint8_t image[MNIST_NINPUT];
|
||||
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
|
||||
float * buf = ggml_get_data_f32(images);
|
||||
|
||||
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
|
||||
for (int64_t iex = 0; iex < images->ne[1]; ++iex) {
|
||||
fin.read((char *) image, sizeof(image));
|
||||
|
||||
for (int64_t i = 0; i < MNIST_NINPUT; ++i) {
|
||||
buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1]
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) {
|
||||
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
|
||||
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
|
||||
GGML_ASSERT(iex < images->ne[1]);
|
||||
const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT;
|
||||
|
||||
for (int64_t row = 0; row < MNIST_HW; row++) {
|
||||
for (int64_t col = 0; col < MNIST_HW; col++) {
|
||||
const int rgb = roundf(255.0f * image[row*MNIST_HW + col]);
|
||||
#ifdef _WIN32
|
||||
fprintf(stream, "%s", rgb >= 220 ? "##" : "__"); // Represented via text.
|
||||
#else
|
||||
fprintf(stream, "\033[48;2;%d;%d;%dm \033[0m", rgb, rgb, rgb); // Represented via colored blocks.
|
||||
#endif // _WIN32
|
||||
}
|
||||
fprintf(stream, "\n");
|
||||
}
|
||||
}
|
||||
|
||||
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) {
|
||||
auto fin = std::ifstream(fname, std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open labels file %s\n", fname.c_str());
|
||||
return 0;
|
||||
}
|
||||
fin.seekg(8);
|
||||
|
||||
uint8_t label;
|
||||
struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset);
|
||||
float * buf = ggml_get_data_f32(labels);
|
||||
|
||||
GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES);
|
||||
for (int64_t iex = 0; iex < labels->ne[1]; ++iex) {
|
||||
fin.read((char *) &label, sizeof(label));
|
||||
|
||||
for (int64_t i = 0; i < MNIST_NCLASSES; ++i) {
|
||||
buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality:
|
||||
bool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) {
|
||||
FILE * f = ggml_fopen(fname, "rb");
|
||||
if (!f) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t buf_size = 4*1024*1024;
|
||||
void * buf = malloc(buf_size);
|
||||
|
||||
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
|
||||
for (int i = 0; i < n_tensors; i++) {
|
||||
const char * name = gguf_get_tensor_name(ctx_gguf, i);
|
||||
|
||||
struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name);
|
||||
if (!tensor) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
|
||||
|
||||
if (fseek(f, offs, SEEK_SET) != 0) {
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
|
||||
const size_t nbytes = ggml_nbytes(tensor);
|
||||
for (size_t pos = 0; pos < nbytes; pos += buf_size) {
|
||||
const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos;
|
||||
|
||||
if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) {
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return false;
|
||||
}
|
||||
|
||||
ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy);
|
||||
}
|
||||
}
|
||||
|
||||
fclose(f);
|
||||
free(buf);
|
||||
return true;
|
||||
}
|
||||
|
||||
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
|
||||
mnist_model model(backend, nbatch_logical, nbatch_physical);
|
||||
fprintf(stderr, "%s: loading model weights from '%s'\n", __func__, fname.c_str());
|
||||
|
||||
struct gguf_context * ctx;
|
||||
{
|
||||
struct gguf_init_params params = {
|
||||
/*.no_alloc =*/ true,
|
||||
/*.ctx =*/ &model.ctx_gguf,
|
||||
};
|
||||
ctx = gguf_init_from_file(fname.c_str(), params);
|
||||
if (!ctx) {
|
||||
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, "general.architecture"));
|
||||
fprintf(stderr, "%s: model arch is %s\n", __func__, model.arch.c_str());
|
||||
|
||||
if (model.arch == "mnist-fc") {
|
||||
model.fc1_weight = ggml_get_tensor(model.ctx_gguf, "fc1.weight");
|
||||
GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT);
|
||||
GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc1_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc1_weight->ne[3] == 1);
|
||||
|
||||
model.fc1_bias = ggml_get_tensor(model.ctx_gguf, "fc1.bias");
|
||||
GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc1_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.fc1_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc1_bias->ne[3] == 1);
|
||||
|
||||
model.fc2_weight = ggml_get_tensor(model.ctx_gguf, "fc2.weight");
|
||||
GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN);
|
||||
GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.fc2_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc2_weight->ne[3] == 1);
|
||||
|
||||
model.fc2_bias = ggml_get_tensor(model.ctx_gguf, "fc2.bias");
|
||||
GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.fc2_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.fc2_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.fc2_bias->ne[3] == 1);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, "conv1.kernel");
|
||||
GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[0] == 3);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[1] == 3);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[2] == 1);
|
||||
GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB);
|
||||
|
||||
model.conv1_bias = ggml_get_tensor(model.ctx_gguf, "conv1.bias");
|
||||
GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv1_bias->ne[0] == 1);
|
||||
GGML_ASSERT(model.conv1_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(model.conv1_bias->ne[3] == 1);
|
||||
|
||||
model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, "conv2.kernel");
|
||||
GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[0] == 3);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[1] == 3);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2);
|
||||
|
||||
model.conv2_bias = ggml_get_tensor(model.ctx_gguf, "conv2.bias");
|
||||
GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.conv2_bias->ne[0] == 1);
|
||||
GGML_ASSERT(model.conv2_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(model.conv2_bias->ne[3] == 1);
|
||||
|
||||
model.dense_weight = ggml_get_tensor(model.ctx_gguf, "dense.weight");
|
||||
GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
|
||||
GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.dense_weight->ne[2] == 1);
|
||||
GGML_ASSERT(model.dense_weight->ne[3] == 1);
|
||||
|
||||
model.dense_bias = ggml_get_tensor(model.ctx_gguf, "dense.bias");
|
||||
GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.dense_bias->ne[1] == 1);
|
||||
GGML_ASSERT(model.dense_bias->ne[2] == 1);
|
||||
GGML_ASSERT(model.dense_bias->ne[3] == 1);
|
||||
} else {
|
||||
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
|
||||
}
|
||||
|
||||
model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]);
|
||||
|
||||
if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) {
|
||||
fprintf(stderr, "%s: loading weights from %s failed\n", __func__, fname.c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// The space in ctx_gguf exactly fits the model weights,
|
||||
// the images (which also need to be statically allocated) need to be put in a different context.
|
||||
|
||||
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical);
|
||||
|
||||
ggml_set_name(model.images, "images");
|
||||
ggml_set_input(model.images);
|
||||
|
||||
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
|
||||
|
||||
fprintf(stderr, "%s: successfully loaded weights from %s\n", __func__, fname.c_str());
|
||||
return model;
|
||||
}
|
||||
|
||||
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
|
||||
mnist_model model(backend, nbatch_logical, nbatch_physical);
|
||||
model.arch = arch;
|
||||
|
||||
std::random_device rd{};
|
||||
std::mt19937 gen{rd()};
|
||||
std::normal_distribution<float> nd{0.0f, 1e-2f};
|
||||
std::vector<ggml_tensor *> init_tensors;
|
||||
|
||||
if (model.arch == "mnist-fc") {
|
||||
fprintf(stderr, "%s: initializing random weights for a fully connected model\n", __func__);
|
||||
|
||||
model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN);
|
||||
model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN);
|
||||
model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES);
|
||||
model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
|
||||
|
||||
ggml_set_name(model.fc1_weight, "fc1.weight");
|
||||
ggml_set_name(model.fc1_bias, "fc1.bias");
|
||||
ggml_set_name(model.fc2_weight, "fc2.weight");
|
||||
ggml_set_name(model.fc2_bias, "fc2.bias");
|
||||
|
||||
init_tensors.push_back(model.fc1_weight);
|
||||
init_tensors.push_back(model.fc1_bias);
|
||||
init_tensors.push_back(model.fc2_weight);
|
||||
init_tensors.push_back(model.fc2_bias);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB);
|
||||
model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB);
|
||||
model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2);
|
||||
model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2);
|
||||
model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES);
|
||||
model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
|
||||
|
||||
ggml_set_name(model.conv1_kernel, "conv1.kernel");
|
||||
ggml_set_name(model.conv1_bias, "conv1.bias");
|
||||
ggml_set_name(model.conv2_kernel, "conv2.kernel");
|
||||
ggml_set_name(model.conv2_bias, "conv2.bias");
|
||||
ggml_set_name(model.dense_weight, "dense.weight");
|
||||
ggml_set_name(model.dense_bias, "dense.bias");
|
||||
|
||||
init_tensors.push_back(model.conv1_kernel);
|
||||
init_tensors.push_back(model.conv1_bias);
|
||||
init_tensors.push_back(model.conv2_kernel);
|
||||
init_tensors.push_back(model.conv2_bias);
|
||||
init_tensors.push_back(model.dense_weight);
|
||||
init_tensors.push_back(model.dense_bias);
|
||||
} else {
|
||||
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
|
||||
}
|
||||
|
||||
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL);
|
||||
ggml_set_name(model.images, "images");
|
||||
ggml_set_input(model.images);
|
||||
|
||||
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
|
||||
|
||||
for (ggml_tensor * t : init_tensors) {
|
||||
GGML_ASSERT(t->type == GGML_TYPE_F32);
|
||||
const int64_t ne = ggml_nelements(t);
|
||||
std::vector<float> tmp(ne);
|
||||
|
||||
for (int64_t i = 0; i < ne; ++i) {
|
||||
tmp[i] = nd(gen);
|
||||
}
|
||||
ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t));
|
||||
}
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
void mnist_model_build(mnist_model & model) {
|
||||
if (model.arch == "mnist-fc") {
|
||||
ggml_set_param(model.fc1_weight);
|
||||
ggml_set_param(model.fc1_bias);
|
||||
ggml_set_param(model.fc2_weight);
|
||||
ggml_set_param(model.fc2_bias);
|
||||
|
||||
ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images),
|
||||
model.fc1_bias));
|
||||
model.logits = ggml_add(model.ctx_compute,
|
||||
ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1),
|
||||
model.fc2_bias);
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
ggml_set_param(model.conv1_kernel);
|
||||
ggml_set_param(model.conv1_bias);
|
||||
ggml_set_param(model.conv2_kernel);
|
||||
ggml_set_param(model.conv2_bias);
|
||||
ggml_set_param(model.dense_weight);
|
||||
ggml_set_param(model.dense_bias);
|
||||
|
||||
struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]);
|
||||
|
||||
struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1),
|
||||
model.conv1_bias));
|
||||
GGML_ASSERT(conv1_out->ne[0] == MNIST_HW);
|
||||
GGML_ASSERT(conv1_out->ne[1] == MNIST_HW);
|
||||
GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
|
||||
GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB);
|
||||
GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
|
||||
ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1),
|
||||
model.conv2_bias));
|
||||
GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2);
|
||||
GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical);
|
||||
|
||||
struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
|
||||
GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4);
|
||||
GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4);
|
||||
GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2);
|
||||
GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical);
|
||||
|
||||
dense_in = ggml_reshape_2d(model.ctx_compute,
|
||||
ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)),
|
||||
(MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical);
|
||||
GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
|
||||
GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical);
|
||||
GGML_ASSERT(dense_in->ne[2] == 1);
|
||||
GGML_ASSERT(dense_in->ne[3] == 1);
|
||||
|
||||
model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias);
|
||||
} else {
|
||||
GGML_ASSERT(false);
|
||||
}
|
||||
|
||||
ggml_set_name(model.logits, "logits");
|
||||
ggml_set_output(model.logits);
|
||||
GGML_ASSERT(model.logits->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES);
|
||||
GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical);
|
||||
GGML_ASSERT(model.logits->ne[2] == 1);
|
||||
GGML_ASSERT(model.logits->ne[3] == 1);
|
||||
}
|
||||
|
||||
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) {
|
||||
ggml_opt_result_t result = ggml_opt_result_init();
|
||||
|
||||
ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
|
||||
params.ctx_compute = model.ctx_compute;
|
||||
params.inputs = model.images;
|
||||
params.outputs = model.logits;
|
||||
params.build_type = GGML_OPT_BUILD_TYPE_FORWARD;
|
||||
ggml_opt_context_t opt_ctx = ggml_opt_init(params);
|
||||
|
||||
{
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr);
|
||||
|
||||
const int64_t t_total_us = ggml_time_us() - t_start_us;
|
||||
const double t_total_ms = 1e-3*t_total_us;
|
||||
const int nex = ggml_opt_dataset_data(dataset)->ne[1];
|
||||
fprintf(stderr, "%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\n",
|
||||
__func__, nex, t_total_ms, (double) t_total_us/nex);
|
||||
}
|
||||
|
||||
ggml_opt_free(opt_ctx);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) {
|
||||
ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset,
|
||||
GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false);
|
||||
}
|
||||
|
||||
void mnist_model_save(mnist_model & model, const std::string & fname) {
|
||||
printf("%s: saving model to '%s'\n", __func__, fname.c_str());
|
||||
|
||||
struct ggml_context * ggml_ctx;
|
||||
{
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ 100 * 1024*1024,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ false,
|
||||
};
|
||||
ggml_ctx = ggml_init(params);
|
||||
}
|
||||
|
||||
gguf_context * gguf_ctx = gguf_init_empty();
|
||||
gguf_set_val_str(gguf_ctx, "general.architecture", model.arch.c_str());
|
||||
|
||||
std::vector<struct ggml_tensor *> weights;
|
||||
if (model.arch == "mnist-fc") {
|
||||
weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias};
|
||||
} else if (model.arch == "mnist-cnn") {
|
||||
weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias};
|
||||
} else {
|
||||
GGML_ASSERT(false);
|
||||
}
|
||||
for (struct ggml_tensor * t : weights) {
|
||||
struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t);
|
||||
ggml_set_name(copy, t->name);
|
||||
ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t));
|
||||
gguf_add_tensor(gguf_ctx, copy);
|
||||
}
|
||||
gguf_write_to_file(gguf_ctx, fname.c_str(), false);
|
||||
|
||||
ggml_free(ggml_ctx);
|
||||
gguf_free(gguf_ctx);
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
int wasm_eval(uint8_t * digitPtr) {
|
||||
std::vector<float> digit(digitPtr, digitPtr + MNIST_NINPUT);
|
||||
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1);
|
||||
struct ggml_tensor * data = ggml_opt_dataset_data(dataset);
|
||||
|
||||
float * buf = ggml_get_data_f32(data);
|
||||
for (int i = 0; i < MNIST_NINPUT; ++i) {
|
||||
buf[i] = digitPtr[i] / 255.0f;
|
||||
}
|
||||
ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed.
|
||||
|
||||
mnist_model model = mnist_model_init_from_file("mnist-f32.gguf", "CPU", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1);
|
||||
mnist_model_build(model);
|
||||
ggml_opt_result_t result = mnist_model_eval(model, dataset);
|
||||
|
||||
int32_t pred;
|
||||
ggml_opt_result_pred(result, &pred);
|
||||
|
||||
return pred;
|
||||
}
|
||||
|
||||
int wasm_random_digit(char * digitPtr) {
|
||||
auto fin = std::ifstream("t10k-images-idx3-ubyte", std::ios::binary);
|
||||
if (!fin) {
|
||||
fprintf(stderr, "failed to open digits file\n");
|
||||
return 0;
|
||||
}
|
||||
srand(time(NULL));
|
||||
|
||||
// Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000)
|
||||
fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST));
|
||||
fin.read(digitPtr, MNIST_NINPUT);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,166 @@
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml.h"
|
||||
#include "gguf.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#define MNIST_NTRAIN 60000
|
||||
#define MNIST_NTEST 10000
|
||||
|
||||
// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one.
|
||||
// The logical batch size determines how many datapoints are used for a gradient update.
|
||||
// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory.
|
||||
#define MNIST_NBATCH_LOGICAL 1000
|
||||
#define MNIST_NBATCH_PHYSICAL 500
|
||||
|
||||
static_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, "MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0");
|
||||
static_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
|
||||
static_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
|
||||
|
||||
#define MNIST_HW 28
|
||||
#define MNIST_NINPUT (MNIST_HW*MNIST_HW)
|
||||
#define MNIST_NCLASSES 10
|
||||
|
||||
#define MNIST_NHIDDEN 500
|
||||
|
||||
// NCB = number of channels base
|
||||
#define MNIST_CNN_NCB 8
|
||||
|
||||
struct mnist_model {
|
||||
std::string arch;
|
||||
ggml_backend_sched_t backend_sched;
|
||||
std::vector<ggml_backend_t> backends;
|
||||
const int nbatch_logical;
|
||||
const int nbatch_physical;
|
||||
|
||||
struct ggml_tensor * images = nullptr;
|
||||
struct ggml_tensor * logits = nullptr;
|
||||
|
||||
struct ggml_tensor * fc1_weight = nullptr;
|
||||
struct ggml_tensor * fc1_bias = nullptr;
|
||||
struct ggml_tensor * fc2_weight = nullptr;
|
||||
struct ggml_tensor * fc2_bias = nullptr;
|
||||
|
||||
struct ggml_tensor * conv1_kernel = nullptr;
|
||||
struct ggml_tensor * conv1_bias = nullptr;
|
||||
struct ggml_tensor * conv2_kernel = nullptr;
|
||||
struct ggml_tensor * conv2_bias = nullptr;
|
||||
struct ggml_tensor * dense_weight = nullptr;
|
||||
struct ggml_tensor * dense_bias = nullptr;
|
||||
|
||||
struct ggml_context * ctx_gguf = nullptr;
|
||||
struct ggml_context * ctx_static = nullptr;
|
||||
struct ggml_context * ctx_compute = nullptr;
|
||||
ggml_backend_buffer_t buf_gguf = nullptr;
|
||||
ggml_backend_buffer_t buf_static = nullptr;
|
||||
|
||||
mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical)
|
||||
: nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) {
|
||||
std::vector<ggml_backend_dev_t> devices;
|
||||
const int ncores_logical = std::thread::hardware_concurrency();
|
||||
const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2);
|
||||
|
||||
// Add primary backend:
|
||||
if (!backend_name.empty()) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str());
|
||||
if (dev == nullptr) {
|
||||
fprintf(stderr, "%s: ERROR: backend %s not found, available:\n", __func__, backend_name.c_str());
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev_i = ggml_backend_dev_get(i);
|
||||
fprintf(stderr, " - %s (%s)\n", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i));
|
||||
}
|
||||
exit(1);
|
||||
}
|
||||
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
||||
GGML_ASSERT(backend);
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, nthreads);
|
||||
}
|
||||
|
||||
backends.push_back(backend);
|
||||
devices.push_back(dev);
|
||||
}
|
||||
|
||||
// Add all available backends as fallback.
|
||||
// A "backend" is a stream on a physical device so there is no problem with adding multiple backends for the same device.
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
|
||||
GGML_ASSERT(backend);
|
||||
|
||||
if (ggml_backend_is_cpu(backend)) {
|
||||
ggml_backend_cpu_set_n_threads(backend, nthreads);
|
||||
}
|
||||
|
||||
backends.push_back(backend);
|
||||
devices.push_back(dev);
|
||||
}
|
||||
|
||||
// The order of the backends passed to ggml_backend_sched_new determines which backend is given priority.
|
||||
backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);
|
||||
fprintf(stderr, "%s: using %s (%s) as primary backend\n",
|
||||
__func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0]));
|
||||
if (backends.size() >= 2) {
|
||||
fprintf(stderr, "%s: unsupported operations will be executed on the following fallback backends (in order of priority):\n", __func__);
|
||||
for (size_t i = 1; i < backends.size(); ++i) {
|
||||
fprintf(stderr, "%s: - %s (%s)\n", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i]));
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const size_t size_meta = 1024*ggml_tensor_overhead();
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ size_meta,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_static = ggml_init(params);
|
||||
}
|
||||
|
||||
{
|
||||
// The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step).
|
||||
const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead();
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ size_meta,
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_compute = ggml_init(params);
|
||||
}
|
||||
}
|
||||
|
||||
~mnist_model() {
|
||||
ggml_free(ctx_gguf);
|
||||
ggml_free(ctx_static);
|
||||
ggml_free(ctx_compute);
|
||||
|
||||
ggml_backend_buffer_free(buf_gguf);
|
||||
ggml_backend_buffer_free(buf_static);
|
||||
ggml_backend_sched_free(backend_sched);
|
||||
for (ggml_backend_t backend : backends) {
|
||||
ggml_backend_free(backend);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset);
|
||||
void mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex);
|
||||
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset);
|
||||
|
||||
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
|
||||
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
|
||||
void mnist_model_build(mnist_model & model);
|
||||
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset);
|
||||
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split);
|
||||
void mnist_model_save(mnist_model & model, const std::string & fname);
|
||||
@@ -0,0 +1,67 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-opt.h"
|
||||
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
srand(time(NULL));
|
||||
ggml_time_init();
|
||||
|
||||
if (argc != 4 && argc != 5) {
|
||||
fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);
|
||||
|
||||
if (!mnist_image_load(argv[2], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
if (!mnist_label_load(argv[3], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int iex = rand() % MNIST_NTEST;
|
||||
mnist_image_print(stdout, dataset, iex);
|
||||
|
||||
const std::string backend = argc >= 5 ? argv[4] : "";
|
||||
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
|
||||
mnist_model_build(model);
|
||||
const int64_t t_load_us = ggml_time_us() - t_start_us;
|
||||
fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0);
|
||||
|
||||
ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);
|
||||
|
||||
std::vector<int32_t> pred(MNIST_NTEST);
|
||||
ggml_opt_result_pred(result_eval, pred.data());
|
||||
fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]);
|
||||
|
||||
double loss;
|
||||
double loss_unc;
|
||||
ggml_opt_result_loss(result_eval, &loss, &loss_unc);
|
||||
fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc);
|
||||
|
||||
double accuracy;
|
||||
double accuracy_unc;
|
||||
ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);
|
||||
fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc);
|
||||
|
||||
ggml_opt_result_free(result_eval);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from time import time
|
||||
import gguf
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
|
||||
|
||||
def train(model_path):
|
||||
# Model / data parameters
|
||||
num_classes = 10
|
||||
input_shape = (28, 28, 1)
|
||||
|
||||
# Load the data and split it between train and test sets
|
||||
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
|
||||
|
||||
# Scale images to the [0, 1] range
|
||||
x_train = x_train.astype("float32") / 255
|
||||
x_test = x_test.astype("float32") / 255
|
||||
x_train = np.expand_dims(x_train, -1)
|
||||
x_test = np.expand_dims(x_test, -1)
|
||||
print("x_train shape:", x_train.shape)
|
||||
print(x_train.shape[0], "train samples")
|
||||
print(x_test.shape[0], "test samples")
|
||||
|
||||
# convert class vectors to binary class matrices
|
||||
y_train = keras.utils.to_categorical(y_train, num_classes)
|
||||
y_test = keras.utils.to_categorical(y_test, num_classes)
|
||||
|
||||
model = keras.Sequential(
|
||||
[
|
||||
keras.Input(shape=input_shape, dtype=tf.float32),
|
||||
layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
|
||||
layers.MaxPooling2D(pool_size=(2, 2)),
|
||||
layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
|
||||
layers.MaxPooling2D(pool_size=(2, 2)),
|
||||
layers.Flatten(),
|
||||
layers.Dense(num_classes, activation="softmax", dtype=tf.float32),
|
||||
]
|
||||
)
|
||||
|
||||
model.summary()
|
||||
batch_size = 1000
|
||||
epochs = 30
|
||||
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
|
||||
|
||||
t_start = time()
|
||||
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
|
||||
print(f"Training took {time()-t_start:.2f}s")
|
||||
|
||||
score = model.evaluate(x_test, y_test, verbose=0)
|
||||
print(f"Test loss: {score[0]:.6f}")
|
||||
print(f"Test accuracy: {100*score[1]:.2f}%")
|
||||
|
||||
gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn")
|
||||
|
||||
conv1_kernel = model.layers[0].weights[0].numpy()
|
||||
conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])
|
||||
gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3))
|
||||
|
||||
conv1_bias = model.layers[0].weights[1].numpy()
|
||||
gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1))
|
||||
|
||||
conv2_kernel = model.layers[2].weights[0].numpy()
|
||||
conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])
|
||||
gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3))
|
||||
|
||||
conv2_bias = model.layers[2].weights[1].numpy()
|
||||
gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1))
|
||||
|
||||
dense_weight = model.layers[-1].weights[0].numpy()
|
||||
dense_weight = dense_weight.transpose()
|
||||
gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16))
|
||||
|
||||
dense_bias = model.layers[-1].weights[1].numpy()
|
||||
gguf_writer.add_tensor("dense.bias", dense_bias)
|
||||
|
||||
gguf_writer.write_header_to_file()
|
||||
gguf_writer.write_kv_data_to_file()
|
||||
gguf_writer.write_tensors_to_file()
|
||||
gguf_writer.close()
|
||||
print(f"GGUF model saved to '{model_path}'")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <model_path>")
|
||||
sys.exit(1)
|
||||
train(sys.argv[1])
|
||||
@@ -0,0 +1,131 @@
|
||||
import gguf
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torchvision.datasets as dsets
|
||||
import torchvision.transforms as transforms
|
||||
from torch.autograd import Variable
|
||||
|
||||
import sys
|
||||
from time import time
|
||||
|
||||
input_size = 784 # img_size = (28,28) ---> 28*28=784 in total
|
||||
hidden_size = 500 # number of nodes at hidden layer
|
||||
num_classes = 10 # number of output classes discrete range [0,9]
|
||||
num_epochs = 30 # number of times which the entire dataset is passed throughout the model
|
||||
batch_size = 1000 # the size of input data used for one iteration
|
||||
lr = 1e-3 # size of step
|
||||
|
||||
|
||||
class Net(nn.Module):
|
||||
def __init__(self, input_size, hidden_size, num_classes):
|
||||
super(Net, self).__init__()
|
||||
self.fc1 = nn.Linear(input_size, hidden_size)
|
||||
self.relu = nn.ReLU()
|
||||
self.fc2 = nn.Linear(hidden_size, num_classes)
|
||||
|
||||
def forward(self, x):
|
||||
out = self.fc1(x)
|
||||
out = self.relu(out)
|
||||
out = self.fc2(out)
|
||||
return out
|
||||
|
||||
|
||||
def train(model_path):
|
||||
train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)
|
||||
test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())
|
||||
|
||||
assert len(train_data) == 60000
|
||||
assert len(test_data) == 10000
|
||||
|
||||
kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)
|
||||
train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test)
|
||||
test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test)
|
||||
|
||||
net = Net(input_size, hidden_size, num_classes)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
net.cuda()
|
||||
|
||||
loss_function = nn.CrossEntropyLoss()
|
||||
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
|
||||
|
||||
t_start = time()
|
||||
for epoch in range(num_epochs):
|
||||
loss_history = []
|
||||
ncorrect = 0
|
||||
|
||||
for i, (images, labels) in enumerate(train_gen):
|
||||
images = Variable(images.view(-1, 28*28))
|
||||
labels = Variable(labels)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
images = images.cuda()
|
||||
labels = labels.cuda()
|
||||
|
||||
optimizer.zero_grad()
|
||||
outputs = net(images)
|
||||
loss = loss_function(outputs, labels)
|
||||
|
||||
loss_history.append(loss.cpu().data)
|
||||
_, predictions = torch.max(outputs, 1)
|
||||
ncorrect += (predictions == labels).sum()
|
||||
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
if (i + 1)*batch_size % 10000 == 0:
|
||||
loss_mean = np.mean(loss_history)
|
||||
accuracy = ncorrect / ((i + 1) * batch_size)
|
||||
print(
|
||||
f"Epoch [{epoch+1:02d}/{num_epochs}], "
|
||||
f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], "
|
||||
f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%")
|
||||
print()
|
||||
print(f"Training took {time()-t_start:.2f}s")
|
||||
|
||||
loss_history = []
|
||||
ncorrect = 0
|
||||
|
||||
for i, (images, labels) in enumerate(test_gen):
|
||||
images = Variable(images.view(-1, 28*28))
|
||||
labels = Variable(labels)
|
||||
|
||||
if torch.cuda.is_available():
|
||||
images = images.cuda()
|
||||
labels = labels.cuda()
|
||||
|
||||
outputs = net(images)
|
||||
loss = loss_function(outputs, labels)
|
||||
|
||||
loss_history.append(loss.cpu().data)
|
||||
_, predictions = torch.max(outputs, 1)
|
||||
ncorrect += (predictions == labels).sum().cpu().numpy()
|
||||
|
||||
loss_mean = np.mean(loss_history)
|
||||
loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1)
|
||||
accuracy_mean = ncorrect / (len(test_gen) * batch_size)
|
||||
accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))
|
||||
print()
|
||||
print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%")
|
||||
|
||||
gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc")
|
||||
|
||||
print()
|
||||
print(f"Model tensors saved to {model_path}:")
|
||||
for tensor_name in net.state_dict().keys():
|
||||
data = net.state_dict()[tensor_name].squeeze().cpu().numpy()
|
||||
print(tensor_name, "\t", data.shape)
|
||||
gguf_writer.add_tensor(tensor_name, data)
|
||||
|
||||
gguf_writer.write_header_to_file()
|
||||
gguf_writer.write_kv_data_to_file()
|
||||
gguf_writer.write_tensors_to_file()
|
||||
gguf_writer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <model_path>")
|
||||
sys.exit(1)
|
||||
train(sys.argv[1])
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "ggml-opt.h"
|
||||
#include "mnist-common.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
#include <string>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(disable: 4244 4267) // possible loss of data
|
||||
#endif
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc != 5 && argc != 6) {
|
||||
fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.
|
||||
// With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).
|
||||
// A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.
|
||||
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);
|
||||
|
||||
if (!mnist_image_load(argv[3], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
if (!mnist_label_load(argv[4], dataset)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
|
||||
|
||||
mnist_model_build(model);
|
||||
|
||||
mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);
|
||||
|
||||
mnist_model_save(model, argv[2]);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import http.server
|
||||
import socketserver
|
||||
import os
|
||||
import sys
|
||||
|
||||
DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))
|
||||
PORT = 8000
|
||||
|
||||
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, directory=DIRECTORY, **kwargs)
|
||||
|
||||
def end_headers(self):
|
||||
# Add required headers for SharedArrayBuffer
|
||||
self.send_header("Cross-Origin-Opener-Policy", "same-origin")
|
||||
self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
super().end_headers()
|
||||
|
||||
# Enable address reuse
|
||||
class CustomServer(socketserver.TCPServer):
|
||||
allow_reuse_address = True
|
||||
|
||||
try:
|
||||
with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
|
||||
print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
|
||||
print(f"Application context root: http://localhost:{PORT}/")
|
||||
try:
|
||||
httpd.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\nServer stopped.")
|
||||
# Force complete exit
|
||||
sys.exit(0)
|
||||
except OSError as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,7 @@
|
||||
#
|
||||
# perf-metal
|
||||
|
||||
set(TEST_TARGET perf-metal)
|
||||
add_executable(${TEST_TARGET} perf-metal.cpp)
|
||||
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
// basic tool to experiment with the Metal backend
|
||||
//
|
||||
// 1. Get GPU trace of a dummy graph:
|
||||
//
|
||||
// rm -rf /tmp/perf-metal.gputrace
|
||||
// make -j perf-metal && METAL_CAPTURE_ENABLED=1 ./bin/perf-metal
|
||||
// open /tmp/perf-metal.gputrace
|
||||
//
|
||||
// https://github.com/ggerganov/llama.cpp/issues/9507
|
||||
//
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-metal.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <vector>
|
||||
#include <thread>
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
int n_op = 1024;
|
||||
int n_iter = 128;
|
||||
|
||||
if (argc > 1) {
|
||||
n_op = std::atoi(argv[1]);
|
||||
}
|
||||
|
||||
if (argc > 2) {
|
||||
n_iter = std::atoi(argv[2]);
|
||||
}
|
||||
|
||||
printf("%s: n_op = %d, n_iter = %d\n", __func__, n_op, n_iter);
|
||||
|
||||
const int ne00 = 8;
|
||||
const int ne01 = 8;
|
||||
const int ne11 = 8;
|
||||
|
||||
std::vector<float> data0(ne00*ne01, 1.0f);
|
||||
std::vector<float> data1(ne00*ne01, 1.0f/ne00);
|
||||
|
||||
ggml_backend_t backend = ggml_backend_metal_init();
|
||||
if (!backend) {
|
||||
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const size_t ctx_size = 2 * ggml_tensor_overhead();
|
||||
|
||||
struct ggml_init_params params = {
|
||||
/*.mem_size =*/ ctx_size,
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
|
||||
struct ggml_tensor * t0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne01);
|
||||
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne11);
|
||||
|
||||
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);
|
||||
|
||||
ggml_backend_tensor_set(t0, data0.data(), 0, ggml_nbytes(t0));
|
||||
ggml_backend_tensor_set(t1, data1.data(), 0, ggml_nbytes(t1));
|
||||
|
||||
struct ggml_cgraph * gf = NULL;
|
||||
|
||||
struct ggml_context * ctx_cgraph = NULL;
|
||||
|
||||
// create a dummy compute graph:
|
||||
//
|
||||
// x = mul_mat(t0, t1)
|
||||
// x = x * 1.0f
|
||||
// x = mul_mat(x, t1)
|
||||
// x = x * 1.0f
|
||||
// ... repeat n_op times ...
|
||||
//
|
||||
{
|
||||
struct ggml_init_params params0 = {
|
||||
/*.mem_size =*/ 4*n_op*ggml_tensor_overhead() + ggml_graph_overhead(),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ctx_cgraph = ggml_init(params0);
|
||||
|
||||
gf = ggml_new_graph_custom(ctx_cgraph, 4*n_op, false);
|
||||
|
||||
struct ggml_tensor * cur = ggml_mul_mat(ctx_cgraph, t0, t1);
|
||||
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
|
||||
|
||||
for (int i = 0; i < n_op - 1; i++) {
|
||||
cur = ggml_mul_mat(ctx_cgraph, cur, t1);
|
||||
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
|
||||
}
|
||||
|
||||
cur = ggml_scale(ctx_cgraph, cur, 42.0f);
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
}
|
||||
|
||||
printf("%s: graph nodes = %d\n", __func__, ggml_graph_n_nodes(gf));
|
||||
|
||||
ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
|
||||
ggml_gallocr_alloc_graph(allocr, gf);
|
||||
|
||||
{
|
||||
// warm-up
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
for (int iter = 0; iter < n_iter; iter++) {
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
}
|
||||
|
||||
const int64_t t_end = ggml_time_us();
|
||||
|
||||
// actual trace
|
||||
ggml_backend_metal_capture_next_compute(backend);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
|
||||
ggml_backend_metal_capture_next_compute(backend);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
|
||||
ggml_backend_metal_capture_next_compute(backend);
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
printf("%s: time = %f ms\n", __func__, (t_end - t_start) / 1000.0 / n_iter);
|
||||
}
|
||||
|
||||
{
|
||||
struct ggml_tensor * res = ggml_graph_node(gf, -1);
|
||||
|
||||
std::vector<float> data(res->ne[0] * res->ne[1], 0.0f);
|
||||
|
||||
ggml_backend_tensor_get(res, data.data(), 0, ggml_nbytes(res));
|
||||
|
||||
for (int i1 = 0; i1 < res->ne[1]; i1++) {
|
||||
for (int i0 = 0; i0 < res->ne[0]; i0++) {
|
||||
printf("%f ", data[i1*res->ne[0] + i0]);
|
||||
}
|
||||
printf("\n");
|
||||
}
|
||||
}
|
||||
|
||||
ggml_free(ctx_cgraph);
|
||||
ggml_gallocr_free(allocr);
|
||||
ggml_free(ctx);
|
||||
ggml_backend_buffer_free(buffer);
|
||||
ggml_backend_free(backend);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 12092,3645,2
|
||||
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
|
||||
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
|
||||
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
|
||||
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
|
||||
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
|
||||
The book costs $19.99 => 510,1984,4815,370,746,15,1525
|
||||
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
|
||||
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
|
||||
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
|
||||
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
|
||||
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
|
||||
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
|
||||
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
|
||||
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
|
||||
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
|
||||
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
|
||||
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
|
||||
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
|
||||
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
|
||||
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
|
||||
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
|
||||
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
|
||||
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
|
||||
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
|
||||
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
|
||||
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
|
||||
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
|
||||
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
|
||||
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
|
||||
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
|
||||
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
|
||||
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
|
||||
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
|
||||
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
|
||||
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
|
||||
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
|
||||
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
|
||||
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
|
||||
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
|
||||
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
|
||||
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
|
||||
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
|
||||
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
|
||||
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
|
||||
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
|
||||
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
|
||||
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
|
||||
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
|
||||
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
|
||||
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
|
||||
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
|
||||
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
|
||||
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
|
||||
They are renovating their house. => 3726,403,30074,839,616,2419,15
|
||||
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
|
||||
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
|
||||
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
|
||||
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
|
||||
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
|
||||
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
|
||||
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
|
||||
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
|
||||
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
|
||||
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
|
||||
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
|
||||
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
|
||||
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
|
||||
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
|
||||
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
|
||||
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
|
||||
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
|
||||
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
|
||||
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
|
||||
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
|
||||
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
|
||||
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
|
||||
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
|
||||
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
|
||||
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
|
||||
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
|
||||
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
|
||||
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
|
||||
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
|
||||
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
|
||||
The traffic is congest => 510,7137,310,25801
|
||||
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
|
||||
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
|
||||
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
|
||||
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
|
||||
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
|
||||
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
|
||||
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
|
||||
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
|
||||
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
|
||||
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
|
||||
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
|
||||
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
|
||||
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
|
||||
@@ -0,0 +1 @@
|
||||
请问洗手间在哪里？ => 6435,7309,3819,2797,7313,1762,1525,7027,8043
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 15496,2159,0
|
||||
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
|
||||
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
|
||||
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
|
||||
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
|
||||
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
|
||||
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
|
||||
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
|
||||
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
|
||||
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
|
||||
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
|
||||
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
|
||||
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
|
||||
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
|
||||
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
|
||||
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
|
||||
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
|
||||
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
|
||||
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
|
||||
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
|
||||
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
|
||||
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
|
||||
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
|
||||
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
|
||||
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
|
||||
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
|
||||
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
|
||||
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
|
||||
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
|
||||
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
|
||||
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
|
||||
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
|
||||
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
|
||||
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
|
||||
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
|
||||
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
|
||||
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
|
||||
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
|
||||
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
|
||||
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
|
||||
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
|
||||
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
|
||||
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
|
||||
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
|
||||
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
|
||||
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
|
||||
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
|
||||
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
|
||||
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
|
||||
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
|
||||
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
|
||||
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
|
||||
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
|
||||
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
|
||||
They are renovating their house. => 2990,389,24317,803,511,2156,13
|
||||
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
|
||||
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
|
||||
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
|
||||
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
|
||||
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
|
||||
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
|
||||
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
|
||||
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
|
||||
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
|
||||
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
|
||||
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
|
||||
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
|
||||
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
|
||||
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
|
||||
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
|
||||
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
|
||||
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
|
||||
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
|
||||
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
|
||||
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
|
||||
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
|
||||
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
|
||||
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
|
||||
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
|
||||
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
|
||||
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
|
||||
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
|
||||
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
|
||||
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
|
||||
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
|
||||
The traffic is congest => 464,4979,318,22791
|
||||
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
|
||||
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
|
||||
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
|
||||
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
|
||||
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
|
||||
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
|
||||
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
|
||||
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
|
||||
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
|
||||
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
|
||||
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
|
||||
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
|
||||
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 15496,2159,0
|
||||
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
|
||||
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
|
||||
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
|
||||
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
|
||||
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
|
||||
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
|
||||
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
|
||||
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
|
||||
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
|
||||
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
|
||||
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
|
||||
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
|
||||
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
|
||||
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
|
||||
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
|
||||
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
|
||||
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
|
||||
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
|
||||
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
|
||||
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
|
||||
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
|
||||
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
|
||||
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
|
||||
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
|
||||
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
|
||||
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
|
||||
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
|
||||
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
|
||||
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
|
||||
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
|
||||
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
|
||||
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
|
||||
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
|
||||
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
|
||||
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
|
||||
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
|
||||
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
|
||||
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
|
||||
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
|
||||
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
|
||||
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
|
||||
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
|
||||
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
|
||||
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
|
||||
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
|
||||
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
|
||||
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
|
||||
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
|
||||
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
|
||||
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
|
||||
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
|
||||
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
|
||||
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
|
||||
They are renovating their house. => 2990,389,24317,803,511,2156,13
|
||||
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
|
||||
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
|
||||
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
|
||||
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
|
||||
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
|
||||
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
|
||||
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
|
||||
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
|
||||
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
|
||||
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
|
||||
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
|
||||
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
|
||||
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
|
||||
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
|
||||
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
|
||||
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
|
||||
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
|
||||
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
|
||||
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
|
||||
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
|
||||
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
|
||||
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
|
||||
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
|
||||
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
|
||||
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
|
||||
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
|
||||
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
|
||||
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
|
||||
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
|
||||
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
|
||||
The traffic is congest => 464,4979,318,22791
|
||||
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
|
||||
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
|
||||
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
|
||||
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
|
||||
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
|
||||
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
|
||||
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
|
||||
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
|
||||
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
|
||||
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
|
||||
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
|
||||
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
|
||||
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
|
||||
@@ -0,0 +1 @@
|
||||
明日の天気はどうですか。 => 263,7353,268,18461,271,1722,18405,265
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 12092,3645,2
|
||||
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
|
||||
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
|
||||
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
|
||||
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
|
||||
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
|
||||
The book costs $19.99 => 510,1984,4815,370,746,15,1525
|
||||
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
|
||||
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
|
||||
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
|
||||
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
|
||||
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
|
||||
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
|
||||
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
|
||||
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
|
||||
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
|
||||
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
|
||||
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
|
||||
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
|
||||
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
|
||||
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
|
||||
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
|
||||
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
|
||||
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
|
||||
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
|
||||
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
|
||||
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
|
||||
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
|
||||
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
|
||||
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
|
||||
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
|
||||
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
|
||||
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
|
||||
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
|
||||
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
|
||||
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
|
||||
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
|
||||
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
|
||||
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
|
||||
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
|
||||
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
|
||||
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
|
||||
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
|
||||
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
|
||||
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
|
||||
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
|
||||
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
|
||||
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
|
||||
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
|
||||
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
|
||||
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
|
||||
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
|
||||
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
|
||||
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
|
||||
They are renovating their house. => 3726,403,30074,839,616,2419,15
|
||||
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
|
||||
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
|
||||
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
|
||||
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
|
||||
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
|
||||
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
|
||||
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
|
||||
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
|
||||
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
|
||||
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
|
||||
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
|
||||
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
|
||||
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
|
||||
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
|
||||
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
|
||||
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
|
||||
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
|
||||
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
|
||||
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
|
||||
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
|
||||
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
|
||||
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
|
||||
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
|
||||
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
|
||||
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
|
||||
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
|
||||
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
|
||||
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
|
||||
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
|
||||
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
|
||||
The traffic is congest => 510,7137,310,25801
|
||||
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
|
||||
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
|
||||
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
|
||||
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
|
||||
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
|
||||
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
|
||||
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
|
||||
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
|
||||
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
|
||||
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
|
||||
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
|
||||
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
|
||||
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
|
||||
@@ -0,0 +1,3 @@
|
||||
이것은 테스트 이다. => 12271,296,6474,28037,17
|
||||
걱정할 필요 없다. => 18311,482,1062,550,267,17
|
||||
버그는 언젠가 고쳐진다. => 6904,272,8575,10381,1765,17
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 6466,147,2317,350
|
||||
I can't believe it's already Friday!" => 286,512,172,185,13392,393,172,155,3239,147,29249,8537
|
||||
The URL for the website is https://www.example.com." => 505,5635,250,170,11745,235,147,303,262,552,148,811,148,241,148,161
|
||||
"She said, 'I love to travel.'" => 161,10386,4089,150,206,286,8440,194,147,12363,148,172,161
|
||||
'The temperature is 25.5°C.' => 172,505,147,9502,235,147,20022,8516,228,148,172
|
||||
"Let's meet at 2:30 p.m. in the park." => 161,8997,172,155,17120,536,147,162,5245,147,207,148,204,148,219,170,147,17664,148,161
|
||||
The book costs $19.99 => 505,147,2277,17494,236,166,11824
|
||||
"John's favorite color is blue." => 161,7475,172,155,147,11105,147,349,235,17046,148,161
|
||||
Th@nk y0u f0r y0ur h3lp! => 6309,240,9019,147,237,159,247,147,202,159,223,147,237,159,2458,147,226,171,3899,350
|
||||
C@n I g3t a c0ffee, pl3@se? => 228,240,211,398,147,267,171,185,216,147,196,159,13360,163,150,147,1287,171,240,155,163,272
|
||||
W0w! Th@t's @m@zing! => 450,159,274,350,147,6309,240,185,172,155,268,204,240,301,248,350
|
||||
H0w 4re y0u t0d@y? => 304,159,274,320,440,147,237,159,247,147,185,159,182,240,237,272
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 286,997,159,1290,147,185,159,147,490,240,3893,268,223,159,3981,170,147,274,159,223,2833,148
|
||||
Wh@t's y0ur f@v0rite m0vie? => 450,226,240,185,172,155,147,237,159,2458,147,202,240,252,159,5961,163,147,204,159,24373,272
|
||||
The cat is sleeping on the mat. => 505,147,1604,235,147,3987,248,347,170,147,1297,148
|
||||
I need to buy some groceries for dinner. => 286,1645,194,147,8068,1499,147,10022,1037,10023,250,147,182,2749,148
|
||||
The sun is shining brightly in the sky. => 505,147,5852,235,147,7304,2967,147,215,649,391,219,170,147,7310,148
|
||||
She is reading a book in the park. => 10386,235,9838,216,147,2277,219,170,147,17664,148
|
||||
We went for a walk on the beach yesterday. => 3250,10825,250,216,147,8156,347,170,294,5371,147,28830,148
|
||||
He plays the guitar like a pro. => 5301,7084,155,170,147,4604,2214,1425,216,3474,148
|
||||
They are going to the movies tonight. => 18815,429,6552,194,170,147,15877,194,7907,148
|
||||
The flowers are blooming in the garden. => 505,147,22953,155,429,147,10411,2799,248,219,170,147,22140,148
|
||||
I enjoy listening to classical music. => 286,23162,15876,248,194,239,4251,147,7395,148
|
||||
We need to buy groceries for the week. => 3250,1645,194,147,8068,147,10022,1037,10023,250,170,9238,148
|
||||
The dog is chasing its tail in circles. => 505,147,6540,235,147,196,916,248,1602,147,5129,219,147,4095,155,148
|
||||
She is wearing a beautiful red dress. => 10386,235,147,16427,248,216,147,23447,147,1160,147,14592,148
|
||||
He is a talented actor in Hollywood. => 5301,235,216,147,29750,246,147,5112,219,147,16924,391,10477,148
|
||||
The children are playing in the playground. => 505,7934,429,7084,248,219,170,7084,12055,148
|
||||
I'm going to visit my grandparents this weekend. => 286,172,204,6552,194,9939,1247,147,11806,12019,291,9238,314,148
|
||||
The coffee tastes bitter without sugar. => 505,147,21526,147,20931,155,5145,1430,1988,147,28759,148
|
||||
They are planning a surprise party for her. => 18815,429,147,23661,216,147,29240,147,7344,250,1869,148
|
||||
She sings like an angel on stage. => 10386,147,155,6502,1425,426,147,26028,347,12685,148
|
||||
We should take a vacation to relax. => 3250,936,4654,216,147,15388,946,194,1998,2744,148
|
||||
He is studying medicine at the university. => 5301,235,7959,248,147,20742,1668,536,170,147,8025,148
|
||||
The rain is pouring heavily outside. => 505,147,6885,235,5306,248,1189,5451,391,8096,148
|
||||
I enjoy watching romantic movies. => 286,23162,147,3355,248,147,26080,4140,147,15877,148
|
||||
They are celebrating their anniversary today. => 18815,429,147,30000,5841,1669,147,24734,5464,1770,13386,148
|
||||
She dances gracefully to the music. => 10386,147,182,1626,155,147,267,8771,8001,194,170,147,7395,148
|
||||
He is an excellent basketball player. => 5301,235,426,147,12300,675,185,147,26646,5132,6294,148
|
||||
The baby is sleeping soundly in the crib. => 505,147,23597,235,147,3987,248,12642,391,219,170,147,7696,215,148
|
||||
I need to finish my homework before dinner. => 286,1645,194,147,6717,1247,147,1071,2722,2643,147,182,2749,148
|
||||
They are organizing a charity event next month. => 18815,429,147,16442,248,216,1054,1511,1663,2399,12821,148
|
||||
She is cooking a delicious meal for us. => 10386,235,147,20453,248,216,3936,23455,147,26658,250,147,539,148
|
||||
We should go hiking in the mountains. => 3250,936,4242,147,2254,5357,219,170,147,204,18028,155,148
|
||||
The car broke down on the way to work. => 505,7553,147,510,10036,4288,347,170,3699,194,1916,148
|
||||
He loves playing video games in his free time. => 5301,8440,155,7084,248,8722,147,11281,219,1439,4002,801,148
|
||||
The birds are chirping in the trees. => 505,147,13043,155,429,147,3904,223,4639,219,170,5311,155,148
|
||||
I want to learn how to play the piano. => 286,1857,194,14167,2496,194,7084,170,147,207,23635,148
|
||||
They are building a new shopping mall in the city. => 18815,429,11038,216,277,147,22184,147,204,609,219,170,147,2416,148
|
||||
She is writing a novel in her spare time. => 10386,235,3242,216,147,25814,219,1869,6772,2382,801,148
|
||||
We are going to the zoo this Saturday. => 3250,429,6552,194,170,147,25101,291,147,31426,148
|
||||
The cake looks delicious with chocolate frosting. => 505,147,24422,16303,3936,23455,312,147,5619,533,2239,147,202,3973,3431,148
|
||||
He is a talented painter who sells his artwork. => 5301,235,216,147,29750,246,147,9226,279,2888,13004,155,1439,12234,2722,148
|
||||
The students are studying for their exams. => 505,15707,429,7959,248,250,1669,147,12398,155,148
|
||||
I enjoy swimming in the ocean. => 286,23162,147,4729,8528,248,219,170,147,26193,148
|
||||
They are renovating their house. => 18815,429,991,10724,3643,1669,13788,148
|
||||
She is practicing yoga to stay healthy. => 10386,235,147,18453,248,147,5063,1186,194,15344,147,28550,148
|
||||
We should plant flowers in the garden. => 3250,936,147,9212,147,22953,155,219,170,147,22140,148
|
||||
The traffic is heavy during rush hour. => 505,147,11097,235,147,22232,4340,147,22319,147,5686,148
|
||||
He is a skilled chef who creates amazing dishes. => 5301,235,216,147,8891,246,9784,202,2888,13720,147,28880,147,23852,383,148
|
||||
The baby is crawling on the floor. => 505,147,23597,235,147,22120,248,347,170,147,5895,148
|
||||
I need to buy a new pair of shoes. => 286,1645,194,147,8068,216,277,12632,210,147,155,21953,155,148
|
||||
They are going on a road trip across the country. => 18815,429,6552,347,216,147,6362,147,11395,9762,170,11305,148
|
||||
She is playing the piano beautifully. => 10386,235,7084,248,170,147,207,23635,147,23447,391,148
|
||||
We are going to a concert tomorrow night. => 3250,429,6552,194,216,1710,4391,29524,12716,148
|
||||
The cake tastes delicious with vanilla frosting. => 505,147,24422,147,20931,155,3936,23455,312,5535,7476,147,202,3973,3431,148
|
||||
He is a dedicated teacher who inspires his students. => 5301,235,216,326,8298,3460,147,9675,2888,147,28801,155,1439,15707,148
|
||||
The students are participating in a science fair. => 505,15707,429,147,30961,3643,219,216,147,10587,147,7636,148
|
||||
I enjoy hiking in the mountains. => 286,23162,147,2254,5357,219,170,147,204,18028,155,148
|
||||
They are organizing a beach cleanup next weekend. => 18815,429,147,16442,248,216,294,5371,147,10401,2399,9238,314,148
|
||||
She is taking photographs of nature. => 10386,235,147,12345,147,4709,1547,155,210,147,211,8603,148
|
||||
We should try a new restaurant in town. => 3250,936,147,746,216,277,147,11007,219,147,10200,148
|
||||
The traffic is moving slowly on the highway. => 505,147,11097,235,147,8601,147,9880,391,347,170,5976,3330,148
|
||||
He is a talented singer with a beautiful voice. => 5301,235,216,147,29750,246,147,155,248,279,312,216,147,23447,147,9316,148
|
||||
The baby is laughing and giggling. => 505,147,23597,235,147,23066,248,221,147,2341,3631,2869,148
|
||||
I need to do laundry and wash my clothes. => 286,1645,194,543,960,3981,2154,221,147,27589,1247,147,22141,383,148
|
||||
They are planning a trip to Europe. => 18815,429,147,23661,216,147,11395,194,13131,148
|
||||
She is learning how to play the guitar. => 10386,235,11754,2496,194,7084,170,147,4604,2214,148
|
||||
We are going to a museum this Sunday. => 3250,429,6552,194,216,147,204,433,1177,291,147,29111,148
|
||||
The coffee smells amazing in the morning. => 505,147,21526,31454,155,147,28880,219,170,20701,148
|
||||
He is a hardworking farmer who grows crops. => 5301,235,216,8524,14992,147,16679,279,2888,147,6044,155,147,8650,155,148
|
||||
The students are presenting their research projects. => 505,15707,429,5130,248,1669,13217,14235,148
|
||||
I enjoy playing soccer with my friends. => 286,23162,7084,248,147,9351,5318,312,1247,147,5347,155,148
|
||||
They are volunteering at a local shelter. => 18815,429,147,5238,7478,163,12798,536,216,2491,2905,1359,279,148
|
||||
She is practicing martial arts for self-defense. => 10386,235,147,18453,248,147,3261,185,4381,12234,155,250,623,153,29896,148
|
||||
We should try a new recipe for dinner. => 3250,936,147,746,216,277,147,9851,250,147,182,2749,148
|
||||
The traffic is congest => 505,147,11097,235,1710,14169
|
||||
The sun is shining brightly today. => 505,147,5852,235,147,7304,2967,147,215,649,391,13386,148
|
||||
I enjoy reading books in my free time. => 286,23162,9838,147,9670,219,1247,4002,801,148
|
||||
She plays the piano beautifully. => 10386,7084,155,170,147,207,23635,147,23447,391,148
|
||||
The cat chased the mouse around the room. => 505,147,1604,147,196,916,246,170,12551,6890,170,9654,148
|
||||
I love eating pizza with extra cheese. => 286,8440,147,163,3643,147,207,8403,312,8230,9784,383,163,148
|
||||
He always wears a hat wherever he goes. => 5301,5418,147,16427,155,216,147,4879,2171,2433,1189,16177,148
|
||||
The flowers in the garden are blooming. => 505,147,22953,155,219,170,147,22140,429,147,10411,2799,248,148
|
||||
She danced gracefully on the stage. => 10386,13378,12408,147,267,8771,8001,347,170,12685,148
|
||||
The dog barked loudly in the park. => 505,147,6540,147,973,293,246,147,30182,391,219,170,147,17664,148
|
||||
We went swimming in the ocean yesterday. => 3250,10825,147,4729,8528,248,219,170,147,26193,147,28830,148
|
||||
He speaks fluent French and Spanish. => 5301,147,13285,155,147,21677,147,254,17590,221,147,31519,148
|
||||
The train arrived at the station on time. => 505,147,872,147,20712,182,536,170,147,7184,347,801,148
|
||||
She cooked a delicious meal for her family. => 10386,147,20453,246,216,3936,23455,147,26658,250,1869,147,2002,148
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 8279,10896,19
|
||||
I can't believe it's already Friday!" => 59,883,1330,13710,561,1182,3425,506,25674,11555
|
||||
The URL for the website is https://www.example.com." => 1318,3834,436,322,9575,438,1678,555,1499,32,2763,32,508,3107
|
||||
"She said, 'I love to travel.'" => 20,25387,9884,30,330,59,14290,372,25283,29329
|
||||
'The temperature is 25.5°C.' => 25,1318,13587,438,225,36,39,32,39,23767,53,4564
|
||||
"Let's meet at 2:30 p.m. in the park." => 20,9809,1182,18450,821,225,36,44,37,34,298,32,95,32,328,322,880,93,3107
|
||||
The book costs $19.99 => 1318,7618,25950,398,35,43,32,43,43
|
||||
"John's favorite color is blue." => 20,19693,1182,27448,1963,438,10087,3107
|
||||
Th@nk y0u f0r y0ur h3lp! => 1027,50,19877,533,34,103,296,34,100,533,34,305,420,37,1915,19
|
||||
C@n I g3t a c0ffee, pl3@se? => 53,50,96,439,485,37,102,312,281,34,21298,30,1278,37,50,277,49
|
||||
W0w! Th@t's @m@zing! => 73,34,105,19,947,50,102,1182,477,95,50,26768,19
|
||||
H0w 4re y0u t0d@y? => 58,34,105,225,38,268,533,34,103,273,34,86,50,107,49
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 59,456,34,587,273,34,554,50,1203,477,100,34,642,322,341,34,100,1381,32
|
||||
Wh@t's y0ur f@v0rite m0vie? => 2444,50,102,1182,533,34,305,296,50,104,34,1049,345,34,104,1075,49
|
||||
The cat is sleeping on the mat. => 1318,10501,438,9368,299,544,322,2491,32
|
||||
I need to buy some groceries for dinner. => 59,1849,372,16968,1629,20234,85,6958,436,343,3369,32
|
||||
The sun is shining brightly in the sky. => 1318,15323,438,787,19068,38231,631,328,322,26718,32
|
||||
She is reading a book in the park. => 25387,438,9175,312,7618,328,322,880,93,32
|
||||
We went for a walk on the beach yesterday. => 3122,14236,436,312,13503,544,322,526,867,39485,32
|
||||
He plays the guitar like a pro. => 1331,41271,322,3932,19931,2124,312,534,32
|
||||
They are going to the movies tonight. => 31805,884,6783,372,322,27889,26076,694,32
|
||||
The flowers are blooming in the garden. => 1318,7290,483,884,323,18466,299,328,322,485,22461,32
|
||||
I enjoy listening to classical music. => 59,31567,20498,372,443,1578,17522,32
|
||||
We need to buy groceries for the week. => 3122,1849,372,16968,20234,85,6958,436,322,8209,32
|
||||
The dog is chasing its tail in circles. => 1318,27435,438,663,9949,2819,13203,328,46428,32
|
||||
She is wearing a beautiful red dress. => 25387,438,996,6992,312,36493,3346,343,714,32
|
||||
He is a talented actor in Hollywood. => 1331,438,312,273,9556,318,16038,328,48228,631,21118,32
|
||||
The children are playing in the playground. => 1318,5713,884,19788,328,322,4654,1749,32
|
||||
I'm going to visit my grandparents this weekend. => 59,3464,6783,372,7725,1672,33162,19277,458,40618,32
|
||||
The coffee tastes bitter without sugar. => 1318,36917,273,633,307,3493,391,2876,309,18628,32
|
||||
They are planning a surprise party for her. => 31805,884,26116,312,6178,9251,15270,436,7791,32
|
||||
She sings like an angel on stage. => 25387,309,2052,2124,600,600,17691,544,10019,32
|
||||
We should take a vacation to relax. => 3122,1395,4818,312,29164,367,372,41972,32
|
||||
He is studying medicine at the university. => 1331,438,14866,299,32388,482,821,322,707,9190,32
|
||||
The rain is pouring heavily outside. => 1318,36987,438,9202,299,46003,2801,11127,32
|
||||
I enjoy watching romantic movies. => 59,31567,37652,26045,7268,27889,32
|
||||
They are celebrating their anniversary today. => 31805,884,48278,839,1741,3623,23921,5810,672,11610,32
|
||||
She dances gracefully to the music. => 25387,343,3151,31376,4938,372,322,17522,32
|
||||
He is an excellent basketball player. => 1331,438,600,39203,48400,11653,4362,32
|
||||
The baby is sleeping soundly in the crib. => 1318,323,17156,438,9368,299,9934,631,328,322,281,7972,32
|
||||
I need to finish my homework before dinner. => 59,1849,372,11361,1672,6765,1007,2670,343,3369,32
|
||||
They are organizing a charity event next month. => 31805,884,10558,6183,312,1351,543,1692,2354,6811,32
|
||||
She is cooking a delicious meal for us. => 25387,438,23682,299,312,409,406,2406,597,279,436,1770,32
|
||||
We should go hiking in the mountains. => 3122,1395,1983,420,1546,299,328,322,10874,1907,32
|
||||
The car broke down on the way to work. => 1318,6346,43289,2835,544,322,3352,372,1389,32
|
||||
He loves playing video games in his free time. => 1331,598,4954,19788,6027,19705,328,6697,3741,1133,32
|
||||
The birds are chirping in the trees. => 1318,8424,3210,884,663,476,7075,328,322,23453,32
|
||||
I want to learn how to play the piano. => 59,2637,372,7350,2624,372,4654,322,298,25757,32
|
||||
They are building a new shopping mall in the city. => 31805,884,9038,312,537,40692,345,464,328,322,11297,32
|
||||
She is writing a novel in her spare time. => 25387,438,4127,312,32913,328,7791,1869,586,1133,32
|
||||
We are going to the zoo this Saturday. => 3122,884,6783,372,322,1288,604,458,358,30288,32
|
||||
The cake looks delicious with chocolate frosting. => 1318,281,1062,7780,409,406,2406,623,10408,27589,296,20932,299,32
|
||||
He is a talented painter who sells his artwork. => 1331,438,312,273,9556,318,42300,6560,10800,101,6697,5549,1007,32
|
||||
The students are studying for their exams. => 1318,16512,884,14866,299,436,3623,538,1462,32
|
||||
I enjoy swimming in the ocean. => 59,31567,2535,449,6714,328,322,337,18857,32
|
||||
They are renovating their house. => 31805,884,316,15007,1741,3623,17075,32
|
||||
She is practicing yoga to stay healthy. => 25387,438,11808,11636,533,40067,372,20005,44538,32
|
||||
We should plant flowers in the garden. => 3122,1395,26795,7290,483,328,322,485,22461,32
|
||||
The traffic is heavy during rush hour. => 1318,16391,438,32389,5929,540,1372,12021,32
|
||||
He is a skilled chef who creates amazing dishes. => 1331,438,312,3001,12088,44051,6560,9585,36986,1214,4279,32
|
||||
The baby is crawling on the floor. => 1318,323,17156,438,281,1294,2920,544,322,17648,32
|
||||
I need to buy a new pair of shoes. => 59,1849,372,16968,312,537,6092,432,787,37764,32
|
||||
They are going on a road trip across the country. => 31805,884,6783,544,312,24122,19337,10160,322,10769,32
|
||||
She is playing the piano beautifully. => 25387,438,19788,322,298,25757,526,4846,325,514,107,32
|
||||
We are going to a concert tomorrow night. => 3122,884,6783,372,312,457,6989,31841,19212,32
|
||||
The cake tastes delicious with vanilla frosting. => 1318,281,1062,273,633,307,409,406,2406,623,44653,296,20932,299,32
|
||||
He is a dedicated teacher who inspires his students. => 1331,438,312,23112,30877,6560,26194,8017,6697,16512,32
|
||||
The students are participating in a science fair. => 1318,16512,884,24623,1741,328,312,27536,19375,32
|
||||
I enjoy hiking in the mountains. => 59,31567,420,1546,299,328,322,10874,1907,32
|
||||
They are organizing a beach cleanup next weekend. => 31805,884,10558,6183,312,526,867,13144,2354,40618,32
|
||||
She is taking photographs of nature. => 25387,438,15137,15110,23626,432,24406,32
|
||||
We should try a new restaurant in town. => 3122,1395,1596,312,537,43719,328,38212,32
|
||||
The traffic is moving slowly on the highway. => 1318,16391,438,14089,12899,631,544,322,3857,3073,32
|
||||
He is a talented singer with a beautiful voice. => 1331,438,312,273,9556,318,309,10118,623,312,36493,20309,32
|
||||
The baby is laughing and giggling. => 1318,323,17156,438,2317,2943,299,461,485,365,36088,32
|
||||
I need to do laundry and wash my clothes. => 59,1849,372,745,2317,642,994,461,341,917,1672,7375,46948,32
|
||||
They are planning a trip to Europe. => 31805,884,26116,312,19337,372,27268,32
|
||||
She is learning how to play the guitar. => 25387,438,9608,2624,372,4654,322,3932,19931,32
|
||||
We are going to a museum this Sunday. => 3122,884,6783,372,312,345,539,378,458,358,28036,32
|
||||
The coffee smells amazing in the morning. => 1318,36917,309,42153,101,36986,328,322,33768,32
|
||||
He is a hardworking farmer who grows crops. => 1331,438,312,6784,13578,9019,2302,6560,485,2138,25170,1069,32
|
||||
The students are presenting their research projects. => 1318,16512,884,5024,299,3623,13234,8528,32
|
||||
I enjoy playing soccer with my friends. => 59,31567,19788,22682,10035,623,1672,22523,32
|
||||
They are volunteering at a local shelter. => 31805,884,3920,45585,8637,821,312,2196,309,2542,391,32
|
||||
She is practicing martial arts for self-defense. => 25387,438,11808,11636,345,502,564,5549,101,436,630,31,43694,32
|
||||
We should try a new recipe for dinner. => 3122,1395,1596,312,537,15233,436,343,3369,32
|
||||
The traffic is congest => 1318,16391,438,457,2776
|
||||
The sun is shining brightly today. => 1318,15323,438,787,19068,38231,631,11610,32
|
||||
I enjoy reading books in my free time. => 59,31567,9175,21739,328,1672,3741,1133,32
|
||||
She plays the piano beautifully. => 25387,41271,322,298,25757,526,4846,325,514,107,32
|
||||
The cat chased the mouse around the room. => 1318,10501,663,16109,322,8459,6835,322,8355,32
|
||||
I love eating pizza with extra cheese. => 59,14290,484,1741,47630,623,6717,8277,30315,32
|
||||
He always wears a hat wherever he goes. => 1331,5182,996,4177,312,25793,2154,424,938,13107,32
|
||||
The flowers in the garden are blooming. => 1318,7290,483,328,322,485,22461,884,323,18466,299,32
|
||||
She danced gracefully on the stage. => 25387,343,6087,31376,4938,544,322,10019,32
|
||||
The dog barked loudly in the park. => 1318,27435,323,1087,318,598,836,631,328,322,880,93,32
|
||||
We went swimming in the ocean yesterday. => 3122,14236,2535,449,6714,328,322,337,18857,39485,32
|
||||
He speaks fluent French and Spanish. => 1331,24498,101,38055,43652,461,14911,1708,32
|
||||
The train arrived at the station on time. => 1318,5683,2099,32114,821,322,18662,544,1133,32
|
||||
She cooked a delicious meal for her family. => 25387,23682,318,312,409,406,2406,597,279,436,7791,13872,32
|
||||
@@ -0,0 +1,110 @@
|
||||
# test case format
|
||||
# <language>: <sentence>
|
||||
|
||||
English: Hello World!
|
||||
English: I can't believe it's already Friday!"
|
||||
English: The URL for the website is https://www.example.com."
|
||||
English: "She said, 'I love to travel.'"
|
||||
English: 'The temperature is 25.5°C.'
|
||||
English: "Let's meet at 2:30 p.m. in the park."
|
||||
English: The book costs $19.99
|
||||
English: "John's favorite color is blue."
|
||||
English: Th@nk y0u f0r y0ur h3lp!
|
||||
English: C@n I g3t a c0ffee, pl3@se?
|
||||
English: W0w! Th@t's @m@zing!
|
||||
English: H0w 4re y0u t0d@y?
|
||||
English: I l0ve t0 tr@vel @r0und the w0rld.
|
||||
English: Wh@t's y0ur f@v0rite m0vie?
|
||||
English: The cat is sleeping on the mat.
|
||||
English: I need to buy some groceries for dinner.
|
||||
English: The sun is shining brightly in the sky.
|
||||
English: She is reading a book in the park.
|
||||
English: We went for a walk on the beach yesterday.
|
||||
English: He plays the guitar like a pro.
|
||||
English: They are going to the movies tonight.
|
||||
English: The flowers are blooming in the garden.
|
||||
English: I enjoy listening to classical music.
|
||||
English: We need to buy groceries for the week.
|
||||
English: The dog is chasing its tail in circles.
|
||||
English: She is wearing a beautiful red dress.
|
||||
English: He is a talented actor in Hollywood.
|
||||
English: The children are playing in the playground.
|
||||
English: I'm going to visit my grandparents this weekend.
|
||||
English: The coffee tastes bitter without sugar.
|
||||
English: They are planning a surprise party for her.
|
||||
English: She sings like an angel on stage.
|
||||
English: We should take a vacation to relax.
|
||||
English: He is studying medicine at the university.
|
||||
English: The rain is pouring heavily outside.
|
||||
English: I enjoy watching romantic movies.
|
||||
English: They are celebrating their anniversary today.
|
||||
English: She dances gracefully to the music.
|
||||
English: He is an excellent basketball player.
|
||||
English: The baby is sleeping soundly in the crib.
|
||||
English: I need to finish my homework before dinner.
|
||||
English: They are organizing a charity event next month.
|
||||
English: She is cooking a delicious meal for us.
|
||||
English: We should go hiking in the mountains.
|
||||
English: The car broke down on the way to work.
|
||||
English: He loves playing video games in his free time.
|
||||
English: The birds are chirping in the trees.
|
||||
English: I want to learn how to play the piano.
|
||||
English: They are building a new shopping mall in the city.
|
||||
English: She is writing a novel in her spare time.
|
||||
English: We are going to the zoo this Saturday.
|
||||
English: The cake looks delicious with chocolate frosting.
|
||||
English: He is a talented painter who sells his artwork.
|
||||
English: The students are studying for their exams.
|
||||
English: I enjoy swimming in the ocean.
|
||||
English: They are renovating their house.
|
||||
English: She is practicing yoga to stay healthy.
|
||||
English: We should plant flowers in the garden.
|
||||
English: The traffic is heavy during rush hour.
|
||||
English: He is a skilled chef who creates amazing dishes.
|
||||
English: The baby is crawling on the floor.
|
||||
English: I need to buy a new pair of shoes.
|
||||
English: They are going on a road trip across the country.
|
||||
English: She is playing the piano beautifully.
|
||||
English: We are going to a concert tomorrow night.
|
||||
English: The cake tastes delicious with vanilla frosting.
|
||||
English: He is a dedicated teacher who inspires his students.
|
||||
English: The students are participating in a science fair.
|
||||
English: I enjoy hiking in the mountains.
|
||||
English: They are organizing a beach cleanup next weekend.
|
||||
English: She is taking photographs of nature.
|
||||
English: We should try a new restaurant in town.
|
||||
English: The traffic is moving slowly on the highway.
|
||||
English: He is a talented singer with a beautiful voice.
|
||||
English: The baby is laughing and giggling.
|
||||
English: I need to do laundry and wash my clothes.
|
||||
English: They are planning a trip to Europe.
|
||||
English: She is learning how to play the guitar.
|
||||
English: We are going to a museum this Sunday.
|
||||
English: The coffee smells amazing in the morning.
|
||||
English: He is a hardworking farmer who grows crops.
|
||||
English: The students are presenting their research projects.
|
||||
English: I enjoy playing soccer with my friends.
|
||||
English: They are volunteering at a local shelter.
|
||||
English: She is practicing martial arts for self-defense.
|
||||
English: We should try a new recipe for dinner.
|
||||
English: The traffic is congest
|
||||
English: The sun is shining brightly today.
|
||||
English: I enjoy reading books in my free time.
|
||||
English: She plays the piano beautifully.
|
||||
English: The cat chased the mouse around the room.
|
||||
English: I love eating pizza with extra cheese.
|
||||
English: He always wears a hat wherever he goes.
|
||||
English: The flowers in the garden are blooming.
|
||||
English: She danced gracefully on the stage.
|
||||
English: The dog barked loudly in the park.
|
||||
English: We went swimming in the ocean yesterday.
|
||||
English: He speaks fluent French and Spanish.
|
||||
English: The train arrived at the station on time.
|
||||
English: She cooked a delicious meal for her family.
|
||||
Korean: 이것은 테스트 이다.
|
||||
Korean: 걱정할 필요 없다.
|
||||
Korean: 버그는 언젠가 고쳐진다.
|
||||
Japanese: 明日の天気はどうですか。
|
||||
Chinese: 请问洗手间在哪里？
|
||||
Emoji: I'm feeling 😄 today!
|
||||
Unicode: ◑ ▢ ▣ ◱
|
||||
@@ -0,0 +1,65 @@
|
||||
import os
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
os.environ['TOKENIZERS_PARALLELISM'] = "false"
|
||||
|
||||
list_repo_hf = ["databricks/dolly-v2-3b", # dolly-v2 (3b, 7b, 12b models share the same tokenizer)
|
||||
"gpt2", # gpt-2 (gpt2-xl, gpt2-large share the same tokenizer)
|
||||
"uer/gpt2-chinese-cluecorpussmall", # gpt-2-chinese
|
||||
"EleutherAI/gpt-j-6b", # gpt-j
|
||||
"EleutherAI/gpt-neox-20b", # gpt-neox
|
||||
"EleutherAI/polyglot-ko-1.3b", # gpt-neox (polyglot-ko 5.8b and 12.8b share the same tokenizer")
|
||||
"rinna/japanese-gpt-neox-3.6b", # gpt-neox
|
||||
# mpt-7b (uses gpt-neox-20b tokenizer)
|
||||
"replit/replit-code-v1-3b", # replit
|
||||
"bigcode/starcoder", # starcoder (huggingface-cli login required)
|
||||
"openai/whisper-tiny" # whisper (base, large, large-v2 share the same tokenizer)
|
||||
]
|
||||
|
||||
repo2ggml = {"databricks/dolly-v2-3b" : "dolly-v2",
|
||||
"gpt2" : "gpt-2",
|
||||
"uer/gpt2-chinese-cluecorpussmall" : "gpt-2-chinese",
|
||||
"EleutherAI/gpt-j-6b" : "gpt-j",
|
||||
"EleutherAI/gpt-neox-20b" : "gpt-neox",
|
||||
"EleutherAI/polyglot-ko-1.3b" : "polyglot-ko",
|
||||
"rinna/japanese-gpt-neox-3.6b" : "gpt-neox-japanese",
|
||||
"replit/replit-code-v1-3b" : "replit",
|
||||
"bigcode/starcoder" : "starcoder",
|
||||
"openai/whisper-tiny" : "whisper"}
|
||||
|
||||
repo2language = {"databricks/dolly-v2-3b" : "english",
|
||||
"gpt2" : "english",
|
||||
"uer/gpt2-chinese-cluecorpussmall" : "chinese",
|
||||
"EleutherAI/gpt-j-6b" : "english",
|
||||
"EleutherAI/gpt-neox-20b" : "english",
|
||||
"EleutherAI/polyglot-ko-1.3b" : "korean",
|
||||
"rinna/japanese-gpt-neox-3.6b" : "japanese",
|
||||
"replit/replit-code-v1-3b" : "english",
|
||||
"bigcode/starcoder" : "english",
|
||||
"openai/whisper-tiny" : "english"}
|
||||
|
||||
delimeter = ": "
|
||||
test_sentences = []
|
||||
with open("test-cases.txt", "r") as f:
|
||||
lines = [l.rstrip() for l in f.readlines()]
|
||||
for l in lines:
|
||||
if delimeter in l:
|
||||
language = l[:l.index(delimeter)]
|
||||
sentence = l[l.index(delimeter) + len(delimeter):]
|
||||
test_sentences.append((language.lower(), sentence))
|
||||
|
||||
for repo in list_repo_hf:
|
||||
|
||||
target_language = repo2language[repo]
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
|
||||
|
||||
tokens_hf = []
|
||||
for language, sentence in test_sentences:
|
||||
if language == target_language:
|
||||
tokens = tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentence))
|
||||
tokens_hf.append((sentence, tokens))
|
||||
|
||||
save_txt = repo2ggml[repo] + ".txt"
|
||||
with open(save_txt, "w") as f:
|
||||
f.writelines([sentence + " => " + ",".join(str(t) for t in tokens) + "\n" for sentence, tokens in tokens_hf])
|
||||
@@ -0,0 +1,100 @@
|
||||
Hello World! => 15947,3937,0
|
||||
I can't believe it's already Friday!" => 40,393,380,1697,309,311,1217,6984,2963
|
||||
The URL for the website is https://www.example.com." => 2278,12905,337,220,3322,3144,307,34426,21492,17919,13,3121,335,781,13,1112,889
|
||||
"She said, 'I love to travel.'" => 1,9526,848,11,922,40,959,220,1353,220,17227,779,28763
|
||||
'The temperature is 25.5°C.' => 6,2278,220,18275,610,1503,307,3552,13,20,11782,34,4443
|
||||
"Let's meet at 2:30 p.m. in the park." => 1,8373,311,1677,412,568,25,3446,280,13,76,13,294,220,3322,3884,889
|
||||
The book costs $19.99 => 2278,1446,5497,1848,3405,13,8494
|
||||
"John's favorite color is blue." => 1,16938,311,2954,2017,307,3344,889
|
||||
Th@nk y0u f0r y0ur h3lp! => 2434,31,77,74,288,15,84,283,15,81,288,15,374,276,18,75,79,0
|
||||
C@n I g3t a c0ffee, pl3@se? => 34,31,77,286,290,18,83,257,269,15,4617,11,499,18,31,405,30
|
||||
W0w! Th@t's @m@zing! => 54,15,86,0,334,31,83,311,10428,76,31,8781,0
|
||||
H0w 4re y0u t0d@y? => 39,15,86,1017,265,288,15,84,220,83,15,67,31,88,30
|
||||
I l0ve t0 tr@vel @r0und the w0rld. => 40,287,15,303,220,83,15,220,6903,31,779,10428,81,15,997,220,3322,261,15,81,348,13
|
||||
Wh@t's y0ur f@v0rite m0vie? => 2471,31,83,311,288,15,374,283,31,85,15,35002,275,15,12702,30
|
||||
The cat is sleeping on the mat. => 2278,3857,307,8296,322,220,3322,3803,13
|
||||
I need to buy some groceries for dinner. => 40,643,220,1353,2256,512,31391,337,6148,13
|
||||
The sun is shining brightly in the sky. => 2278,3295,307,18269,47418,294,220,3322,5443,13
|
||||
She is reading a book in the park. => 9526,307,3760,257,1446,294,220,3322,3884,13
|
||||
We went for a walk on the beach yesterday. => 4360,1437,337,257,1792,322,220,3322,7534,5186,13
|
||||
He plays the guitar like a pro. => 5205,5749,220,3322,7531,411,257,447,13
|
||||
They are going to the movies tonight. => 8829,366,516,220,1353,220,3322,6233,220,1756,397,13
|
||||
The flowers are blooming in the garden. => 2278,8085,366,45294,294,220,3322,7431,13
|
||||
I enjoy listening to classical music. => 40,2103,4764,220,1353,13735,1318,13
|
||||
We need to buy groceries for the week. => 4360,643,220,1353,2256,31391,337,220,3322,1243,13
|
||||
The dog is chasing its tail in circles. => 2278,3000,307,17876,1080,220,14430,294,13040,13
|
||||
She is wearing a beautiful red dress. => 9526,307,4769,257,2238,2182,5231,13
|
||||
He is a talented actor in Hollywood. => 5205,307,257,220,32831,6003,8747,294,11628,13
|
||||
The children are playing in the playground. => 2278,2227,366,2433,294,220,3322,24646,13
|
||||
I'm going to visit my grandparents this weekend. => 40,478,516,220,1353,3441,452,21876,220,11176,6711,13
|
||||
The coffee tastes bitter without sugar. => 2278,4982,220,83,40246,13871,1553,5076,13
|
||||
They are planning a surprise party for her. => 8829,366,5038,257,6365,3595,337,720,13
|
||||
She sings like an angel on stage. => 9526,23250,411,364,14250,322,3233,13
|
||||
We should take a vacation to relax. => 4360,820,220,27612,257,12830,220,1353,5789,13
|
||||
He is studying medicine at the university. => 5205,307,7601,7195,412,220,3322,5454,13
|
||||
The rain is pouring heavily outside. => 2278,4830,307,20450,10950,2380,13
|
||||
I enjoy watching romantic movies. => 40,2103,1976,13590,6233,13
|
||||
They are celebrating their anniversary today. => 8829,366,15252,220,3322,347,12962,220,83,378,320,13
|
||||
She dances gracefully to the music. => 9526,28322,10042,2277,220,1353,220,3322,1318,13
|
||||
He is an excellent basketball player. => 5205,307,364,7103,11767,4256,13
|
||||
The baby is sleeping soundly in the crib. => 2278,3186,307,8296,1626,356,294,220,3322,47163,13
|
||||
I need to finish my homework before dinner. => 40,643,220,1353,2413,452,14578,949,6148,13
|
||||
They are organizing a charity event next month. => 8829,366,17608,257,16863,2280,958,1618,13
|
||||
She is cooking a delicious meal for us. => 9526,307,6361,257,4809,6791,337,505,13
|
||||
We should go hiking in the mountains. => 4360,820,352,23784,294,220,3322,10233,13
|
||||
The car broke down on the way to work. => 2278,1032,6902,760,322,220,3322,636,220,1353,589,13
|
||||
He loves playing video games in his free time. => 5205,6752,2433,960,2813,294,702,1737,220,3766,13
|
||||
The birds are chirping in the trees. => 2278,9009,366,36682,294,220,3322,220,3599,279,13
|
||||
I want to learn how to play the piano. => 40,528,220,1353,1466,577,220,1353,862,220,3322,9211,13
|
||||
They are building a new shopping mall in the city. => 8829,366,2390,257,777,8688,16026,294,220,3322,2307,13
|
||||
She is writing a novel in her spare time. => 9526,307,3579,257,7613,294,720,13798,220,3766,13
|
||||
We are going to the zoo this Saturday. => 4360,366,516,220,1353,220,3322,25347,220,11176,8803,13
|
||||
The cake looks delicious with chocolate frosting. => 2278,5908,1542,4809,365,6215,37048,13
|
||||
He is a talented painter who sells his artwork. => 5205,307,257,220,32831,6003,26619,567,20897,702,15829,13
|
||||
The students are studying for their exams. => 2278,1731,366,7601,337,220,3322,347,20514,13
|
||||
I enjoy swimming in the ocean. => 40,2103,11989,294,220,3322,7810,13
|
||||
They are renovating their house. => 8829,366,18845,990,220,3322,347,1782,13
|
||||
She is practicing yoga to stay healthy. => 9526,307,11350,15128,220,1353,1754,4627,13
|
||||
We should plant flowers in the garden. => 4360,820,3709,8085,294,220,3322,7431,13
|
||||
The traffic is heavy during rush hour. => 2278,220,17227,3341,307,4676,1830,9300,1773,13
|
||||
He is a skilled chef who creates amazing dishes. => 5205,307,257,19690,10530,567,7829,2243,10814,13
|
||||
The baby is crawling on the floor. => 2278,3186,307,32979,322,220,3322,4123,13
|
||||
I need to buy a new pair of shoes. => 40,643,220,1353,2256,257,777,6119,295,6654,13
|
||||
They are going on a road trip across the country. => 8829,366,516,322,257,3060,220,83,8400,2108,220,3322,1941,13
|
||||
She is playing the piano beautifully. => 9526,307,2433,220,3322,9211,16525,13
|
||||
We are going to a concert tomorrow night. => 4360,366,516,220,1353,257,8543,220,83,298,3162,1818,13
|
||||
The cake tastes delicious with vanilla frosting. => 2278,5908,220,83,40246,4809,365,17528,37048,13
|
||||
He is a dedicated teacher who inspires his students. => 5205,307,257,8374,220,975,4062,567,32566,702,1731,13
|
||||
The students are participating in a science fair. => 2278,1731,366,13950,294,257,3497,3143,13
|
||||
I enjoy hiking in the mountains. => 40,2103,23784,294,220,3322,10233,13
|
||||
They are organizing a beach cleanup next weekend. => 8829,366,17608,257,7534,40991,958,6711,13
|
||||
She is taking photographs of nature. => 9526,307,220,48625,17649,295,3687,13
|
||||
We should try a new restaurant in town. => 4360,820,220,83,627,257,777,6383,294,220,30401,13
|
||||
The traffic is moving slowly on the highway. => 2278,220,17227,3341,307,2684,5692,322,220,3322,17205,13
|
||||
He is a talented singer with a beautiful voice. => 5205,307,257,220,32831,6003,11564,365,257,2238,3177,13
|
||||
The baby is laughing and giggling. => 2278,3186,307,5059,293,290,24542,13
|
||||
I need to do laundry and wash my clothes. => 40,643,220,1353,360,19811,293,5675,452,5534,13
|
||||
They are planning a trip to Europe. => 8829,366,5038,257,220,83,8400,220,1353,3315,13
|
||||
She is learning how to play the guitar. => 9526,307,2539,577,220,1353,862,220,3322,7531,13
|
||||
We are going to a museum this Sunday. => 4360,366,516,220,1353,257,8441,220,11176,7776,13
|
||||
The coffee smells amazing in the morning. => 2278,4982,10036,2243,294,220,3322,2446,13
|
||||
He is a hardworking farmer who grows crops. => 5205,307,257,1152,22475,17891,567,13156,16829,13
|
||||
The students are presenting their research projects. => 2278,1731,366,15578,220,3322,347,2132,4455,13
|
||||
I enjoy playing soccer with my friends. => 40,2103,2433,15469,365,452,1855,13
|
||||
They are volunteering at a local shelter. => 8829,366,33237,412,257,2654,13341,13
|
||||
She is practicing martial arts for self-defense. => 9526,307,11350,20755,8609,337,2698,12,49268,13
|
||||
We should try a new recipe for dinner. => 4360,820,220,83,627,257,777,6782,337,6148,13
|
||||
The traffic is congest => 2278,220,17227,3341,307,31871
|
||||
The sun is shining brightly today. => 2278,3295,307,18269,47418,220,83,378,320,13
|
||||
I enjoy reading books in my free time. => 40,2103,3760,3642,294,452,1737,220,3766,13
|
||||
She plays the piano beautifully. => 9526,5749,220,3322,9211,16525,13
|
||||
The cat chased the mouse around the room. => 2278,3857,33091,220,3322,9719,926,220,3322,1808,13
|
||||
I love eating pizza with extra cheese. => 40,959,3936,8298,365,2857,5399,13
|
||||
He always wears a hat wherever he goes. => 5205,1009,20877,257,2385,8660,415,1709,13
|
||||
The flowers in the garden are blooming. => 2278,8085,294,220,3322,7431,366,45294,13
|
||||
She danced gracefully on the stage. => 9526,32909,10042,2277,322,220,3322,3233,13
|
||||
The dog barked loudly in the park. => 2278,3000,16202,292,22958,294,220,3322,3884,13
|
||||
We went swimming in the ocean yesterday. => 4360,1437,11989,294,220,3322,7810,5186,13
|
||||
He speaks fluent French and Spanish. => 5205,10789,40799,5522,293,8058,13
|
||||
The train arrived at the station on time. => 2278,220,83,7146,6678,412,220,3322,5214,322,220,3766,13
|
||||
She cooked a delicious meal for her family. => 9526,9267,257,4809,6791,337,720,1605,13
|
||||
@@ -0,0 +1,115 @@
|
||||
# Simple autogenerated Python bindings for ggml
|
||||
|
||||
This folder contains:
|
||||
|
||||
- Scripts to generate full Python bindings from ggml headers (+ stubs for autocompletion in IDEs)
|
||||
- Some barebones utils (see [ggml/utils.py](./ggml/utils.py)):
|
||||
- `ggml.utils.init` builds a context that's freed automatically when the pointer gets GC'd
|
||||
- `ggml.utils.copy` **copies between same-shaped tensors (numpy or ggml), w/ automatic (de/re)quantization**
|
||||
- `ggml.utils.numpy` returns a numpy view over a ggml tensor; if it's quantized, it returns a copy (requires `allow_copy=True`)
|
||||
- Very basic examples (anyone wants to port [llama2.c](https://github.com/karpathy/llama2.c)?)
|
||||
|
||||
Provided you set `GGML_LIBRARY=.../path/to/libggml_shared.so` (see instructions below), it's trivial to do some operations on quantized tensors:
|
||||
|
||||
```python
|
||||
# Make sure libllama.so is in your [DY]LD_LIBRARY_PATH, or set GGML_LIBRARY=.../libggml_shared.so
|
||||
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
|
||||
ctx = init(mem_size=12*1024*1024)
|
||||
n = 256
|
||||
n_threads = 4
|
||||
|
||||
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # Can't both be quantized
|
||||
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
|
||||
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, sum)
|
||||
|
||||
copy(np.array([i for i in range(n)], np.float32), a)
|
||||
copy(np.array([i*100 for i in range(n)], np.float32), b)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
|
||||
|
||||
print(numpy(a, allow_copy=True))
|
||||
# 0. 1.0439453 2.0878906 3.131836 4.1757812 5.2197266. ...
|
||||
print(numpy(b))
|
||||
# 0. 100. 200. 300. 400. 500. ...
|
||||
print(numpy(sum, allow_copy=True))
|
||||
# 0. 105.4375 210.875 316.3125 421.75 527.1875 ...
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
|
||||
You'll need a shared library of ggml to use the bindings.
|
||||
|
||||
#### Build libggml_shared.so or libllama.so
|
||||
|
||||
As of this writing the best is to use [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)'s generated `libggml_shared.so` or `libllama.so`, which you can build as follows:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggerganov/llama.cpp
|
||||
# On a CUDA-enabled system add -DLLAMA_CUDA=1
|
||||
# On a Mac add -DLLAMA_METAL=1
|
||||
cmake llama.cpp \
|
||||
-B llama_build \
|
||||
-DCMAKE_C_FLAGS=-Ofast \
|
||||
-DLLAMA_NATIVE=1 \
|
||||
-DLLAMA_LTO=1 \
|
||||
-DBUILD_SHARED_LIBS=1 \
|
||||
-DLLAMA_MPI=1 \
|
||||
-DLLAMA_BUILD_TESTS=0 \
|
||||
-DLLAMA_BUILD_EXAMPLES=0
|
||||
( cd llama_build && make -j )
|
||||
|
||||
# On Mac, this will be libggml_shared.dylib instead
|
||||
export GGML_LIBRARY=$PWD/llama_build/libggml_shared.so
|
||||
# Alternatively, you can just copy it to your system's lib dir, e.g /usr/local/lib
|
||||
```
|
||||
|
||||
#### (Optional) Regenerate the bindings and stubs
|
||||
|
||||
If you added or changed any signatures of the C API, you'll want to regenerate the bindings ([ggml/cffi.py](./ggml/cffi.py)) and stubs ([ggml/__init__.pyi](./ggml/__init__.pyi)).
|
||||
|
||||
Luckily it's a one-liner using [regenerate.py](./regenerate.py):
|
||||
|
||||
```bash
|
||||
pip install -q cffi
|
||||
|
||||
python regenerate.py
|
||||
```
|
||||
|
||||
By default it assumes `llama.cpp` was cloned in ../../../llama.cpp (alongside the ggml folder). You can override this with:
|
||||
|
||||
```bash
|
||||
C_INCLUDE_DIR=$LLAMA_CPP_DIR python regenerate.py
|
||||
```
|
||||
|
||||
You can also edit [api.h](./api.h) to control which files should be included in the generated bindings (defaults to `llama.cpp/ggml*.h`)
|
||||
|
||||
In fact, if you wanted to only generate bindings for the current version of the `ggml` repo itself (instead of `llama.cpp`; you'd loose support for k-quants), you could run:
|
||||
|
||||
```bash
|
||||
API=../../include/ggml.h python regenerate.py
|
||||
```
|
||||
|
||||
## Develop
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
### Alternatives
|
||||
|
||||
This example's goal is to showcase [cffi](https://cffi.readthedocs.io/)-generated bindings that are trivial to use and update, but there are already alternatives in the wild:
|
||||
|
||||
- https://github.com/abetlen/ggml-python: these bindings seem to be hand-written and use [ctypes](https://docs.python.org/3/library/ctypes.html). It has [high-quality API reference docs](https://ggml-python.readthedocs.io/en/latest/api-reference/#ggml.ggml) that can be used with these bindings too, but it doesn't expose Metal, CUDA, MPI or OpenCL calls, doesn't support transparent (de/re)quantization like this example does (see [ggml.utils](./ggml/utils.py) module), and won't pick up your local changes.
|
||||
|
||||
- https://github.com/abetlen/llama-cpp-python: these expose the C++ `llama.cpp` interface, which this example cannot easily be extended to support (`cffi` only generates bindings of C libraries)
|
||||
|
||||
- [pybind11](https://github.com/pybind/pybind11) and [nanobind](https://github.com/wjakob/nanobind) are two alternatives to cffi that support binding C++ libraries, but it doesn't seem either of them have an automatic generator (writing bindings is rather time-consuming).
|
||||
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
List here all the headers you want to expose in the Python bindings,
|
||||
then run `python regenerate.py` (see details in README.md)
|
||||
*/
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-metal.h"
|
||||
#include "ggml-opencl.h"
|
||||
|
||||
// Headers below are currently only present in the llama.cpp repository, comment them out if you don't have them.
|
||||
#include "k_quants.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-cuda.h"
|
||||
#include "ggml-mpi.h"
|
||||
@@ -0,0 +1,25 @@
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
|
||||
ctx = init(mem_size=12*1024*1024) # automatically freed when pointer is GC'd
|
||||
n = 256
|
||||
n_threads = 4
|
||||
|
||||
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # can't both be quantized
|
||||
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
|
||||
|
||||
# See cffi's doc on how to allocate native memory: it's very simple!
|
||||
# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, sum)
|
||||
|
||||
copy(np.array([i for i in range(n)], np.float32), a)
|
||||
copy(np.array([i*100 for i in range(n)], np.float32), b)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
|
||||
|
||||
print(numpy(a, allow_copy=True))
|
||||
print(numpy(b))
|
||||
print(numpy(sum, allow_copy=True))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user