civ/trellis2-ROCm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial release

Browse Source
This commit is contained in:
civ
2026-08-16 18:33:03 +07:00
commit 7ade4e1152
1966 changed files with 412966 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+61
View File
@@ -0,0 +1,61 @@
# build output
/build/
/build-*/
/.deps/
# scratch / generated artifacts (meshes, latents, occupancy grids)
/tmp/
*.o
*.a
*.so
*.dylib
*.dll
# cmake
CMakeCache.txt
CMakeFiles/
cmake_install.cmake
compile_commands.json
# model weights / data (large, regenerated by the converters)
*.gguf
*.safetensors
*.dinodata
*.latent
*.occ
*.obj
# test reference artifacts (regenerated by tests/ref_*.py)
tests/ss_flow_ref.bin
tests/ss_sample_ref.bin
tests/ss_dec_ref.bin
# editor / os
.DS_Store
.vscode/
.idea/
# clangd index
.cache/
corpus_*/
crash-*
# fuzzing
/corpus_*/
crash-*
# demo server build artifacts
/server/trellis2-server
/server/trellis2-server-linux
/server/vendor/
# python bytecode
__pycache__/
*.pyc
# model / reference artifacts
/models/
/ggufs/
/dumps/
/generations/
slow-unit-*
.gitmodules
+3
View File
@@ -0,0 +1,3 @@
[submodule "ggml"]
path = ggml
url = https://github.com/PABannier/ggml.git
CMakeLists.txt
+137
View File
@@ -0,0 +1,137 @@
cmake_minimum_required(VERSION 3.14)
project(trellis2.cpp VERSION 0.1.0 LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release)
endif()
# ─────────────────────────────────────────────────────────────────────────
# External ggml option
#
# When OFF (default): vendor ggml as a subdirectory — the upstream behavior.
# When ON : skip the bundled ggml and resolve `find_package(ggml CONFIG)`
# against an installed ggml. Lets a downstream project link multiple
# ggml-based libraries against a single shared ggml so they don't
# ship duplicate copies into the same process.
# ─────────────────────────────────────────────────────────────────────────
option(TRELLIS2_USE_EXTERNAL_GGML "Consume external ggml via find_package() instead of the bundled submodule" OFF)
# libFuzzer harnesses (clang only). The sanitizer flags apply globally — set
# BEFORE any target so the whole tree (ggml, stb decode, preprocessing,
# loaders) is instrumented. Harness targets live in fuzz/ (added at the end).
option(TRELLIS2_FUZZ "Build libFuzzer harnesses (requires clang)" OFF)
set(TRELLIS2_FUZZ_SANITIZERS "address,undefined" CACHE STRING
"Sanitizers for the fuzz build")
if(TRELLIS2_FUZZ)
add_compile_options(-fsanitize=${TRELLIS2_FUZZ_SANITIZERS} -fsanitize=fuzzer-no-link -fno-omit-frame-pointer -g -O1)
add_link_options(-fsanitize=${TRELLIS2_FUZZ_SANITIZERS})
endif()
# ggml options — enable Metal on Apple (bundled-ggml path only)
option(TRELLIS2_METAL "Enable Metal backend" ON)
if(APPLE AND TRELLIS2_METAL AND NOT TRELLIS2_USE_EXTERNAL_GGML)
set(GGML_METAL ON CACHE BOOL "" FORCE)
set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE)
endif()
if(TRELLIS2_USE_EXTERNAL_GGML)
find_package(ggml CONFIG REQUIRED)
set(TRELLIS2_GGML_TARGET ggml::ggml)
else()
add_subdirectory(ggml)
set(TRELLIS2_GGML_TARGET ggml)
endif()
add_library(trellis2
trellis2.cpp trellis2.h
trellis2_capi.cpp trellis2_capi.h
mesh_export.cpp mesh_export.h
print_remesh.cpp print_remesh.h
third_party/xatlas/xatlas.cpp)
target_include_directories(trellis2 PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/stb
)
# trellis2_capi.cpp reuses the example isosurface extractor for the demo path;
# mesh_export.cpp vendors xatlas for the optional chart-based UV unwrap.
target_include_directories(trellis2 PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/examples
${CMAKE_CURRENT_SOURCE_DIR}/third_party/xatlas)
find_package(Threads REQUIRED) # xatlas uses std::thread
if(TRELLIS2_USE_EXTERNAL_GGML AND DEFINED GGML_INCLUDE_DIR)
target_include_directories(trellis2 PUBLIC ${GGML_INCLUDE_DIR})
endif()
target_link_libraries(trellis2 PUBLIC ${TRELLIS2_GGML_TARGET} PRIVATE Threads::Threads)
target_compile_features(trellis2 PUBLIC cxx_std_14)
# CGAL Alpha Wrap turns arbitrary TRELLIS triangle soups into watertight,
# intersection-free 2-manifolds for printing. Auto-detect it so ordinary builds
# retain no CGAL dependency; the C API reports whether the backend is present.
option(TRELLIS2_CGAL "Enable CGAL Alpha Wrap print remeshing when available" ON)
option(TRELLIS2_FETCH_PRINT_REMESH_DEPS
"Fetch the project-pinned CGAL and Boost headers for print remeshing" OFF)
set(TRELLIS2_PRINT_REMESH_DEPS_DIR "${CMAKE_BINARY_DIR}/_deps/print-remesh" CACHE PATH
"Directory containing project-pinned CGAL and Boost headers")
if(TRELLIS2_CGAL)
if(TRELLIS2_FETCH_PRINT_REMESH_DEPS)
find_program(TRELLIS2_BASH_EXECUTABLE bash)
if(NOT TRELLIS2_BASH_EXECUTABLE)
message(FATAL_ERROR "bash is required to fetch print-remesh dependencies")
endif()
execute_process(
COMMAND "${TRELLIS2_BASH_EXECUTABLE}"
"${CMAKE_CURRENT_SOURCE_DIR}/scripts/fetch_print_remesh_deps.sh"
"${TRELLIS2_PRINT_REMESH_DEPS_DIR}"
RESULT_VARIABLE TRELLIS2_PRINT_REMESH_DEPS_RESULT)
if(NOT TRELLIS2_PRINT_REMESH_DEPS_RESULT EQUAL 0)
message(FATAL_ERROR "Failed to fetch pinned print-remesh dependencies")
endif()
set(CGAL_DIR "${TRELLIS2_PRINT_REMESH_DEPS_DIR}/cgal" CACHE PATH
"CGAL package directory" FORCE)
set(Boost_INCLUDE_DIR "${TRELLIS2_PRINT_REMESH_DEPS_DIR}/boost" CACHE PATH
"Boost header directory" FORCE)
set(Boost_DIR "${TRELLIS2_PRINT_REMESH_DEPS_DIR}/boost-cmake" CACHE PATH
"Header-only Boost CMake package directory" FORCE)
endif()
find_package(CGAL 5.5 QUIET)
if(CGAL_FOUND)
target_link_libraries(trellis2 PRIVATE CGAL::CGAL)
target_compile_definitions(trellis2 PRIVATE TRELLIS2_USE_CGAL=1)
target_compile_features(trellis2 PRIVATE cxx_std_17)
set(TRELLIS2_HAVE_CGAL ON)
message(STATUS "CGAL Alpha Wrap print remeshing: enabled")
elseif(TRELLIS2_FETCH_PRINT_REMESH_DEPS)
message(FATAL_ERROR "The project-pinned CGAL print-remesh dependencies were fetched but not found")
else()
message(STATUS "CGAL Alpha Wrap print remeshing: unavailable (CGAL >= 5.5 not found)")
endif()
endif()
set_property(TARGET trellis2 PROPERTY POSITION_INDEPENDENT_CODE ON)
if(BUILD_SHARED_LIBS)
target_compile_definitions(trellis2 PUBLIC TRELLIS2_SHARED)
target_compile_definitions(trellis2 PRIVATE TRELLIS2_BUILD)
endif()
option(TRELLIS2_BUILD_EXAMPLES "Build example executables" ON)
if(TRELLIS2_BUILD_EXAMPLES AND CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
add_subdirectory(examples)
endif()
option(TRELLIS2_BUILD_TESTS "Build test executables" OFF)
if(TRELLIS2_BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
endif()
if(TRELLIS2_FUZZ)
add_subdirectory(fuzz)
endif()
LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 rms80
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+360
View File
@@ -0,0 +1,360 @@
# trellis2.cpp
A C++/[ggml](https://github.com/ggml-org/ggml) implementation of the
**TRELLIS.2** image-to-3D pipeline: an image goes in, a 3D mesh with per-vertex
PBR textures comes out, with all inference in C++/ggml (no PyTorch at runtime).
The demo can also export the result into a portable, full-density **GLB** with
standard interpolated vertex colour and retained PBR attributes—no reference
container required.
Modeled structurally after [sam3.cpp](https://github.com/rms80/sam3.cpp):
single-file library (`trellis2.h` / `trellis2.cpp`), bundled ggml as a
submodule (Metal on by default on Apple), DLL-export decoration, and a
CMake build with example executables. A flat C ABI (`trellis2_capi.h`) drives
a Go demo server with a browser mesh viewer.
## Quick start (demo)
```sh
git submodule update --init --depth 1 # ggml
scripts/download_ggufs.sh # prebuilt f16 GGUFs -> ggufs/ (~14 GB)
docker build -f docker/Dockerfile.demo -t trellis2-demo docker # CUDA runtime + Go
# build the CUDA shared lib + Go server, then run
docker run --rm -v "$PWD":/work -w /work trellis2-demo bash -c '
cmake -B build-cuda-shared -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON \
-DCMAKE_CUDA_ARCHITECTURES=120 -DBUILD_SHARED_LIBS=ON && cmake --build build-cuda-shared -j
cd server && go build -o trellis2-server-linux .'
docker run --rm --device nvidia.com/gpu=all -v "$PWD":/work -w /work/server -p 8742:8742 \
trellis2-demo ./trellis2-server-linux -lib /work/build-cuda-shared/libtrellis2.so \
-ggufs /work/ggufs -store /work/generations -unload-idle
# open http://localhost:8742 and drop an image
```
Or just run `scripts/demo.sh`, which builds the lib + server, auto-downloads any
missing GGUFs, and launches the container.
### Prebuilt GGUFs
`scripts/download_ggufs.sh` pulls the ready-made f16 GGUFs from three public repos
under the [LocalAI-io](https://huggingface.co/LocalAI-io) org, so you can skip the
safetensors download and the conversion step entirely:
- [`TRELLIS.2-4B-GGUF`](https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF) — MIT
- [`TRELLIS-image-large-GGUF`](https://huggingface.co/LocalAI-io/TRELLIS-image-large-GGUF) — MIT
- [`dinov3-vitl16-pretrain-lvd1689m-GGUF`](https://huggingface.co/LocalAI-io/dinov3-vitl16-pretrain-lvd1689m-GGUF) — DINOv3 License (Built with DINOv3)
**Developers** who need the f32 validation variants, or who want to regenerate the
GGUFs from source, use the original flow instead: `scripts/download_models.sh` (HF
safetensors → `models/`, ~7 GB) then `docker run … trellis2-ref bash
scripts/convert_all.sh` (safetensors → GGUF, f16 + f32). The card + license sources
for the published repos live in `scripts/hf/`, and `scripts/upload_ggufs.sh`
(re)publishes them.
Completed generations are committed atomically under `generations/` (final
mesh, replay frames, and manifest) and restored with the same job IDs after a
server restart. Pass `-store ''` to disable persistence or `-store PATH` to use
a different durable location. Incomplete writes are ignored on startup. With
`-unload-idle`, the HTTP server starts without allocating model VRAM, loads the
pipeline on the first generation, and releases it again when the queue is idle.
The browser UI has a **quality** selector: coarse preview (64³ marching cubes),
512³ fine, or **1024³ cascade** (the TRELLIS.2 default and highest resolution
currently supported here). Upstream's optional 1536 cascade is not yet ported. Coarse falls
back automatically if the shape-SLAT models are absent (`-coarse`); the 1024
cascade needs the extra 1024 model (`-no-1024` disables it).
Enable **free VRAM when idle** to unload the resident model pipeline between
generations; the next queued generation reloads it automatically.
**Live steps** is off by default because each sparse-structure frame requires an
extra CPU occupancy decode between GPU inference steps. Its button always says
`on` or `off`; enabling it records the frames used by replay and showcase mode.
Completed jobs expose `durationMs`, `livePreview`, and per-stage `stageTimings`
through `/api/job/{id}` and persist those diagnostics in their manifest.
The always-visible **asset export** panel preserves the generated polygon count,
can preview component cleanup, optionally keep only the largest connected piece,
restore the original preview, and download a Three.js-ready GLB. Dense generated
materials are stored as standard interpolated vertex colours rather than a
sub-texel per-triangle atlas; original metallic/roughness values are also retained
in the custom `_METALLIC_ROUGHNESS` attribute. All components are preserved by
default; destructive cleanup must be selected explicitly. Showcase mode likewise loads the original
full-density mesh. Open `/showcase` for the separate full-screen storyboard: it
starts each generation with its saved source image centered, moves that image to
the upper-right, replays the recorded stages, and then lingers on a slowly
rotating final model. New generations retain the original upload byte-for-byte
for display, plus the full-resolution processed PNG actually used by TRELLIS for
repeatable server-side regeneration without another upload. Select a saved mesh
and use **regenerate from saved image** to run it again with the current settings.
Older manifests fall back to their thumbnail and can only regenerate when their
older processed source file is available.
When built with CGAL 5.5 or newer, asset export also offers **watertight print
wrap**. It runs CPU Alpha Wrap over the selected components and previews the
exact replacement geometry before download. CGAL guarantees the result is
closed, oriented, intersection-free, and 2-manifold. `detail size` controls the
smallest holes/cavities the wrap enters; `offset` controls how tightly it encloses
the generated surface. Both are percentages of the source bounding-box diagonal.
Because wrapping creates new vertices, its fast browser preview is geometry-only.
When the source is textured, GLB download automatically unwraps the print mesh
with xatlas and rebakes base color, metallic, roughness, and opacity: each atlas
texel is projected through a CGAL AABB tree to the closest source triangle and
receives its barycentrically interpolated dense PBR. This mirrors upstream's
GPU remesh texture-transfer strategy on the CPU. Sources without PBR remain grey.
This fixes solid topology; physical scale, minimum wall thickness, and supports
still need to be chosen for the target printer/material.
The same path is available without the server:
```sh
./build/examples/mesh2glb mesh.t2mesh printable.glb --print 1 0.0333
# percentages: alpha/detail size, then enclosing offset
```
On a 16 GB RTX 50-series: the 512 fine path runs image→mesh in ~110 s (~1M-vertex
512³ mesh); the 1024 cascade adds a second 1.3B-model pass and the 1024³ decoder
for a ~5M-vertex mesh (~5 min, ~10 GB VRAM, and a ~14 GB host-RAM spike for the
1024³ sparse-conv decode).
## Pipeline
```
image (RGB/RGBA)
→ background cleanup border-connected black/white → feathered alpha [C++/browser]
→ preprocess alpha bbox crop, premultiply, PIL-exact Lanczos-512 [C++, byte-exact]
→ DINOv3 ViT-L/16 [1, 1029, 1024] conditioning tokens [C++/ggml]
→ SS-flow DiT 1.3B dense DiT, 12-step CFG flow-Euler → z_s [C++/ggml]
→ SS decoder dense 3D-conv → 64³ occupancy → 32³ voxel scaffold [C++/ggml]
→ shape-SLAT DiT 1.3B sparse DiT over active voxels, 12-step CFG [C++/ggml]
→ shape VAE decoder sparse ConvNeXt U-Net, 16× up → decoded dual grid [C++/ggml]
├→ flexible dual grid → triangle mesh [C++]
└→ shape VAE encoder validated dual-grid → shape SLat + subdivision guide [C++/ggml]
→ texture-SLAT DiT shape-SLat concat conditioning [C++/ggml]
→ texture decoder replay subdivision → sparse 6-channel PBR volume [C++/ggml]
→ material sampling trilinear PBR at surface vertices [C++]
```
The **1024 cascade** (default in TRELLIS.2) adds a second pass on top: the 512
result's decoder `.upsample(×4)` predicts a denser coordinate scaffold, which is
quantized to 64³ and fed to a second 1.3B shape-SLAT flow (the 1024 model,
conditioned on a 1024-res DINOv3 encode) and the same decoder at 1024³ — a
~5M-vertex mesh. The ~49k-token HR attention only fits in VRAM via flash
attention (`sdpa_auto`); see [docs/VERIFICATION.md](docs/VERIFICATION.md).
The neural components are validated tap-by-tap against the PyTorch reference,
with separate integration regressions for subdivision guidance, sparse material
sampling, and GLB alpha preservation — see
[docs/VERIFICATION.md](docs/VERIFICATION.md). Highlights: preprocessing is
byte-exact, the DINOv3 encoder matches to rel-L2 ≤ 7e-7 across 40 taps, and the
sparse U-Net decoder is numerically exact through all four conv levels.
## Components
- **Image preprocessing + DINOv3 encoder** — `trellis2_preprocess_rgba()`
reproduces `pipeline.preprocess_image` (the has-alpha path) with a
PIL-compatible fixed-point Lanczos-3 resampler (byte-exact vs Pillow).
`trellis2_remove_solid_background_rgba()` first converts a detected near-black
or near-white background connected to the image border into softly feathered
alpha, while preserving enclosed black/white subject details and existing
alpha masks. The demo exposes automatic, forced-black, forced-white, and keep
original modes.
`trellis2_dino_encode()` runs the full DINOv3 ViT-L/16 (axial-2D RoPE,
LayerScale, exact-GELU MLP) and applies the affine-free final LayerNorm the
flow models expect — the `[1, 1029, 1024]` conditioning that used to come
from an external `dump_dinodata.py`. `dino_encode` chains them:
```sh
./build/examples/dino_encode ggufs/dino_f16.gguf image.png cond.dinodata
```
- **`.dinodata` loader** — `trellis2_load_dinodata()` still reads/writes the
precomputed conditioning tensor (1 CLS + 4 register + 1024 patch, last layer,
affine-free LN; `neg_cond = zeros_like(cond)`), for testing and CLI chaining.
- **SS-flow DiT weights** — `convert_ss_flow_to_gguf.py` converts the stage-1
`ss_flow_img_dit_1_3B_64_bf16` checkpoint to GGUF; `trellis2_ss_flow_load()`
reads it back through ggml (hparams from `trellis2.ss_flow.*` KV metadata,
weights keyed by their original checkpoint names).
- **SS-flow DiT forward pass** — `trellis2_ss_flow_forward()` builds the full
ggml graph: input projection, sinusoidal timestep + shared adaLN-Zero
modulation, 30 cross-blocks (self-attention with 3D interleaved RoPE +
QK-RMSNorm, cross-attention to the DINOv3 tokens, GELU-tanh FFN), and the
final LayerNorm + output projection. Runs on an **auto-selected backend** —
the first GPU exposed by ggml (CUDA / Metal / Vulkan / ...), falling back to
CPU, like sam3.cpp. Validated against a PyTorch f32 reference to **<1e-3
relative L2** on CPU, Metal (f32), and Metal (f16) (see *Validation* below).
- **Stage-1 sampler** — `trellis2_ss_flow_sample()` runs the full flow-Euler
loop with classifier-free guidance (interval [0.6,1.0], strength 7.5, rescale
0.7, rescale_t 5.0, 12 steps; `neg_cond = zeros`) to turn a DINOv3 cond into
the sparse-structure latent z_s. Validated against the real
`FlowEulerGuidanceIntervalSampler`: **rel L2 5.7e-3, 99.85% sign agreement**
(the SS decoder thresholds z_s at 0). Run it:
```sh
./build/examples/ss_sample ss_flow_dit_f16.gguf /path/img.dinodata out.latent
# -> z_s [8,16,16,16], occupancy(>0) ~50%
```
- **Stage-1 SS decoder** — `trellis2_ss_dec_decode()` runs the
`SparseStructureDecoder` (a dense 3D-conv ResNet) that turns the z_s latent
`[8,16³]` into an occupancy logit grid `[1,64³]`, upsampling 16→32→64 with two
`pixel_shuffle_3d` blocks. The coarse voxel scaffold is `logit > 0`. Runs fully
on the GPU (ggml `conv_3d_direct`, channel-LayerNorm, in-graph pixel-shuffle).
Validated against the real PyTorch decoder to **rel L2 5e-7 (f32) / 2e-5 (f16),
100% sign agreement** on a sampled z_s. Run it:
```sh
./build/examples/ss_decode ss_dec_f16.gguf out.latent out.occ
# -> logits [1,64,64,64], occupied(>0) grid (the coarse voxel scaffold)
```
- **Occupancy → coarse mesh** — `ss_mesh` decodes a z_s latent and exports the
`{logit = 0}` isosurface as a watertight OBJ via a self-contained marching
cubes (`examples/marching_cubes.h`, the tetrahedral / Freudenthal variant — no
256-row table, provably manifold). This is the fast preview path:
```sh
./build/examples/ss_sample ss_flow_dit_f16.gguf /path/img.dinodata z_s.latent
./build/examples/ss_mesh ss_dec_f16.gguf z_s.latent shape.obj --normalize
# -> watertight shape.obj in the centered unit cube; open in any 3D viewer
```
- **Shape-SLAT flow + decoder (fine geometry)** — `trellis2_slat_flow_sample()`
runs the sparse 1.3B DiT over the active voxels of the 32³ scaffold (same
block structure as the SS-flow DiT, 3D RoPE over each voxel's coords),
denormalized with `shape_slat_normalization` baked into the GGUF.
`trellis2_shape_dec_decode()` runs `FlexiDualGridVaeDecoder` — a sparse
ConvNeXt U-Net whose 3×3×3 submanifold convolutions are expressed as 27
gather+GEMM steps, with each level's learned subdivision growing the active
set (32³ → 512³, 16×). `examples/flexible_dual_grid.h` turns the 7-channel
per-voxel output (dual-vertex offset, per-axis intersection flags, quad split
weight) into the triangle mesh. This is the real TRELLIS.2 geometry, driven
end-to-end by the demo server.
- **PBR texture generation** — the decoded dual grid is encoded to the shape
SLat used to condition texture flow, using the numerically validated
standalone texturing path. The decoded six-channel volume (base color,
metallic, roughness, alpha) is sampled trilinearly at the actual dual-grid
surface positions. Collapsed all-saturated outputs are rejected instead of
being persisted as apparently successful textures. The browser linearizes base
color before PBR lighting and preserves opacity. Material sampling steps are
controlled separately from geometry steps, matching upstream's defaults.
## Validate the forward pass
```sh
# 1. lossless f32 weights for an exact comparison
python convert_ss_flow_to_gguf.py --output ss_flow_dit_f32.gguf --ftype 0
# 2. PyTorch f32 reference forward -> tests/ss_flow_ref.bin
python tests/ref_ss_flow.py --dinodata /path/MushroomBoy.dinodata
# 3. build + run the C++ comparison
cmake -B build -DTRELLIS2_BUILD_TESTS=ON && cmake --build build -j
./build/tests/test_ss_flow_forward ss_flow_dit_f32.gguf tests/ss_flow_ref.bin
# -> rel L2 err ~2.8e-4, RESULT: PASS
```
## Validate the SS decoder
```sh
# 1. lossless f32 decoder weights
python convert_ss_dec_to_gguf.py --output ss_dec_f32.gguf --ftype 0
# 2. PyTorch f32 reference decode of a sampled z_s -> tests/ss_dec_ref.bin
./build/examples/ss_sample ss_flow_dit_f16.gguf /path/img.dinodata z_s.latent
python tests/ref_ss_dec.py --latent z_s.latent
# 3. build + run the C++ comparison
./build/tests/test_ss_dec ss_dec_f32.gguf tests/ss_dec_ref.bin
# -> rel L2 err ~5e-7, RESULT: PASS
```
## Convert the stage-1 weights
```sh
# needs safetensors + torch + numpy (e.g. the trellis2-shiv venv)
python convert_ss_flow_to_gguf.py --output ss_flow_dit_f16.gguf --ftype 1 # DiT
python convert_ss_dec_to_gguf.py --output ss_dec_f16.gguf --ftype 1 # decoder
```
`--model` / `--config` default to the `microsoft/TRELLIS.2-4B` HF cache
snapshot. `--ftype`: `0` = f32 (lossless upcast from bf16), `1` = f16
(default — big 2-D weight matrices only; norms/gammas/modulation stay f32),
`2` = bf16 (lossless, needs bf16-capable ggml). The f16 file is ~2.6 GB.
Inspect it (validates that ggml can read every tensor):
```sh
./build/examples/ss_flow_info ss_flow_dit_f16.gguf # metadata only
./build/examples/ss_flow_info ss_flow_dit_f16.gguf --load # + read all weights
```
## Build
```sh
git clone --recursive <this-repo> trellis2cpp
cd trellis2cpp
cmake -B build -DCMAKE_BUILD_TYPE=Release
cmake --build build -j
```
CGAL is auto-detected. Install CGAL 5.5 or newer before configuring to enable
the portable CPU print-remesh backend, or pass `-DTRELLIS2_CGAL=OFF` to disable
the probe explicitly. CMake prints whether Alpha Wrap was enabled.
For reproducible builds without system CGAL/Boost packages, let trellis2.cpp
fetch its checksum-pinned header set:
```sh
cmake -B build -DTRELLIS2_FETCH_PRINT_REMESH_DEPS=ON
```
`TRELLIS2_PRINT_REMESH_DEPS_DIR` can point multiple build variants at one
shared cache. The versions, upstream SHA-256 digests, fetch behavior, demo, and
scheduled update PRs are all owned by this repository; downstream projects
only need to pin a tested trellis2.cpp commit.
If you already cloned without `--recursive`:
```sh
git submodule update --init --recursive
```
## Try it
```sh
./build/examples/dino_info /path/to/MushroomBoy.dinodata
```
Prints the shape, token breakdown, and fingerprints (min/max/mean/sum/l2).
`min`/`max`/`count` match the matching `<stem>.dino.txt` JSON sidecar exactly
(they are true element values); `sum`/`l2` agree to float32 precision — the C++
side reduces in `double` and is slightly more accurate than numpy's float32
reduction.
## Layout
| path | what |
|----------------|--------------------------------------------------------|
| `trellis2.h` | public API (DLL-decorated, versioned) |
| `trellis2.cpp` | implementation |
| `convert_ss_flow_to_gguf.py` | stage-1 DiT checkpoint → GGUF converter |
| `convert_ss_dec_to_gguf.py` | stage-1 decoder checkpoint → GGUF converter |
| `mesh_export.{h,cpp}` | CUDA-free GLB export with direct vertex PBR or projected UV-atlas textures |
| `print_remesh.{h,cpp}` | optional CGAL Alpha Wrap reconstruction and closest-surface PBR transfer |
| `examples/` | CLI tools (`dino_info`, `ss_flow_info`, `ss_sample`, `ss_decode`, `ss_mesh`, `mesh2glb`) |
| `examples/marching_cubes.h` | single-file isosurface → OBJ extractor |
| `third_party/` | vendored `xatlas` (print-wrap and opt-in ordinary UV unwrap) |
| `ggml/` | submodule, pinned to the same commit as sam3.cpp |
| `stb/` | `stb_image.h` / `stb_image_write.h` for image I/O |
## License
MIT. See [LICENSE](LICENSE). Vendored third-party code is also MIT:
[meshoptimizer](https://github.com/zeux/meshoptimizer) (Arseny Kapoulkine) and
[xatlas](https://github.com/jpcy/xatlas) (Jonathan Young) under `third_party/`,
and `stb` (public domain / MIT).
The optional Alpha Wrap backend links against
[CGAL](https://www.cgal.org/) 5.5 or newer. CGAL's 3D Alpha Wrapping package is
GPL-3.0-or-later (or available under a commercial CGAL license), so binaries
built with `TRELLIS2_CGAL=ON` and CGAL detected are subject to those terms.
buildhip.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
rm -rf build-shared
mkdir build-shared
cd build-shared
cmake .. -DGGML_HIP=ON -DGPU_TARGETS=gfx1100 -DCMAKE_BUILD_TYPE=Release
cmake --build . --config Release -j "$(nproc)"
cd ../server
CGO_ENABLED=0 go build -o trellis2-server-linux .
cd ..
convert_dino_to_gguf.py
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
"""
Convert the DINOv3 ViT-L/16 image-conditioning encoder (HF format,
facebook/dinov3-vitl16-pretrain-lvd1689m or a mirror) to a GGUF file for
trellis2.cpp.
TRELLIS.2 uses this model (transformers DINOv3ViTModel) to turn the
preprocessed 512x512 input image into the [1, 1029, 1024] conditioning tensor
consumed by the flow models: 1 CLS + 4 register + 32x32 patch tokens, last
transformer layer output passed through an affine-free layer norm (the model's
own final `norm` layer is NOT applied).
Architecture (config.json): hidden 1024, 24 layers, 16 heads, MLP 4096 (exact
GELU, not gated), patch 16, axial 2D RoPE over patch-center coords with
theta=100, LayerScale on both residual branches, q/v/o biases but no k bias.
Self-contained like the other converters (safetensors + numpy). Tensors keep
their HF state-dict names; hyperparameters travel as `trellis2.dino.*` KV.
Usage:
python convert_dino_to_gguf.py --model models/dinov3-vitl16/model.safetensors \
--output ggufs/dino_f32.gguf --ftype 0
ftype: 0 = f32 (validation), 1 = f16 (matrices f16; norms/biases/1-D f32).
"""
import argparse
import json
import os
import struct
import sys
import numpy as np
# ── GGUF / GGML constants (must match the bundled ggml) ──────────────────────
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGUF_VT_UINT32 = 4
GGUF_VT_FLOAT32 = 6
GGUF_VT_BOOL = 7
GGUF_VT_STRING = 8
ARCH = "trellis2-dino"
KV_PREFIX = "trellis2.dino."
def _gguf_str(s: str) -> bytes:
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key: str, vtype: int, payload: bytes) -> bytes:
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_bool(key, v): return _kv(key, GGUF_VT_BOOL, struct.pack("<?", bool(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n: int, a: int = GGUF_ALIGNMENT) -> int:
return (n + a - 1) // a * a
def choose_type(name, shape, ftype: int) -> int:
if ftype == 0:
return GGML_TYPE_F32
# matrices f16; 1-D (biases, norms, layerscale) and token embeddings f32
if len(shape) < 2:
return GGML_TYPE_F32
if "cls_token" in name or "register_tokens" in name or "mask_token" in name:
return GGML_TYPE_F32
return GGML_TYPE_F16
def to_bytes(arr_f32: np.ndarray, ggml_type: int) -> bytes:
if ggml_type == GGML_TYPE_F32:
return arr_f32.astype("<f4", copy=False).tobytes()
if ggml_type == GGML_TYPE_F16:
return arr_f32.astype("<f2", copy=False).tobytes()
raise ValueError(f"unhandled ggml type {ggml_type}")
def main():
ap = argparse.ArgumentParser(description="Convert DINOv3 ViT-L/16 to GGUF")
ap.add_argument("--model", default=os.path.join(os.path.dirname(__file__),
"models", "dinov3-vitl16", "model.safetensors"))
ap.add_argument("--config", default=None, help="config.json (default: alongside model)")
ap.add_argument("--output", default="dino.gguf")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1])
args = ap.parse_args()
from safetensors.numpy import load_file
cfg_path = args.config or os.path.join(os.path.dirname(args.model), "config.json")
with open(cfg_path) as f:
cfg = json.load(f)
if cfg.get("model_type") != "dinov3_vit":
sys.exit(f"error: unexpected model_type {cfg.get('model_type')!r}")
if cfg.get("use_gated_mlp"):
sys.exit("error: gated MLP variant not supported")
print(f"model : {args.model}")
print(f"output: {args.output} (ftype={args.ftype})")
print(f"arch : hidden={cfg['hidden_size']} layers={cfg['num_hidden_layers']} "
f"heads={cfg['num_attention_heads']} mlp={cfg['intermediate_size']} "
f"patch={cfg['patch_size']} registers={cfg['num_register_tokens']} "
f"rope_theta={cfg['rope_theta']}")
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", "dinov3-vitl16-pretrain-lvd1689m"),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "hidden_size", cfg["hidden_size"]),
kv_u32(KV_PREFIX + "n_layers", cfg["num_hidden_layers"]),
kv_u32(KV_PREFIX + "n_heads", cfg["num_attention_heads"]),
kv_u32(KV_PREFIX + "intermediate_size", cfg["intermediate_size"]),
kv_u32(KV_PREFIX + "patch_size", cfg["patch_size"]),
kv_u32(KV_PREFIX + "num_register_tokens", cfg["num_register_tokens"]),
kv_f32(KV_PREFIX + "layer_norm_eps", cfg["layer_norm_eps"]),
kv_f32(KV_PREFIX + "rope_theta", cfg["rope_theta"]),
kv_bool(KV_PREFIX + "key_bias", cfg.get("key_bias", False)),
# ImageNet normalization baked in so inference needs no external config.
kv_f32(KV_PREFIX + "image_mean.0", 0.485),
kv_f32(KV_PREFIX + "image_mean.1", 0.456),
kv_f32(KV_PREFIX + "image_mean.2", 0.406),
kv_f32(KV_PREFIX + "image_std.0", 0.229),
kv_f32(KV_PREFIX + "image_std.1", 0.224),
kv_f32(KV_PREFIX + "image_std.2", 0.225),
]
print("loading state_dict...")
sd = load_file(args.model)
tensors = []
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0}
for name in sorted(sd.keys()):
arr = np.asarray(sd[name]).astype(np.float32)
# squeeze leading singleton batch dims of the token embeddings so they
# arrive as [n_tokens, hidden] / [hidden]
if name.endswith(("cls_token", "mask_token", "register_tokens")):
arr = arr.reshape(arr.shape[-2], arr.shape[-1]) if arr.ndim == 3 else arr
shape = tuple(arr.shape)
# patch_embeddings.weight [OC, IC, KH, KW]: bytes already match ggml's
# ne=[KW, KH, IC, OC] expectation — dims just get written reversed.
gtype = choose_type(name, shape, args.ftype)
raw = to_bytes(np.ascontiguousarray(arr), gtype)
dims = list(reversed(shape)) if len(shape) > 0 else [1]
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]})")
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(args.output, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {args.output} ({os.path.getsize(args.output):,} bytes)")
if __name__ == "__main__":
main()
convert_shape_dec_to_gguf.py
+167
View File
@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 shape-SLAT VAE decoder checkpoint
(shape_dec_next_dc_f16c32_fp16.safetensors) to a GGUF file for trellis2.cpp.
This is FlexiDualGridVaeDecoder (a SparseUnetVaeDecoder): a sparse ConvNeXt
U-Net decoder that turns the 32-channel structured latent on active voxels
into 7 channels per voxel at 16x the input resolution (dual-vertex offset,
per-axis intersection flags, quad split weight) for flexible-dual-grid mesh
extraction. Architecture (config shape_dec_next_dc_f16c32_fp16.json):
model_channels [1024, 512, 256, 128, 64], num_blocks [4, 16, 8, 4, 0],
SparseConvNeXtBlock3d blocks, SparseResBlockC2S3d up-blocks, out 7.
Checkpoint conv weights are stored in FlexGEMM layout [Co, kD, kH, kW, Ci]
(permuted at module init, then saved). ggml tensors are 4-D, so the kernel
axes merge: we reshape to [Co, kD*kH*kW, Ci] (identical bytes) and the C++
side slices per kernel offset into [Ci, Co] GEMM operands.
Usage:
python convert_shape_dec_to_gguf.py --output shape_dec.gguf --ftype 1
ftype: 0 = f32 (lossless upcast, validation), 1 = f16 (default).
"""
import argparse
import json
import os
import struct
import sys
import numpy as np
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGUF_VT_UINT32 = 4
GGUF_VT_FLOAT32 = 6
GGUF_VT_STRING = 8
ARCH = "trellis2-shape-dec"
KV_PREFIX = "trellis2.shape_dec."
def _gguf_str(s):
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key, vtype, payload):
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n, a=GGUF_ALIGNMENT):
return (n + a - 1) // a * a
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 shape decoder to GGUF")
ap.add_argument("--model", default=os.path.join(os.path.dirname(__file__),
"models", "TRELLIS.2-4B", "ckpts", "shape_dec_next_dc_f16c32_fp16.safetensors"))
ap.add_argument("--config", default=None)
ap.add_argument("--output", default="shape_dec.gguf")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1])
args = ap.parse_args()
import torch
from safetensors.torch import load_file
cfg_path = args.config or (os.path.splitext(args.model)[0] + ".json")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
channels = a["model_channels"]
nblocks = a["num_blocks"]
print(f"model : {args.model}")
print(f"output: {args.output} (ftype={args.ftype})")
print(f"arch : channels={channels} blocks={nblocks} latent={a['latent_channels']}")
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", "shape_dec_next_dc_f16c32_fp16"),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "latent_channels", a["latent_channels"]),
kv_u32(KV_PREFIX + "out_channels", 7),
kv_u32(KV_PREFIX + "n_levels", len(channels)),
kv_f32(KV_PREFIX + "norm_eps", 1e-6),
kv_f32(KV_PREFIX + "voxel_margin", 0.5),
]
for i, ch in enumerate(channels):
metadata.append(kv_u32(KV_PREFIX + f"channels.{i}", ch))
for i, nb in enumerate(nblocks):
metadata.append(kv_u32(KV_PREFIX + f"num_blocks.{i}", nb))
print("loading state_dict...")
sd = load_file(args.model)
tensors = []
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0}
for name in sorted(sd.keys()):
t = sd[name].float()
arr = t.numpy().astype(np.float32)
shape = tuple(arr.shape)
if arr.ndim == 5: # FlexGEMM conv weight [Co, kD, kH, kW, Ci] -> [Co, 27, Ci]
Co, kD, kH, kW, Ci = shape
arr = np.ascontiguousarray(arr).reshape(Co, kD * kH * kW, Ci)
shape = arr.shape
gtype = GGML_TYPE_F32
if args.ftype == 1 and len(shape) >= 2:
gtype = GGML_TYPE_F16
raw = (arr.astype("<f2") if gtype == GGML_TYPE_F16 else arr.astype("<f4")).tobytes()
dims = list(reversed(shape)) if len(shape) > 0 else [1]
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]})")
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(args.output, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {args.output} ({os.path.getsize(args.output):,} bytes)")
if __name__ == "__main__":
main()
convert_shape_enc_to_gguf.py
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 shape-SLAT VAE encoder checkpoint
(shape_enc_next_dc_f16c32_fp16.safetensors) to a GGUF file for trellis2.cpp.
This is FlexiDualGridVaeEncoder (a SparseUnetVaeEncoder): the mirror of the
shape/tex decoders. It ingests the res-1024 flexible-dual-grid of the input
mesh as a 6-channel sparse tensor -- 3 dual-vertex offsets + 3 per-axis
intersection flags (concatenated in FlexiDualGridVaeEncoder.forward) -- and
downsamples 16x (four SparseResBlockS2C3d Spatial2Channel steps) to a 32-channel
latent on the res-64 grid. The texture pipeline uses this latent as concat_cond
for the tex flow, and the Spatial2Channel steps record the per-level
subdivision the tex decoder replays to rebuild the res-1024 voxel set.
model_channels [64,128,256,512,1024], num_blocks [0,4,8,16,4],
SparseConvNeXtBlock3d blocks, SparseResBlockS2C3d down-blocks,
input_layer 6->64, to_latent 1024->2*32 (mean/logvar; we take mean).
Conv weights are FlexGEMM [Co,kD,kH,kW,Ci] -> reshaped to [Co, kD*kH*kW, Ci].
Usage: python convert_shape_enc_to_gguf.py --output ggufs/shape_enc_f16.gguf --ftype 1
"""
import argparse
import json
import os
import struct
import numpy as np
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGUF_VT_UINT32 = 4
GGUF_VT_FLOAT32 = 6
GGUF_VT_STRING = 8
ARCH = "trellis2-shape-enc"
KV_PREFIX = "trellis2.shape_enc."
def _gguf_str(s):
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key, vtype, payload):
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n, a=GGUF_ALIGNMENT):
return (n + a - 1) // a * a
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 shape encoder to GGUF")
ap.add_argument("--model", default=os.path.join(os.path.dirname(__file__),
"models", "TRELLIS.2-4B", "ckpts", "shape_enc_next_dc_f16c32_fp16.safetensors"))
ap.add_argument("--config", default=None)
ap.add_argument("--output", default="shape_enc.gguf")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1])
args = ap.parse_args()
from safetensors.torch import load_file
cfg_path = args.config or (os.path.splitext(args.model)[0] + ".json")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
channels = a["model_channels"]
nblocks = a["num_blocks"]
latent = a["latent_channels"]
print(f"model : {args.model}")
print(f"output: {args.output} (ftype={args.ftype})")
print(f"arch : channels={channels} blocks={nblocks} latent={latent}")
sd_keys_in = None
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", "shape_enc_next_dc_f16c32_fp16"),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "in_channels", 6), # 3 dual-vertex offsets + 3 intersection flags
kv_u32(KV_PREFIX + "latent_channels", latent),
kv_u32(KV_PREFIX + "n_levels", len(channels)),
kv_f32(KV_PREFIX + "norm_eps", 1e-6),
]
for i, ch in enumerate(channels):
metadata.append(kv_u32(KV_PREFIX + f"channels.{i}", ch))
for i, nb in enumerate(nblocks):
metadata.append(kv_u32(KV_PREFIX + f"num_blocks.{i}", nb))
print("loading state_dict...")
from safetensors.torch import load_file
sd = load_file(args.model)
# sanity: input_layer must be 6-channel in
il = sd["input_layer.weight"]
assert il.shape[1] == 6, f"encoder input_layer expects 6 channels, got {il.shape}"
tensors = []
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0}
for name in sorted(sd.keys()):
arr = sd[name].float().numpy().astype(np.float32)
shape = tuple(arr.shape)
if arr.ndim == 5: # FlexGEMM conv [Co,kD,kH,kW,Ci] -> [Co, 27, Ci]
Co, kD, kH, kW, Ci = shape
arr = np.ascontiguousarray(arr).reshape(Co, kD * kH * kW, Ci)
shape = arr.shape
gtype = GGML_TYPE_F32
if args.ftype == 1 and len(shape) >= 2:
gtype = GGML_TYPE_F16
raw = (arr.astype("<f2") if gtype == GGML_TYPE_F16 else arr.astype("<f4")).tobytes()
dims = list(reversed(shape)) if len(shape) > 0 else [1]
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]})")
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(args.output, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {args.output} ({os.path.getsize(args.output):,} bytes)")
if __name__ == "__main__":
main()
convert_slat_flow_to_gguf.py
+246
View File
@@ -0,0 +1,246 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 shape-SLAT flow DiT checkpoint (sparse DiT over active
voxels; identical block structure to the SS-flow model, RoPE over voxel coords)
(slat_flow_img2shape_dit_1_3B_512_bf16.safetensors) to a GGUF file for trellis2.cpp.
This is the stage-1 generator: a ~1.3B-param DiT with adaLN-Zero modulation
(share_mod), self-attention + cross-attention to the DINOv3 cond tokens, 3D
RoPE, and QK-RMSNorm. See trellis2/models/sparse_structure_flow.py.
Like the sam3.cpp converters this is a self-contained script (only safetensors
+ numpy + torch). It writes a standard GGUF v3 file — no `gguf` package needed —
so the C++ side loads it with ggml's built-in gguf_init_from_file(): tensors are
keyed by their original checkpoint names and hyperparameters travel as KV
metadata under the `trellis2.slat_flow.*` namespace.
Usage:
python convert_ss_flow_to_gguf.py \
--model /path/to/slat_flow_img2shape_dit_1_3B_512_bf16.safetensors \
--output ss_flow_dit.gguf --ftype 1
# --model/--config default to the microsoft/TRELLIS.2-4B HF cache snapshot.
ftype: 0 = f32 (lossless upcast from the bf16 checkpoint),
1 = f16 (default; big 2-D weight matrices only, norms/gammas stay f32),
2 = bf16 (lossless, native checkpoint precision; needs bf16-capable ggml).
"""
import argparse
import glob
import json
import os
import struct
import sys
import numpy as np
# ── GGUF / GGML constants (must match the bundled ggml) ──────────────────────
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
# GGML tensor types
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGML_TYPE_BF16 = 30
# GGUF metadata value types
GGUF_VT_UINT32 = 4
GGUF_VT_INT32 = 5
GGUF_VT_FLOAT32 = 6
GGUF_VT_BOOL = 7
GGUF_VT_STRING = 8
ARCH = "trellis2-slat-flow"
KV_PREFIX = "trellis2.slat_flow."
DEFAULT_SNAPSHOT = os.path.expanduser(
"~/.cache/huggingface/hub/models--microsoft--TRELLIS.2-4B/snapshots/*/ckpts"
)
CKPT_STEM = "slat_flow_img2shape_dit_1_3B_512_bf16"
# ── GGUF writer (minimal, v3) ────────────────────────────────────────────────
def _gguf_str(s: str) -> bytes:
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key: str, vtype: int, payload: bytes) -> bytes:
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_i32(key, v): return _kv(key, GGUF_VT_INT32, struct.pack("<i", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_bool(key, v): return _kv(key, GGUF_VT_BOOL, struct.pack("<?", bool(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n: int, a: int = GGUF_ALIGNMENT) -> int:
return (n + a - 1) // a * a
# ── ftype policy ─────────────────────────────────────────────────────────────
def choose_type(name: str, shape, ftype: int) -> int:
"""Pick the on-disk ggml type for a tensor given the requested ftype."""
if ftype == 0:
return GGML_TYPE_F32
if ftype == 2:
return GGML_TYPE_BF16
# ftype == 1: f16 for the big 2-D weight matrices, f32 for everything that
# is precision-sensitive (norm gammas, modulation, biases, all 1-D).
keep_f32 = ("gamma" in name) or ("modulation" in name) or ("norm" in name)
if len(shape) >= 2 and not keep_f32:
return GGML_TYPE_F16
return GGML_TYPE_F32
def to_bytes(t, ggml_type: int) -> bytes:
"""torch tensor -> raw little-endian bytes in the chosen ggml type."""
import torch
t = t.detach().cpu().contiguous()
if ggml_type == GGML_TYPE_F32:
return t.float().numpy().astype("<f4", copy=False).tobytes()
if ggml_type == GGML_TYPE_F16:
return t.float().numpy().astype("<f2", copy=False).tobytes()
if ggml_type == GGML_TYPE_BF16:
# bf16 == upper 16 bits of f32, round-to-nearest-even.
u32 = t.float().numpy().view(np.uint32)
rounded = (u32 + 0x7FFF + ((u32 >> 16) & 1)) >> 16
return rounded.astype("<u2", copy=False).tobytes()
raise ValueError(f"unhandled ggml type {ggml_type}")
def resolve_paths(args):
model = args.model
if model is None:
hits = sorted(glob.glob(os.path.join(DEFAULT_SNAPSHOT, CKPT_STEM + ".safetensors")))
if not hits:
sys.exit("error: --model not given and no TRELLIS.2-4B snapshot found in HF cache")
model = hits[-1]
cfg = args.config or (os.path.splitext(model)[0] + ".json")
out = args.output or (CKPT_STEM + ".gguf")
return model, cfg, out
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 SS flow DiT to GGUF")
ap.add_argument("--model", default=None, help="path to ...ss_flow...safetensors (default: HF cache)")
ap.add_argument("--config", default=None, help="path to matching .json (default: alongside model)")
ap.add_argument("--output", default=None, help="output .gguf (default: <stem>.gguf)")
ap.add_argument("--pipeline-json", default=None, help="pipeline.json with shape_slat_normalization (default: alongside ckpts dir)")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1, 2],
help="0=f32, 1=f16 (default), 2=bf16")
args = ap.parse_args()
from safetensors.torch import load_file
model_path, cfg_path, out_path = resolve_paths(args)
print(f"model : {model_path}")
print(f"config: {cfg_path}")
print(f"output: {out_path} (ftype={args.ftype})")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
print(f"arch : {cfg.get('name')} {a['num_blocks']} blocks, "
f"d={a['model_channels']}, heads={a['num_heads']}, cond={a['cond_channels']}")
rope_freq = a.get("rope_freq", (1.0, 10000.0))
# ── KV metadata ──────────────────────────────────────────────────────────
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", CKPT_STEM),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "resolution", a["resolution"]),
kv_u32(KV_PREFIX + "in_channels", a["in_channels"]),
kv_u32(KV_PREFIX + "out_channels", a["out_channels"]),
kv_u32(KV_PREFIX + "model_channels", a["model_channels"]),
kv_u32(KV_PREFIX + "cond_channels", a["cond_channels"]),
kv_u32(KV_PREFIX + "num_blocks", a["num_blocks"]),
kv_u32(KV_PREFIX + "num_heads", a["num_heads"]),
kv_f32(KV_PREFIX + "mlp_ratio", a["mlp_ratio"]),
kv_str(KV_PREFIX + "pe_mode", a.get("pe_mode", "rope")),
kv_bool(KV_PREFIX + "share_mod", a.get("share_mod", False)),
kv_bool(KV_PREFIX + "qk_rms_norm", a.get("qk_rms_norm", False)),
kv_bool(KV_PREFIX + "qk_rms_norm_cross", a.get("qk_rms_norm_cross", False)),
kv_f32(KV_PREFIX + "rope_freq_min", float(rope_freq[0])),
kv_f32(KV_PREFIX + "rope_freq_base", float(rope_freq[1])),
]
# shape_slat_normalization (pipeline.json): the sampler output is
# de-normalized with these before the decoder.
pj_path = args.pipeline_json or os.path.join(
os.path.dirname(os.path.dirname(model_path)), "pipeline.json")
with open(pj_path) as f:
norm = json.load(f)["args"]["shape_slat_normalization"]
assert len(norm["mean"]) == a["out_channels"]
for i, v in enumerate(norm["mean"]):
metadata.append(kv_f32(KV_PREFIX + f"norm_mean.{i}", v))
for i, v in enumerate(norm["std"]):
metadata.append(kv_f32(KV_PREFIX + f"norm_std.{i}", v))
# ── tensors ──────────────────────────────────────────────────────────────
print("loading state_dict...")
sd = load_file(model_path)
tensors = [] # (name, ggml_type, dims_ggml_order, raw_bytes)
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0, GGML_TYPE_BF16: 0}
for name in sorted(sd.keys()):
t = sd[name]
shape = tuple(t.shape)
gtype = choose_type(name, shape, args.ftype)
raw = to_bytes(t, gtype)
dims = list(reversed(shape)) if len(shape) > 0 else [1] # ggml ne[] order
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} "
f"(f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]}, bf16={counts[GGML_TYPE_BF16]})")
# ── assemble header + infos, compute aligned data offsets ────────────────
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(out_path, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
cur = fout.tell() # already aligned per loop invariant
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes)")
if __name__ == "__main__":
main()
convert_ss_dec_to_gguf.py
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 sparse-structure DECODER checkpoint
(ss_dec_conv3d_16l8_fp16.safetensors) to a GGUF file for trellis2.cpp.
This is the stage-1 decoder D_S: a dense 3D-conv ResNet that turns the
sparse-structure latent z_s ([8, 16, 16, 16]) into an occupancy logit grid at
64^3 ([1, 64, 64, 64]). It upsamples 16 -> 32 -> 64 with two pixel-shuffle
blocks. Architecture (config ss_dec_conv3d_16l8_fp16.json):
SparseStructureDecoder(out_channels=1, latent_channels=8,
num_res_blocks=2, num_res_blocks_middle=2,
channels=[512, 128, 32], norm_type="layer")
Like convert_ss_flow_to_gguf.py this is self-contained (safetensors + numpy +
torch) and writes a standard GGUF v3 file read back by ggml's
gguf_init_from_file(). Hyperparameters travel as KV metadata under the
`trellis2.ss_dec.*` namespace; tensors keep their original checkpoint names.
Conv3d weights are 5-D in PyTorch ([OC, IC, kD, kH, kW]). ggml tensors are 4-D,
and ggml_conv_3d_direct wants the kernel as ne = [kW, kH, kD, IC*OC] with the
merged channel index packed oc*IC + ic. A C-contiguous [OC, IC, kD, kH, kW]
array reshaped to [OC*IC, kD, kH, kW] is exactly that packing (and identical
bytes), so we just reshape before writing — no permute, no data movement.
Usage:
python convert_ss_dec_to_gguf.py --output ss_dec.gguf --ftype 0
# --model/--config default to the microsoft/TRELLIS-image-large HF snapshot.
ftype: 0 = f32 (lossless upcast from the fp16 checkpoint; use for validation),
1 = f16 (default; conv weight matrices f16, norms/biases f32).
"""
import argparse
import glob
import json
import os
import struct
import sys
import numpy as np
# ── GGUF / GGML constants (must match the bundled ggml) ──────────────────────
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGUF_VT_UINT32 = 4
GGUF_VT_INT32 = 5
GGUF_VT_FLOAT32 = 6
GGUF_VT_BOOL = 7
GGUF_VT_STRING = 8
ARCH = "trellis2-ss-dec"
KV_PREFIX = "trellis2.ss_dec."
DEFAULT_SNAPSHOT = os.path.expanduser(
"~/.cache/huggingface/hub/models--microsoft--TRELLIS-image-large/snapshots/*/ckpts"
)
CKPT_STEM = "ss_dec_conv3d_16l8_fp16"
# ── GGUF writer (minimal, v3) ────────────────────────────────────────────────
def _gguf_str(s: str) -> bytes:
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key: str, vtype: int, payload: bytes) -> bytes:
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_bool(key, v): return _kv(key, GGUF_VT_BOOL, struct.pack("<?", bool(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n: int, a: int = GGUF_ALIGNMENT) -> int:
return (n + a - 1) // a * a
def choose_type(shape, ftype: int) -> int:
"""f32 always for ftype 0; for ftype 1 the big conv weight matrices are f16
while everything 1-D (biases, norm weight/bias) stays f32."""
if ftype == 0:
return GGML_TYPE_F32
return GGML_TYPE_F16 if len(shape) >= 2 else GGML_TYPE_F32
def to_bytes(arr_f32: np.ndarray, ggml_type: int) -> bytes:
if ggml_type == GGML_TYPE_F32:
return arr_f32.astype("<f4", copy=False).tobytes()
if ggml_type == GGML_TYPE_F16:
return arr_f32.astype("<f2", copy=False).tobytes()
raise ValueError(f"unhandled ggml type {ggml_type}")
def resolve_paths(args):
model = args.model
if model is None:
hits = sorted(glob.glob(os.path.join(DEFAULT_SNAPSHOT, CKPT_STEM + ".safetensors")))
if not hits:
sys.exit("error: --model not given and no TRELLIS-image-large snapshot in HF cache")
model = hits[-1]
cfg = args.config or (os.path.splitext(model)[0] + ".json")
out = args.output or (CKPT_STEM + ".gguf")
return model, cfg, out
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 SS decoder to GGUF")
ap.add_argument("--model", default=None, help="path to ...ss_dec...safetensors (default: HF cache)")
ap.add_argument("--config", default=None, help="path to matching .json (default: alongside model)")
ap.add_argument("--output", default=None, help="output .gguf (default: <stem>.gguf)")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1],
help="0=f32 (lossless, for validation), 1=f16 (default)")
args = ap.parse_args()
from safetensors.numpy import load_file
model_path, cfg_path, out_path = resolve_paths(args)
print(f"model : {model_path}")
print(f"config: {cfg_path}")
print(f"output: {out_path} (ftype={args.ftype})")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
channels = a["channels"]
print(f"arch : {cfg.get('name')} channels={channels}, "
f"res_blocks={a['num_res_blocks']}, mid={a['num_res_blocks_middle']}, "
f"latent={a['latent_channels']}, out={a['out_channels']}")
# ── KV metadata ──────────────────────────────────────────────────────────
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", CKPT_STEM),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "out_channels", a["out_channels"]),
kv_u32(KV_PREFIX + "latent_channels", a["latent_channels"]),
kv_u32(KV_PREFIX + "num_res_blocks", a["num_res_blocks"]),
kv_u32(KV_PREFIX + "num_res_blocks_middle", a["num_res_blocks_middle"]),
kv_u32(KV_PREFIX + "n_levels", len(channels)),
kv_str(KV_PREFIX + "norm_type", a.get("norm_type", "layer")),
kv_f32(KV_PREFIX + "norm_eps", 1e-5),
]
for i, ch in enumerate(channels):
metadata.append(kv_u32(KV_PREFIX + f"channels.{i}", ch))
# ── tensors ──────────────────────────────────────────────────────────────
print("loading state_dict...")
sd = load_file(model_path) # numpy arrays (fp16/fp32 as stored)
tensors = [] # (name, ggml_type, dims_ggml_order, raw_bytes)
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0}
for name in sorted(sd.keys()):
arr = np.asarray(sd[name]).astype(np.float32)
shape = tuple(arr.shape)
if arr.ndim == 5: # Conv3d weight [OC, IC, kD, kH, kW] -> [OC*IC, kD, kH, kW]
OC, IC, kD, kH, kW = shape
arr = np.ascontiguousarray(arr).reshape(OC * IC, kD, kH, kW)
shape = arr.shape
gtype = choose_type(shape, args.ftype)
raw = to_bytes(np.ascontiguousarray(arr), gtype)
dims = list(reversed(shape)) if len(shape) > 0 else [1] # ggml ne[] order
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]})")
# ── assemble header + infos, compute aligned data offsets ────────────────
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(out_path, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes)")
if __name__ == "__main__":
main()
convert_ss_flow_to_gguf.py
+232
View File
@@ -0,0 +1,232 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 sparse-structure flow DiT checkpoint
(ss_flow_img_dit_1_3B_64_bf16.safetensors) to a GGUF file for trellis2.cpp.
This is the stage-1 generator: a ~1.3B-param DiT with adaLN-Zero modulation
(share_mod), self-attention + cross-attention to the DINOv3 cond tokens, 3D
RoPE, and QK-RMSNorm. See trellis2/models/sparse_structure_flow.py.
Like the sam3.cpp converters this is a self-contained script (only safetensors
+ numpy + torch). It writes a standard GGUF v3 file — no `gguf` package needed —
so the C++ side loads it with ggml's built-in gguf_init_from_file(): tensors are
keyed by their original checkpoint names and hyperparameters travel as KV
metadata under the `trellis2.ss_flow.*` namespace.
Usage:
python convert_ss_flow_to_gguf.py \
--model /path/to/ss_flow_img_dit_1_3B_64_bf16.safetensors \
--output ss_flow_dit.gguf --ftype 1
# --model/--config default to the microsoft/TRELLIS.2-4B HF cache snapshot.
ftype: 0 = f32 (lossless upcast from the bf16 checkpoint),
1 = f16 (default; big 2-D weight matrices only, norms/gammas stay f32),
2 = bf16 (lossless, native checkpoint precision; needs bf16-capable ggml).
"""
import argparse
import glob
import json
import os
import struct
import sys
import numpy as np
# ── GGUF / GGML constants (must match the bundled ggml) ──────────────────────
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
# GGML tensor types
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGML_TYPE_BF16 = 30
# GGUF metadata value types
GGUF_VT_UINT32 = 4
GGUF_VT_INT32 = 5
GGUF_VT_FLOAT32 = 6
GGUF_VT_BOOL = 7
GGUF_VT_STRING = 8
ARCH = "trellis2-ss-flow"
KV_PREFIX = "trellis2.ss_flow."
DEFAULT_SNAPSHOT = os.path.expanduser(
"~/.cache/huggingface/hub/models--microsoft--TRELLIS.2-4B/snapshots/*/ckpts"
)
CKPT_STEM = "ss_flow_img_dit_1_3B_64_bf16"
# ── GGUF writer (minimal, v3) ────────────────────────────────────────────────
def _gguf_str(s: str) -> bytes:
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key: str, vtype: int, payload: bytes) -> bytes:
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_i32(key, v): return _kv(key, GGUF_VT_INT32, struct.pack("<i", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_bool(key, v): return _kv(key, GGUF_VT_BOOL, struct.pack("<?", bool(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n: int, a: int = GGUF_ALIGNMENT) -> int:
return (n + a - 1) // a * a
# ── ftype policy ─────────────────────────────────────────────────────────────
def choose_type(name: str, shape, ftype: int) -> int:
"""Pick the on-disk ggml type for a tensor given the requested ftype."""
if ftype == 0:
return GGML_TYPE_F32
if ftype == 2:
return GGML_TYPE_BF16
# ftype == 1: f16 for the big 2-D weight matrices, f32 for everything that
# is precision-sensitive (norm gammas, modulation, biases, all 1-D).
keep_f32 = ("gamma" in name) or ("modulation" in name) or ("norm" in name)
if len(shape) >= 2 and not keep_f32:
return GGML_TYPE_F16
return GGML_TYPE_F32
def to_bytes(t, ggml_type: int) -> bytes:
"""torch tensor -> raw little-endian bytes in the chosen ggml type."""
import torch
t = t.detach().cpu().contiguous()
if ggml_type == GGML_TYPE_F32:
return t.float().numpy().astype("<f4", copy=False).tobytes()
if ggml_type == GGML_TYPE_F16:
return t.float().numpy().astype("<f2", copy=False).tobytes()
if ggml_type == GGML_TYPE_BF16:
# bf16 == upper 16 bits of f32, round-to-nearest-even.
u32 = t.float().numpy().view(np.uint32)
rounded = (u32 + 0x7FFF + ((u32 >> 16) & 1)) >> 16
return rounded.astype("<u2", copy=False).tobytes()
raise ValueError(f"unhandled ggml type {ggml_type}")
def resolve_paths(args):
model = args.model
if model is None:
hits = sorted(glob.glob(os.path.join(DEFAULT_SNAPSHOT, CKPT_STEM + ".safetensors")))
if not hits:
sys.exit("error: --model not given and no TRELLIS.2-4B snapshot found in HF cache")
model = hits[-1]
cfg = args.config or (os.path.splitext(model)[0] + ".json")
out = args.output or (CKPT_STEM + ".gguf")
return model, cfg, out
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 SS flow DiT to GGUF")
ap.add_argument("--model", default=None, help="path to ...ss_flow...safetensors (default: HF cache)")
ap.add_argument("--config", default=None, help="path to matching .json (default: alongside model)")
ap.add_argument("--output", default=None, help="output .gguf (default: <stem>.gguf)")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1, 2],
help="0=f32, 1=f16 (default), 2=bf16")
args = ap.parse_args()
from safetensors.torch import load_file
model_path, cfg_path, out_path = resolve_paths(args)
print(f"model : {model_path}")
print(f"config: {cfg_path}")
print(f"output: {out_path} (ftype={args.ftype})")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
print(f"arch : {cfg.get('name')} {a['num_blocks']} blocks, "
f"d={a['model_channels']}, heads={a['num_heads']}, cond={a['cond_channels']}")
rope_freq = a.get("rope_freq", (1.0, 10000.0))
# ── KV metadata ──────────────────────────────────────────────────────────
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", CKPT_STEM),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "resolution", a["resolution"]),
kv_u32(KV_PREFIX + "in_channels", a["in_channels"]),
kv_u32(KV_PREFIX + "out_channels", a["out_channels"]),
kv_u32(KV_PREFIX + "model_channels", a["model_channels"]),
kv_u32(KV_PREFIX + "cond_channels", a["cond_channels"]),
kv_u32(KV_PREFIX + "num_blocks", a["num_blocks"]),
kv_u32(KV_PREFIX + "num_heads", a["num_heads"]),
kv_f32(KV_PREFIX + "mlp_ratio", a["mlp_ratio"]),
kv_str(KV_PREFIX + "pe_mode", a.get("pe_mode", "rope")),
kv_bool(KV_PREFIX + "share_mod", a.get("share_mod", False)),
kv_bool(KV_PREFIX + "qk_rms_norm", a.get("qk_rms_norm", False)),
kv_bool(KV_PREFIX + "qk_rms_norm_cross", a.get("qk_rms_norm_cross", False)),
kv_f32(KV_PREFIX + "rope_freq_min", float(rope_freq[0])),
kv_f32(KV_PREFIX + "rope_freq_base", float(rope_freq[1])),
]
# ── tensors ──────────────────────────────────────────────────────────────
print("loading state_dict...")
sd = load_file(model_path)
tensors = [] # (name, ggml_type, dims_ggml_order, raw_bytes)
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0, GGML_TYPE_BF16: 0}
for name in sorted(sd.keys()):
t = sd[name]
shape = tuple(t.shape)
gtype = choose_type(name, shape, args.ftype)
raw = to_bytes(t, gtype)
dims = list(reversed(shape)) if len(shape) > 0 else [1] # ggml ne[] order
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} "
f"(f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]}, bf16={counts[GGML_TYPE_BF16]})")
# ── assemble header + infos, compute aligned data offsets ────────────────
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(out_path, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
cur = fout.tell() # already aligned per loop invariant
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes)")
if __name__ == "__main__":
main()
convert_tex_dec_to_gguf.py
+162
View File
@@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""
Convert the TRELLIS.2 texture-SLAT VAE decoder checkpoint
(tex_dec_next_dc_f16c32_fp16.safetensors) to a GGUF file for trellis2.cpp.
This is a SparseUnetVaeDecoder — the SAME sparse ConvNeXt U-Net as the shape
decoder (FlexiDualGridVaeDecoder) except:
* out_channels = 6 (PBR: base_color[3], metallic[1], roughness[1], alpha[1]),
no dual-grid geometry head;
* pred_subdiv = False -> NO `to_subdiv` layers. The decoder does not predict
which children to keep; it replays the subdivision structure recorded by the
shape encoder's Spatial2Channel steps (threaded through the SparseTensor
spatial cache). So the tex decoder reconstructs exactly the encoder's
res-1024 voxel set; the C++ side must supply that per-level subdivision.
Same architecture/weight layout as the shape decoder otherwise:
model_channels [1024,512,256,128,64], num_blocks [4,16,8,4,0],
SparseConvNeXtBlock3d blocks, SparseResBlockC2S3d up-blocks.
Conv weights are FlexGEMM [Co,kD,kH,kW,Ci] -> reshaped to [Co, kD*kH*kW, Ci].
Usage: python convert_tex_dec_to_gguf.py --output ggufs/tex_dec_f16.gguf --ftype 1
"""
import argparse
import json
import os
import struct
import numpy as np
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGUF_VT_UINT32 = 4
GGUF_VT_FLOAT32 = 6
GGUF_VT_STRING = 8
ARCH = "trellis2-tex-dec"
KV_PREFIX = "trellis2.tex_dec."
def _gguf_str(s):
b = s.encode("utf-8")
return struct.pack("<Q", len(b)) + b
def _kv(key, vtype, payload):
return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n, a=GGUF_ALIGNMENT):
return (n + a - 1) // a * a
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 texture decoder to GGUF")
ap.add_argument("--model", default=os.path.join(os.path.dirname(__file__),
"models", "TRELLIS.2-4B", "ckpts", "tex_dec_next_dc_f16c32_fp16.safetensors"))
ap.add_argument("--config", default=None)
ap.add_argument("--output", default="tex_dec.gguf")
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1])
args = ap.parse_args()
from safetensors.torch import load_file
cfg_path = args.config or (os.path.splitext(args.model)[0] + ".json")
with open(cfg_path) as f:
cfg = json.load(f)
a = cfg["args"]
channels = a["model_channels"]
nblocks = a["num_blocks"]
out_channels = a["out_channels"]
assert out_channels == 6, f"expected 6 PBR channels, got {out_channels}"
assert a.get("pred_subdiv", True) is False, "tex decoder must be pred_subdiv=False"
print(f"model : {args.model}")
print(f"output: {args.output} (ftype={args.ftype})")
print(f"arch : channels={channels} blocks={nblocks} latent={a['latent_channels']} out={out_channels}")
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", "tex_dec_next_dc_f16c32_fp16"),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "latent_channels", a["latent_channels"]),
kv_u32(KV_PREFIX + "out_channels", out_channels),
kv_u32(KV_PREFIX + "n_levels", len(channels)),
kv_f32(KV_PREFIX + "norm_eps", 1e-6),
]
for i, ch in enumerate(channels):
metadata.append(kv_u32(KV_PREFIX + f"channels.{i}", ch))
for i, nb in enumerate(nblocks):
metadata.append(kv_u32(KV_PREFIX + f"num_blocks.{i}", nb))
print("loading state_dict...")
sd = load_file(args.model)
tensors = []
counts = {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0}
for name in sorted(sd.keys()):
arr = sd[name].float().numpy().astype(np.float32)
shape = tuple(arr.shape)
if arr.ndim == 5: # FlexGEMM conv [Co,kD,kH,kW,Ci] -> [Co, 27, Ci]
Co, kD, kH, kW, Ci = shape
arr = np.ascontiguousarray(arr).reshape(Co, kD * kH * kW, Ci)
shape = arr.shape
gtype = GGML_TYPE_F32
if args.ftype == 1 and len(shape) >= 2:
gtype = GGML_TYPE_F16
raw = (arr.astype("<f2") if gtype == GGML_TYPE_F16 else arr.astype("<f4")).tobytes()
dims = list(reversed(shape)) if len(shape) > 0 else [1]
tensors.append((name, gtype, dims, raw))
counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]})")
header = bytearray()
header += GGUF_MAGIC
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata:
header += m
infos = bytearray()
offset = 0
offsets = []
for name, gtype, dims, raw in tensors:
offsets.append(offset)
offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name)
infos += struct.pack("<I", len(dims))
for d in dims:
infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype)
infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(args.output, "wb") as fout:
fout.write(header)
fout.write(infos)
fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad:
fout.write(b"\x00" * pad)
print(f"wrote {args.output} ({os.path.getsize(args.output):,} bytes)")
if __name__ == "__main__":
main()
convert_tex_flow_to_gguf.py
+173
View File
@@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""
Convert a TRELLIS.2 texture-SLAT flow DiT checkpoint
(slat_flow_imgshape2tex_dit_1_3B_{512,1024}_bf16.safetensors) to GGUF.
Architecturally identical to the shape SLAT flow (same 1536-d / 30-block /
12-head sparse DiT, adaLN-Zero share_mod, self+cross attention to DINOv3 cond,
3D RoPE, QK-RMSNorm) with ONE difference: it is conditioned on the shape via
`concat_cond`. So in_channels = 64 = 32 tex-noise + 32 shape-SLAT, out = 32.
At sample time the (normalized) shape SLAT is concatenated onto the noise before
the input layer each step.
Emits the same `trellis2.slat_flow.*` KV as the shape-flow converter so the C++
loader is shared, plus:
* trellis2.slat_flow.concat_cond_channels = 32 (0 on the shape flow)
* norm_mean/std = tex_slat_normalization (output de-norm, 32)
* concat_norm_mean/std = shape_slat_normalization (concat_cond in-norm, 32)
both read from texturing_pipeline.json.
Usage:
python convert_tex_flow_to_gguf.py \
--model models/TRELLIS.2-4B/ckpts/slat_flow_imgshape2tex_dit_1_3B_512_bf16.safetensors \
--texturing-json models/TRELLIS.2-4B/texturing_pipeline.json \
--output ggufs/tex_slat_flow_512_f16.gguf --ftype 1
"""
import argparse
import json
import os
import struct
import numpy as np
GGUF_MAGIC = b"GGUF"
GGUF_VERSION = 3
GGUF_ALIGNMENT = 32
GGML_TYPE_F32 = 0
GGML_TYPE_F16 = 1
GGML_TYPE_BF16 = 30
GGUF_VT_UINT32 = 4
GGUF_VT_FLOAT32 = 6
GGUF_VT_BOOL = 7
GGUF_VT_STRING = 8
ARCH = "trellis2-slat-flow" # shared loader with the shape SLAT flow
KV_PREFIX = "trellis2.slat_flow."
def _gguf_str(s): b = s.encode("utf-8"); return struct.pack("<Q", len(b)) + b
def _kv(key, vtype, payload): return _gguf_str(key) + struct.pack("<I", vtype) + payload
def kv_u32(key, v): return _kv(key, GGUF_VT_UINT32, struct.pack("<I", int(v)))
def kv_f32(key, v): return _kv(key, GGUF_VT_FLOAT32, struct.pack("<f", float(v)))
def kv_bool(key, v): return _kv(key, GGUF_VT_BOOL, struct.pack("<?", bool(v)))
def kv_str(key, v): return _kv(key, GGUF_VT_STRING, _gguf_str(str(v)))
def _align(n, a=GGUF_ALIGNMENT): return (n + a - 1) // a * a
def choose_type(name, shape, ftype):
if ftype == 0: return GGML_TYPE_F32
if ftype == 2: return GGML_TYPE_BF16
keep_f32 = ("gamma" in name) or ("modulation" in name) or ("norm" in name)
return GGML_TYPE_F16 if (len(shape) >= 2 and not keep_f32) else GGML_TYPE_F32
def to_bytes(t, ggml_type):
t = t.detach().cpu().contiguous()
if ggml_type == GGML_TYPE_F32: return t.float().numpy().astype("<f4", copy=False).tobytes()
if ggml_type == GGML_TYPE_F16: return t.float().numpy().astype("<f2", copy=False).tobytes()
if ggml_type == GGML_TYPE_BF16:
u32 = t.float().numpy().view(np.uint32)
return (((u32 + 0x7FFF + ((u32 >> 16) & 1)) >> 16).astype("<u2", copy=False)).tobytes()
raise ValueError(ggml_type)
def main():
ap = argparse.ArgumentParser(description="Convert TRELLIS.2 texture SLAT flow DiT to GGUF")
ap.add_argument("--model", required=True)
ap.add_argument("--config", default=None)
ap.add_argument("--texturing-json", default=None,
help="texturing_pipeline.json (tex + shape slat normalization)")
ap.add_argument("--output", required=True)
ap.add_argument("--ftype", type=int, default=1, choices=[0, 1, 2])
args = ap.parse_args()
from safetensors.torch import load_file
cfg_path = args.config or (os.path.splitext(args.model)[0] + ".json")
with open(cfg_path) as f:
a = json.load(f)["args"]
print(f"model : {args.model}")
print(f"output: {args.output} (ftype={args.ftype})")
print(f"arch : in={a['in_channels']} out={a['out_channels']} cond={a['cond_channels']} "
f"d={a['model_channels']} blks={a['num_blocks']} heads={a['num_heads']}")
concat_cond_channels = a["in_channels"] - a["out_channels"]
assert concat_cond_channels > 0, "tex flow must have concat_cond (in > out)"
tj = args.texturing_json or os.path.join(os.path.dirname(os.path.dirname(args.model)),
"texturing_pipeline.json")
with open(tj) as f:
tpa = json.load(f)["args"]
tex_norm = tpa["tex_slat_normalization"]
shape_norm = tpa["shape_slat_normalization"]
assert len(tex_norm["mean"]) == a["out_channels"], "tex_slat_norm must match out_channels"
assert len(shape_norm["mean"]) == concat_cond_channels, "shape_slat_norm must match concat_cond"
rope_freq = a.get("rope_freq", (1.0, 10000.0))
metadata = [
kv_str("general.architecture", ARCH),
kv_str("general.name", os.path.splitext(os.path.basename(args.model))[0]),
kv_u32("general.file_type", args.ftype),
kv_u32("general.alignment", GGUF_ALIGNMENT),
kv_u32(KV_PREFIX + "resolution", a["resolution"]),
kv_u32(KV_PREFIX + "in_channels", a["in_channels"]),
kv_u32(KV_PREFIX + "out_channels", a["out_channels"]),
kv_u32(KV_PREFIX + "concat_cond_channels", concat_cond_channels),
kv_u32(KV_PREFIX + "model_channels", a["model_channels"]),
kv_u32(KV_PREFIX + "cond_channels", a["cond_channels"]),
kv_u32(KV_PREFIX + "num_blocks", a["num_blocks"]),
kv_u32(KV_PREFIX + "num_heads", a["num_heads"]),
kv_f32(KV_PREFIX + "mlp_ratio", a["mlp_ratio"]),
kv_str(KV_PREFIX + "pe_mode", a.get("pe_mode", "rope")),
kv_bool(KV_PREFIX + "share_mod", a.get("share_mod", False)),
kv_bool(KV_PREFIX + "qk_rms_norm", a.get("qk_rms_norm", False)),
kv_bool(KV_PREFIX + "qk_rms_norm_cross", a.get("qk_rms_norm_cross", False)),
kv_f32(KV_PREFIX + "rope_freq_min", float(rope_freq[0])),
kv_f32(KV_PREFIX + "rope_freq_base", float(rope_freq[1])),
]
# output de-normalization (tex slat) and concat_cond in-normalization (shape slat)
for i, v in enumerate(tex_norm["mean"]): metadata.append(kv_f32(KV_PREFIX + f"norm_mean.{i}", v))
for i, v in enumerate(tex_norm["std"]): metadata.append(kv_f32(KV_PREFIX + f"norm_std.{i}", v))
for i, v in enumerate(shape_norm["mean"]): metadata.append(kv_f32(KV_PREFIX + f"concat_norm_mean.{i}", v))
for i, v in enumerate(shape_norm["std"]): metadata.append(kv_f32(KV_PREFIX + f"concat_norm_std.{i}", v))
print("loading state_dict...")
sd = load_file(args.model)
tensors, counts = [], {GGML_TYPE_F32: 0, GGML_TYPE_F16: 0, GGML_TYPE_BF16: 0}
for name in sorted(sd.keys()):
t = sd[name]; shape = tuple(t.shape)
gtype = choose_type(name, shape, args.ftype)
raw = to_bytes(t, gtype)
dims = list(reversed(shape)) if len(shape) > 0 else [1]
tensors.append((name, gtype, dims, raw)); counts[gtype] += 1
print(f"tensors: {len(tensors)} (f32={counts[GGML_TYPE_F32]}, f16={counts[GGML_TYPE_F16]}, bf16={counts[GGML_TYPE_BF16]})")
header = bytearray(GGUF_MAGIC)
header += struct.pack("<I", GGUF_VERSION)
header += struct.pack("<Q", len(tensors))
header += struct.pack("<Q", len(metadata))
for m in metadata: header += m
infos = bytearray()
offset, offsets = 0, []
for name, gtype, dims, raw in tensors:
offsets.append(offset); offset = _align(offset + len(raw))
for (name, gtype, dims, raw), off in zip(tensors, offsets):
infos += _gguf_str(name); infos += struct.pack("<I", len(dims))
for d in dims: infos += struct.pack("<Q", int(d))
infos += struct.pack("<I", gtype); infos += struct.pack("<Q", off)
pre_data = len(header) + len(infos)
pad0 = _align(pre_data) - pre_data
with open(args.output, "wb") as fout:
fout.write(header); fout.write(infos); fout.write(b"\x00" * pad0)
for (name, gtype, dims, raw), off in zip(tensors, offsets):
fout.write(raw)
pad = _align(len(raw)) - len(raw)
if pad: fout.write(b"\x00" * pad)
print(f"wrote {args.output} ({os.path.getsize(args.output):,} bytes)")
if __name__ == "__main__":
main()
docker/Dockerfile.demo
+21
View File
@@ -0,0 +1,21 @@
# Demo/runtime image: the reference image (CUDA 12.8 runtime + toolchain)
# plus a Go toolchain, so both libtrellis2.so (CUDA) and the Go demo server
# build against the SAME glibc/libstdc++ that run them. NixOS host binaries
# don't survive inside the container (different dynamic loader), hence this.
#
# Build: docker build -f docker/Dockerfile.demo -t trellis2-demo docker
# Usage: see scripts/demo.sh
FROM trellis2-ref
# The mounted source tree's CMake fetches its own pinned CGAL + Boost header
# set. The image only provides the archive tools, so dependency versions have
# one owner instead of drifting between the demo and downstream builds.
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl unzip \
&& rm -rf /var/lib/apt/lists/*
ADD https://go.dev/dl/go1.24.4.linux-amd64.tar.gz /tmp/go.tgz
RUN tar -C /usr/local -xzf /tmp/go.tgz && rm /tmp/go.tgz
ENV PATH=/usr/local/go/bin:$PATH
# go.sum is committed; `go build` fetches the single dep (purego) on demand.
# A committed server/vendor/ dir (if present) is used automatically.
docker/Dockerfile.ref
+25
View File
@@ -0,0 +1,25 @@
# Python reference environment for parity dumps.
#
# Deliberately stock-PyTorch only: no FlexGEMM / flash-attn / o-voxel / CuMesh
# CUDA extensions. Sparse attention and sparse conv are monkeypatched to dense
# equivalents by scripts/ref_common.py (batch=1 makes them equal), so reference
# activations can be produced on GPU or CPU without custom kernels — that also
# keeps the RTX 5070 Ti (sm_120) out of extension-build trouble.
#
# Build: docker build -f docker/Dockerfile.ref -t trellis2-ref docker
# Run : docker run --rm --device nvidia.com/gpu=all \
# -v "$PWD":/work -v "$PWD/../python/TRELLIS.2":/trellis2 \
# -e PYTHONPATH=/trellis2 -w /work trellis2-ref python scripts/...
FROM pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel
RUN pip install --no-cache-dir \
"transformers==4.57.1" \
safetensors \
pillow \
numpy \
easydict \
opencv-python-headless \
trimesh \
tqdm \
imageio \
gguf
docs/PLAN.md
+197
View File
@@ -0,0 +1,197 @@
# trellis2.cpp completion plan
Goal: image in → 3D mesh out, pure C++/ggml inference behind a Go demo server with a
browser mesh viewer. No PBR textures for now. Process mirrors depth-anything.cpp
(layer-by-layer parity vs the PyTorch reference) and privacy-filter.cpp (libFuzzer
harnesses on untrusted inputs, sanitizer builds).
## Pipeline target
TRELLIS.2 "512" pipeline type (non-cascade), geometry only:
```
image (RGBA preferred)
→ preprocess (alpha crop, premultiply, resize 512, ImageNet norm) [C++ port]
→ DINOv3 ViT-L/16 cond tokens [1, 1+4+1024, 1024] [C++ port — was external .dinodata]
→ SS-flow DiT (1.3B dense, 12-step CFG flow Euler) [already ported + validated]
→ SS decoder → 64³ occupancy → max_pool 32³ → coords [already ported; pool/coords new]
→ shape-SLAT flow (1.3B sparse DiT, res 32, 32ch) [to port]
→ FlexiDualGrid VAE decoder (sparse ConvNeXt U-Net, 16× up) [to port]
→ flexible dual grid → triangle mesh [to port, CPU]
→ (postprocess: fill holes — best effort, CuMesh not portable)
```
Fallback shipped at every point in time: marching-cubes preview mesh from the 64³
occupancy (already works), so the demo is demoable before the SLAT stages land.
The upstream neural background-removal net (BiRefNet/RMBG-2.0, a separate ~1 GB
model) remains out of scope. The port does include deterministic removal of
near-black/near-white backgrounds connected to the image border, with soft alpha
edges and browser controls to force or disable it. Texture and cascade status is
tracked below.
## Weights
`scripts/download_models.sh` → `models/`. DINOv3 comes from the ungated
`camenduru/dinov3-vitl16-pretrain-lvd1689m` mirror because the official
`facebook/dinov3-vitl16-pretrain-lvd1689m` is license-gated and this account has no
access yet (403). Request access and re-download officially when possible.
## Validation (depth-anything.cpp process)
- Reference activations dumped from Python via forward hooks into
`dumps/reference_<stage>.gguf` + manifest (GGUF as the dump format so C++ reads it
with the ggml API it already links).
- Reference env: docker `pytorch/pytorch:2.7.1-cuda12.8-cudnn9-devel` + stock pip
deps only. Custom CUDA backends (FlexGEMM/flash-attn/o-voxel) are NOT installed;
sparse attention (batch=1) is monkeypatched to dense SDPA and sparse conv to a
pure-torch gather-GEMM — slow but runs on CPU too, and doubles as an executable
spec for the C++ port. 16 GB VRAM is only a constraint for the reference runs;
stage-at-a-time + low_vram-style model swapping keeps us under it.
- C++ side: per-layer taps compared with `tests/parity.hpp`-style
`atol+rtol*|ref|` gate (2e-3 default), ctest with `SKIP_RETURN_CODE 77` when
fixtures are absent.
- Existing whole-stage tests (ss_flow/ss_sample/ss_dec) keep working; their
ref generators get path fixes (the `trellis2-shiv` layout doesn't exist here).
## Fuzzing (privacy-filter.cpp process)
Non-GGUF untrusted inputs, libFuzzer + ASan/UBSan (clang):
- image bytes → stb_image decode → preprocess (the demo upload path)
- `.dinodata` loader
- `.latent`/occupancy readers used by the example CLIs
GGUF model files are trusted assets (same threat model as privacy-filter).
## Demo
- `server/` Go (stdlib http + purego dlopen of `libtrellis2.so`), depth-anything
server pattern: single inference mutex, `POST /api/generate` (multipart image,
params) → job id, progress polling, `GET /api/mesh/<id>` binary mesh,
self-contained embedded `web/index.html` WebGL viewer (no CDN, no build step).
- C API additions in `trellis2.h` for: dino encode from RGBA buffer, full pipeline
run with progress callback, mesh buffer accessors.
## Order of work
1. infra: ref container, weights, fix existing ref scripts, regenerate refs,
existing 3 tests green on this machine (CPU + asan)
2. DINOv3 encoder port + per-layer validation → image→coarse-mesh e2e in C++
3. Go server + web viewer on coarse pipeline (user-visible milestone)
4. fuzz harnesses + short campaigns
5. shape-SLAT flow port (sparse DiT) + validation
6. FDG VAE decoder + flexible-dual-grid mesher + validation → real mesh in demo
7. docs (VERIFICATION.md), CUDA build check, quantized variants, benchmarks
## Status (done)
All of 1–7 complete. Image→mesh works end-to-end, C++/ggml only, coarse
(marching-cubes preview) and fine (512³ dual-grid) paths, in the Go demo. Every
stage validated tap-by-tap (docs/VERIFICATION.md): preprocessing byte-exact,
DINOv3 rel-L2 ≤7e-7, sparse U-Net decoder exact through 4 levels. CUDA build
(sm_120) works; 3D-conv decoders pinned to CPU. One fuzz bug found+fixed.
Runtime on the 16 GB RTX 5070 Ti (GPU shape decode auto-enabled): fine (512³)
**~39 s**, 1024 cascade **~133 s**. (Was 85 s for 512 with the decoder on CPU.)
### Where the fine-path time goes (measured, `TRELLIS2_TIMING=1`)
Per-stage wall-clock for one 512³ generation (f16, ~1.09 M-vertex mesh), CPU
decoder vs the auto GPU decoder:
| stage | CPU decode | GPU decode | device (GPU mode) |
|------------|-----------:|-----------:|-------------------|
| dino | 0.1 s | 0.1 s | GPU |
| ss_flow | 21 s | 21 s | GPU |
| ss_dec | 3 s | 3 s | CPU (dense CONV_3D)|
| slat_flow | 11 s | 10 s | GPU |
| shape_dec | **49 s** | **2.5 s** | **GPU** (was 59 % of the run) |
| mesh | 0.2 s | 0.2 s | CPU |
| **total** | **85 s** | **39 s** | |
The original "GPU ~30 %" reading was the *average*: the biggest stage (the
FlexiDualGrid VAE decoder) ran on the CPU with the GPU idle. It now runs on the
GPU — **~2.5 s vs ~49 s, ~20×** — placed there automatically when VRAM allows
(see the shape-decoder note under "Not done"). The flow DiTs (`ss_flow` +
`slat_flow`) are now the bulk of the run. Two facts about them, from
`TRELLIS2_TIMING`:
- Attention is now `ggml_flash_attn_ext` for **all** flow forwards, not just the
cascade's huge token counts (`sdpa_auto`, `TRELLIS2_SDPA_EXACT` restores the
old materialized softmax). Flash is bit-identical to full softmax on the CPU
(both flow forwards still validate at ~2.4e-4 / 2.9e-4 rel-L2) and on the GPU
it is ~30 % faster *and* O(L) memory — the exact path's 805 MB score matrix
`alloc_graph`-fails once the resident pipeline leaves <~2 GB free, so flash is
what keeps the flow fitting under host-VRAM pressure, not only a speedup.
- Each forward still runs at only ~17 % of the card's f16 tensor-core peak; the
matmuls already hit the f16 cores (ggml converts the f32 activations), so the
remaining slack is the f32 elementwise/permute traffic between matmuls. CUDA
graphs do not help (they `cudaGraphInstantiate`-OOM with the pipeline
resident) and are left off.
The shape decoder is a *sparse* net that keeps millions of voxels in the tensor
**row** dimension (`ne1`). ggml's CUDA CONCAT/PAD kernels launch a grid
dimension equal to that count and abort (`invalid configuration argument`) above
the 65535 grid-Y/Z limit — which is why the original port used the CPU. But
CONCAT/PAD were only used to append one missing-neighbor zero row; replacing
that with a clamp-index + 0/1-mask formulation (get_rows a valid row, multiply
the missing ones by zero — byte-identical) keeps every op inside ggml kernels
that tile the voxel dimension (`get_rows`, broadcast `mul`, `mul_mat`, `add`,
`norm`, `silu` all verified OK at 3 M voxels). The decoder then runs on the GPU
in ~2.5 s (512³) / ~12.5 s (1024³).
### VRAM-aware auto-placement of the shape decoder
The decoder's placement is decided from **measured free VRAM**
(`trellis2_gpu_free_vram()` → `cudaMemGetInfo`), not a hardcoded flag:
- At load, `t2_pipeline_load` records the free VRAM *before* the flow DiTs are
loaded (= what freeing them again reclaims) and places the decoder on the GPU
when that covers the target tier's decode peak (~3 GB at 512³, ~7.5 GB at
1024³) plus the decoder weights and a margin; otherwise CPU. `TRELLIS2_SHAPE_DEC_GPU`
/ `_CPU` force it; a CPU-only build (no GPU device) always picks CPU.
- At decode time, `ensure_decode_vram` checks free VRAM again and, if the decode
would not fit, frees the flow DiTs (all finished by then) to reclaim their
~5–7 GB; `reload_flows` brings them back on the next `t2_generate` (a few
seconds from page cache). So 512³ decodes with the flows resident (no reload
cost) while the big 1024³ decode transparently frees-and-reloads. This is the
`T2_LOAD_LOW_VRAM` idea, applied automatically only when the numbers require it.
Validated end-to-end on the 16 GB card with **no env vars**: 512 → 39 s, 1024
cascade → 133 s (the 1024³ GPU decode is 12.5 s), and three back-to-back cascade
generations exercise the free/reload path with no OOM.
## Not done (out of original scope / future)
- **PBR textures — DONE.** The validated standalone texturing path re-encodes
the decoded dual grid to condition texture flow, then trilinearly samples the
decoded six-channel PBR volume at the surface. The browser and GLB path
preserve alpha and use the correct base-color space. The first integrated
subdivision-guide path was removed after it produced collapsed materials.
- **`1024_cascade` — DONE.** Full high-resolution path: LR flow (512 model) →
`trellis2_shape_dec_upsample(×4)` → quantize+dedup to the 64³ HR scaffold →
HR flow (1024 model, cond_1024) → 1024³ decode → dual-grid mesh. Enabled by
flash attention (`sdpa_auto`) for the HR token counts. Validated by
`test_cascade`: upsample coords + HR scaffold exact (to 1 voxel of ~995k),
HR flow forward rel-L2 3.1e-4 (CPU). The 1024³ decode is the same decoder
validated exactly at the 512 tier; its explicit parity gate is behind
`TRELLIS2_CASCADE_DECODE` because it transiently needs ~14 GB host RAM.
- **Still to do:** `1536_cascade` (add the token-reduction loop + res 1536). The
`<8 GB` case is now partly covered: the shape decoder auto-frees the flow DiTs
around a GPU decode (see below), so the pieces of the `T2_LOAD_LOW_VRAM`
lifecycle exist; a full per-stage load/free mode would extend that to the flow
DiTs themselves for cards that can't hold even one 1.3B DiT + a decode.
- **Background removal** (BiRefNet/RMBG-2.0) — the demo instructs a transparent
PNG and uses the image as-is otherwise. A separate ~1 GB seg model.
- **GPU shape decoder — DONE.** The mask-conv change plus the VRAM-aware
placement + free/reload lifecycle (both described above) make the decoder run
on the GPU automatically when it fits: 512³ ~2.5 s and 1024³ ~12.5 s vs ~49 s /
minutes on the CPU. No env var required; `TRELLIS2_SHAPE_DEC_{GPU,CPU}` still
force it, and it stays on the CPU on cards too small or in a CPU-only build.
- **CUDA CONV_3D** — the SS *occupancy* decoder (`ss_dec`, 3 s) uses a genuine
dense `ggml_conv_3d_direct` with no CUDA kernel, so it stays on CPU regardless.
- **Quantized shipping variants / benchmarks / CI** — f16 is the demo default;
q8/q4 conversion + a benchmark table + a two-tier CI workflow are the natural
next hardening steps (privacy-filter.cpp has the template).
- **CPU-only SLAT sampler tightness in ctest** — the `slat` ctest is labeled
slow and validated in-container; the sampler validates tightly on CPU
(TRELLIS2_SLAT_STRICT=1) but that CPU run is minutes-long.
docs/VERIFICATION.md
+86
View File
@@ -0,0 +1,86 @@
# Verification
Every ported stage is validated numerically against the PyTorch reference, per
component, not just end-to-end — the depth-anything.cpp approach. Reference
activations are dumped from `microsoft/TRELLIS.2` (via `scripts/refgen.sh`
inside the CUDA container) into GGUF files under `dumps/`, and the C++ side is
compared tap-by-tap with `tests/parity.hpp` (gate: `|got-ref| <= atol +
rtol*|ref|`, reported as max-abs / rel-L2 / per-row).
## One-time setup
```sh
docker build -f docker/Dockerfile.ref -t trellis2-ref docker # PyTorch reference env
scripts/download_models.sh # HF checkpoints -> models/
docker run --rm -v "$PWD":/work -w /work trellis2-ref bash -c '
python convert_dino_to_gguf.py --output ggufs/dino_f32.gguf --ftype 0
python convert_ss_flow_to_gguf.py --model models/TRELLIS.2-4B/ckpts/ss_flow_img_dit_1_3B_64_bf16.safetensors --output ggufs/ss_flow_f32.gguf --ftype 0
python convert_ss_dec_to_gguf.py --model models/TRELLIS-image-large/ckpts/ss_dec_conv3d_16l8_fp16.safetensors --output ggufs/ss_dec_f32.gguf --ftype 0
python convert_slat_flow_to_gguf.py --model models/TRELLIS.2-4B/ckpts/slat_flow_img2shape_dit_1_3B_512_bf16.safetensors --pipeline-json models/TRELLIS.2-4B/pipeline.json --output ggufs/slat_flow_f32.gguf --ftype 0
python convert_shape_dec_to_gguf.py --output ggufs/shape_dec_f32.gguf --ftype 0'
scripts/refgen.sh # dump reference activations
```
## Run
```sh
cmake -B build -DTRELLIS2_BUILD_TESTS=ON && cmake --build build -j
ctest --test-dir build -LE model # fast, no assets (marching cubes, preprocess)
ctest --test-dir build # full parity (needs ggufs/ + dumps/)
```
## Parity table (f32 GGUF vs true-fp32 reference)
| stage | test | tap coverage | result |
|---|---|---|---|
| image preprocess (alpha crop, premultiply, Lanczos-512) | `test_preprocess` | full 512×512 RGB | **byte-exact** (0/786432 differ) |
| DINOv3 ViT-L/16 encoder | `test_dino` | 40 taps: embeddings, RoPE, per-layer output + first/last-layer detail (norm/attn/layerscale/mlp), final affine-free LN | **PASS**, rel-L2 ≤ 7e-7 all taps |
| SS-flow DiT forward | `test_ss_flow_forward` | full output | **PASS**, rel-L2 2.4e-4 |
| SS-flow Euler sampler (12-step CFG) | `test_ss_sample` | z_s latent | **PASS**, rel-L2 5.7e-3, sign 99.85% (CPU) |
| SS decoder (dense 3D-conv → 64³ occupancy) | `test_ss_dec` | occupancy logits | **PASS**, rel-L2 2e-5 |
| shape-SLAT flow forward | `test_slat` | full output | **PASS**, rel-L2 2.9e-4 (CPU) / 8e-4 (GPU) |
| shape-SLAT VAE decoder (sparse ConvNeXt U-Net, 4 levels, 16× up) | `test_slat` | per-level features + subdivision logits + final 7-ch output, all 5 levels | **PASS**, rel-L2 ≤ 6e-7 (levels 0–3 exact; final set within 0.0001%) |
| integrated subdivision guide | `test_slat` | all decoder levels; final guide coordinates equal decoded shape coordinates | **PASS** |
| standalone shape encoder → texture flow → texture decoder | `test_texture` | shape latent, flow forward/sampler, guided 6-channel PBR decode | parity-gated; sampler backend drift uses the documented loose gate |
| sparse PBR surface sampling | `test_pbr_sampling` | dense trilinear interpolation + sparse-boundary normalization | **PASS** |
| GLB PBR/alpha export | `test_mesh_export` | direct vertex RGBA, retained metallic/roughness, glTF alpha mode | **PASS** |
| dual-grid mesh extraction | `test_marching_cubes` (invariants) + visual | watertight-manifold, Euler characteristic, winding | **PASS** |
| **1024 cascade** — decoder upsample(×4) → 512³ coords | `test_cascade` | full coord set + quantized 64³ HR scaffold | **PASS**, set match to 0.0001% (1 voxel of 995k) |
| **1024 cascade** — HR (1024-model) flow forward | `test_cascade` | full output | **PASS**, rel-L2 ~3e-4 (CPU); ~1e-2 on GPU flash |
| **1024 cascade** — final 1024³ decode (3.97M voxels) | `test_cascade` | per-level features + subdivision + 7-ch output | **PASS**, rel-L2 ≤ 2e-2, set within 0.0001% |
Notes:
- **Flash attention (default for every flow forward).** `sdpa_auto()` uses
`ggml_flash_attn_ext` for both flow DiTs at all token counts;
`TRELLIS2_SDPA_EXACT` restores the old materialized `[L_k, L_q, heads]`
softmax. Flash is bit-faithful to full softmax on CPU (SS-flow 2.4e-4, SLAT
2.9e-4 — identical to exact, so the tap parity above is unaffected) but incurs
~3e-3 rel-L2 on the CUDA F16-MMA kernel. It was already required for the HR
cascade (49,152-token attention fits on 16 GB vs a 108 GiB exact matrix); it
is now the default because on the GPU it is also ~30 % faster per forward and
O(L) memory — the exact path's 805 MB SS-flow score matrix `alloc_graph`-fails
when the resident pipeline leaves little free VRAM. Was previously gated to
score matrices >1 GiB (i.e. only the HR stage). See docs/PLAN.md for the
per-stage runtime profile.
- **TF32 matters.** PyTorch's default CUDA matmul/attention uses TF32 (≈10-bit
mantissa) and reduced-precision flash SDPA, which shows up as ~1e-3 relative
error versus true fp32. `scripts/ref_common.py` disables it so the golden
dumps are real fp32; otherwise a correct port looks like it has a 0.08-rel-L2
bug (this exact trap cost a debugging session — see the flow-forward gate).
- **Sampler drift.** The 12-step Euler + CFG-rescale loop chaotically amplifies
per-step fp differences between backends; it validates tightly on CPU and
drifts to ~0.1 rel-L2 on GPU. The decoder gate therefore decodes the
*reference* SLAT so decoder parity is independent of sampler trajectory.
- **Subdivision boundary.** A handful of level-3 subdivision logits sit within
fp-noise of zero; the `>0` threshold can flip them, so the final active-voxel
set differs by ~4 voxels out of ~4 million (0.0001%) run-to-run and
hardware-to-hardware. This is inherent to a hard threshold, not a port bug.
## GPU
`-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=120` (Blackwell / RTX 50-series). The
flow DiTs and DINOv3 run on CUDA; the 3D-conv decoders (SS decoder CONV_3D and
the sparse-conv gather-GEMM) run on CPU because the bundled ggml has no CUDA
CONV_3D kernel — they are a small fraction of total inference time. GPU f32
matmul is fp16-class, so tap parity on CUDA is ~1e-3 (deterministic, not device
noise); CPU is the tight-tolerance reference backend.
examples/CMakeLists.txt
+34
View File
@@ -0,0 +1,34 @@
# Example: inspect a .dinodata DINOv3 conditioning file.
add_executable(dino_info dino_info.cpp)
target_link_libraries(dino_info PRIVATE trellis2)
target_compile_features(dino_info PRIVATE cxx_std_14)
# Example: inspect a converted SS-flow DiT GGUF (hparams + tensor inventory).
add_executable(ss_flow_info ss_flow_info.cpp)
target_link_libraries(ss_flow_info PRIVATE trellis2)
target_compile_features(ss_flow_info PRIVATE cxx_std_14)
# Example: stage-1 sampling, .dinodata cond -> sparse-structure latent z_s.
add_executable(ss_sample ss_sample.cpp)
target_link_libraries(ss_sample PRIVATE trellis2)
target_compile_features(ss_sample PRIVATE cxx_std_14)
# Example: stage-1 decoding, z_s latent -> occupancy logit grid (64^3).
add_executable(ss_decode ss_decode.cpp)
target_link_libraries(ss_decode PRIVATE trellis2)
target_compile_features(ss_decode PRIVATE cxx_std_14)
# Example: decode + extract the occupancy isosurface as an OBJ (marching cubes).
add_executable(ss_mesh ss_mesh.cpp)
target_link_libraries(ss_mesh PRIVATE trellis2)
target_compile_features(ss_mesh PRIVATE cxx_std_14)
# Example: image -> preprocessed 512x512 -> DINOv3 conditioning (.dinodata).
add_executable(dino_encode dino_encode.cpp)
target_link_libraries(dino_encode PRIVATE trellis2)
target_compile_features(dino_encode PRIVATE cxx_std_14)
# Example: export a demo mesh (T2MESH01/02/03) into a portable GLB (no GPU).
add_executable(mesh2glb mesh2glb.cpp)
target_link_libraries(mesh2glb PRIVATE trellis2)
target_compile_features(mesh2glb PRIVATE cxx_std_14)
examples/dino_encode.cpp
+97
View File
@@ -0,0 +1,97 @@
// Encode an image into the TRELLIS.2 DINOv3 conditioning tensor (.dinodata),
// replacing the external dump_dinodata.py:
//
// image (PNG/JPG; solid black/white backgrounds are removed automatically)
// -> background cleanup -> alpha bbox crop, premultiply, LANCZOS 512
// -> DINOv3 ViT-L/16 -> [1, 1029, 1024] cond -> .dinodata
//
// usage: dino_encode <dino.gguf> <image> [out.dinodata] [--size N] [--pre out.png]
//
#include "trellis2.h"
#define STB_IMAGE_IMPLEMENTATION
#include "stb_image.h"
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "stb_image_write.h"
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>
int main(int argc, char ** argv) {
if (argc < 3) {
std::fprintf(stderr,
"usage: %s <dino.gguf> <image> [out.dinodata] [--size N] [--pre out.png]\n",
argv[0]);
return 2;
}
const std::string gguf_path = argv[1];
const std::string img_path = argv[2];
std::string out_path = "cond.dinodata";
std::string pre_path;
int size = 512;
for (int i = 3; i < argc; ++i) {
if (std::strcmp(argv[i], "--size") == 0 && i + 1 < argc) {
size = std::atoi(argv[++i]);
} else if (std::strcmp(argv[i], "--pre") == 0 && i + 1 < argc) {
pre_path = argv[++i];
} else {
out_path = argv[i];
}
}
int w = 0, h = 0, comp = 0;
unsigned char * pixels = stbi_load(img_path.c_str(), &w, &h, &comp, 4);
if (!pixels) {
std::fprintf(stderr, "failed to decode image %s: %s\n",
img_path.c_str(), stbi_failure_reason());
return 1;
}
const int removed = trellis2_remove_solid_background_rgba(
pixels, w, h, TRELLIS2_BACKGROUND_AUTO);
std::printf("image : %s %dx%d (%d channels, background pixels changed: %d)\n",
img_path.c_str(), w, h, comp, removed);
std::string err;
std::vector<uint8_t> rgb;
if (!trellis2_preprocess_rgba(pixels, w, h, size, rgb, &err)) {
std::fprintf(stderr, "preprocess failed: %s\n", err.c_str());
stbi_image_free(pixels);
return 1;
}
stbi_image_free(pixels);
if (!pre_path.empty()) {
stbi_write_png(pre_path.c_str(), size, size, 3, rgb.data(), size * 3);
std::printf("wrote : %s (preprocessed %dx%d RGB)\n", pre_path.c_str(), size, size);
}
trellis2_dino_model * model = trellis2_dino_load(gguf_path, true, &err);
if (!model) {
std::fprintf(stderr, "model load failed: %s\n", err.c_str());
return 1;
}
std::printf("model : %s (backend %s)\n", gguf_path.c_str(),
trellis2_dino_backend_name(model));
trellis2_dino_cond cond;
if (!trellis2_dino_encode_rgb(model, rgb.data(), size, cond, &err)) {
std::fprintf(stderr, "encode failed: %s\n", err.c_str());
trellis2_dino_free(model);
return 1;
}
trellis2_dino_free(model);
const trellis2_dino_fingerprint fp = trellis2_dino_fingerprints(cond);
std::printf("cond : [1, %lld, %lld] min=%.4f max=%.4f mean=%.6f l2=%.4f\n",
(long long) cond.tokens(), (long long) cond.channels(),
fp.vmin, fp.vmax, fp.mean, fp.l2);
if (!trellis2_save_dinodata(out_path, cond, &err)) {
std::fprintf(stderr, "save failed: %s\n", err.c_str());
return 1;
}
std::printf("wrote : %s (%zu floats)\n", out_path.c_str(), cond.count());
return 0;
}
examples/dino_info.cpp
+54
View File
@@ -0,0 +1,54 @@
// dino_info — load a .dinodata DINOv3 conditioning file and print its shape,
// token breakdown, and fingerprints. The fingerprints should match the values
// in the matching `<stem>.dino.txt` JSON sidecar bit-for-bit.
//
// usage: dino_info <path-to.dinodata>
#include "trellis2.h"
#include <cstdio>
#include <string>
int main(int argc, char ** argv) {
if (argc < 2) {
std::fprintf(stderr, "usage: %s <path-to.dinodata>\n", argv[0]);
return 2;
}
std::printf("trellis2.cpp %s\n", trellis2_version());
const std::string path = argv[1];
trellis2_dino_cond cond;
std::string err;
if (!trellis2_load_dinodata(path, cond, &err)) {
std::fprintf(stderr, "error: %s\n", err.c_str());
return 1;
}
std::printf("file : %s\n", path.c_str());
std::printf("format version : %u\n", cond.format_version);
std::printf("shape : [");
for (size_t i = 0; i < cond.shape.size(); ++i) {
std::printf("%lld%s", (long long) cond.shape[i],
i + 1 < cond.shape.size() ? ", " : "");
}
std::printf("]\n");
const long long tok = (long long) cond.tokens();
std::printf("tokens : %lld (cls:1 + register:4 + patch:%lld)\n",
tok, tok >= 5 ? tok - 5 : 0);
std::printf("channels : %lld\n", (long long) cond.channels());
std::printf("count : %zu floats\n", cond.count());
const trellis2_dino_fingerprint fp = trellis2_dino_fingerprints(cond);
std::printf("fingerprints:\n");
std::printf(" min : %.6f\n", fp.vmin);
std::printf(" max : %.6f\n", fp.vmax);
std::printf(" mean : %.9f\n", fp.mean);
std::printf(" sum : %.6f\n", fp.sum);
std::printf(" l2 : %.6f\n", fp.l2);
std::printf(" count : %zu\n", fp.count);
return 0;
}
examples/flexible_dual_grid.h
+345
View File
@@ -0,0 +1,345 @@
// flexible_dual_grid.h — single-header CPU port of TRELLIS.2's flexible dual
// grid mesh extraction (o-voxel/o_voxel/convert/flexible_dual_grid.py, eval
// path). Turns the shape decoder's per-voxel 7-channel output into a triangle
// mesh, replacing the CUDA hashmap kernel with an std::unordered_map.
//
// Per active voxel v at integer coord c (in [0, grid_size)):
// dual vertex V_v = (c + offset_v) * voxel_size + aabb0 (unit cube here)
// offset_v = (1 + 2*margin) * sigmoid(feat[0:3]) - margin
// intersected feat[3:6] > 0, one flag per axis (x, y, z)
// split_weight softplus(feat[6])
// For each voxel with an intersected axis, the 4 voxels around that edge
// (offsets below) contribute their dual vertices as a quad; if all 4 exist the
// quad is split into 2 triangles along the diagonal chosen by the decoder's
// learned split_weight = softplus(feat[6]) (reference eval-path tie-break).
#pragma once
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <unordered_map>
#include <unordered_set>
#include <vector>
namespace fdg {
struct Mesh {
std::vector<float> verts; // 3 per vertex
std::vector<int> tris; // 3 indices per triangle
size_t n_verts() const { return verts.size() / 3; }
size_t n_tris() const { return tris.size() / 3; }
};
namespace detail {
inline uint64_t key(int32_t x, int32_t y, int32_t z) {
return ((uint64_t) (uint32_t) x << 40) |
((uint64_t) (uint32_t) y << 20) |
(uint64_t) (uint32_t) z;
}
// The 4 neighbor-voxel offsets around an edge, per axis (matches
// edge_neighbor_voxel_offset in the reference).
static const int EDGE_OFF[3][4][3] = {
{{0,0,0},{0,0,1},{0,1,1},{0,1,0}}, // x-axis edge
{{0,0,0},{1,0,0},{1,0,1},{0,0,1}}, // y-axis edge
{{0,0,0},{0,1,0},{1,1,0},{1,0,0}}, // z-axis edge
};
inline void cross(const float * a, const float * b, float * o) {
o[0] = a[1]*b[2] - a[2]*b[1];
o[1] = a[2]*b[0] - a[0]*b[2];
o[2] = a[0]*b[1] - a[1]*b[0];
}
} // namespace detail
// feats: [n_voxels * 7], voxel-major (dec output). coords: [n_voxels * 3].
// grid_size = input_res * decoder upscale (e.g. 32 * 16 = 512). margin 0.5.
inline Mesh extract(const float * feats, const int32_t * coords, int n,
int grid_size, float margin = 0.5f) {
using namespace detail;
Mesh m;
if (n <= 0) return m;
const float vs = 1.0f / (float) grid_size; // voxel size (aabb span 1)
const float aabb0 = -0.5f;
// dual vertices + hashmap
std::vector<float> V((size_t) n * 3);
std::unordered_map<uint64_t, int> idx;
idx.reserve((size_t) n * 2);
for (int v = 0; v < n; ++v) {
const float * f = feats + (size_t) v * 7;
for (int a = 0; a < 3; ++a) {
const float s = 1.0f / (1.0f + std::exp(-f[a])); // sigmoid
const float off = (1.0f + 2.0f * margin) * s - margin;
V[(size_t) v * 3 + a] = ((float) coords[(size_t) v * 3 + a] + off) * vs + aabb0;
}
idx[key(coords[(size_t) v * 3], coords[(size_t) v * 3 + 1], coords[(size_t) v * 3 + 2])] = v;
}
m.verts = V;
// quads from intersected edges
for (int v = 0; v < n; ++v) {
const float * f = feats + (size_t) v * 7;
const int32_t cx = coords[(size_t) v * 3];
const int32_t cy = coords[(size_t) v * 3 + 1];
const int32_t cz = coords[(size_t) v * 3 + 2];
for (int axis = 0; axis < 3; ++axis) {
if (f[3 + axis] <= 0.0f) continue; // not intersected on this axis
int q[4];
bool ok = true;
for (int i = 0; i < 4; ++i) {
const int32_t nx = cx + EDGE_OFF[axis][i][0];
const int32_t ny = cy + EDGE_OFF[axis][i][1];
const int32_t nz = cz + EDGE_OFF[axis][i][2];
auto it = idx.find(key(nx, ny, nz));
if (it == idx.end()) { ok = false; break; }
q[i] = it->second;
}
if (!ok) continue;
// Choose the quad diagonal by the decoder's learned split_weight
// (softplus of feat[6]), exactly as the reference eval path does
// (FlexiDualGridVaeDecoder -> flexible_dual_grid_to_mesh, train=False):
// split 1: (0,1,2)+(0,2,3) when sw0*sw2 > sw1*sw3
// split 2: (0,1,3)+(3,1,2) otherwise
// (A geometric best-aligned-normals heuristic is the reference's
// split_weight=None fallback; the shipped decoder always emits
// feat[6], so we follow the learned choice.)
auto sw = [&](int i) {
const float x = feats[(size_t) q[i] * 7 + 6];
return x > 20.0f ? x : std::log1p(std::exp(x)); // softplus
};
if (sw(0) * sw(2) > sw(1) * sw(3)) {
m.tris.push_back(q[0]); m.tris.push_back(q[1]); m.tris.push_back(q[2]);
m.tris.push_back(q[0]); m.tris.push_back(q[2]); m.tris.push_back(q[3]);
} else {
m.tris.push_back(q[0]); m.tris.push_back(q[1]); m.tris.push_back(q[3]);
m.tris.push_back(q[3]); m.tris.push_back(q[1]); m.tris.push_back(q[2]);
}
}
}
return m;
}
// Per-vertex shading normals for the dual grid's *unoriented*, heavily
// non-manifold mesh.
//
// The reference mesher emits every quad with a fixed vertex order regardless of
// which way the surface crosses the edge, so a large fraction of faces are wound
// opposite to their neighbours. Two failure modes follow: (a) a naive
// area-weighted normal cancels at those seams, and (b) any *sign* fix that leaves
// stray flips is not harmless — the normal is interpolated across the triangle
// *before* the fragment shader's abs(dot), so two adjacent vertices with opposite
// normals make the interpolated normal cross zero mid-face → normalize() explodes
// → speckled/blocky specular. The mesh is too non-manifold to 2-colour the
// winding cleanly (edges shared by >2 faces frustrate it), so we don't try to.
//
// 1. Recover a smooth, winding-INDEPENDENT normal *direction* per vertex as the
// dominant eigenvector of the area-weighted structure tensor Σ area·n̂n̂ᵀ
// (immune to winding sign since n̂n̂ᵀ == (−n̂)(−n̂)ᵀ).
// 2. Resolve the arbitrary per-vertex *sign* consistently with a parity
// union-find over mesh edges, so edge-adjacent vertices share a hemisphere
// and the interpolated normal stays clear of zero. Only genuinely frustrated
// (odd-cycle / non-manifold) edges are left flipped.
// Final shading is orientation-independent (viewer uses abs(dot)), so only local
// smoothness matters, not a globally correct outward sign. (Winding unification
// and sign diffusion were both tried and are worse on this mesh: 2-colouring the
// >2-face non-manifold edges frustrates more, and Jacobi diffusion checkerboards.)
inline std::vector<float> vertex_normals(const Mesh & m) {
const size_t nv = m.n_verts();
std::vector<double> A((size_t) nv * 6, 0.0); // sym structure tensor per vertex
std::vector<float> seed((size_t) nv * 3, 0.0f); // signed area sum: a sign hint
for (size_t t = 0; t < m.tris.size(); t += 3) {
const int i0 = m.tris[t], i1 = m.tris[t + 1], i2 = m.tris[t + 2];
const float * a = &m.verts[(size_t) i0 * 3];
const float * b = &m.verts[(size_t) i1 * 3];
const float * c = &m.verts[(size_t) i2 * 3];
float e1[3], e2[3], fn[3];
for (int k = 0; k < 3; ++k) { e1[k] = b[k]-a[k]; e2[k] = c[k]-a[k]; }
detail::cross(e1, e2, fn);
const double area = std::sqrt((double) fn[0]*fn[0] + (double) fn[1]*fn[1] + (double) fn[2]*fn[2]);
if (area <= 1e-20) continue;
const double inv = 1.0 / area;
const double xx = fn[0]*fn[0]*inv, yy = fn[1]*fn[1]*inv, zz = fn[2]*fn[2]*inv;
const double xy = fn[0]*fn[1]*inv, xz = fn[0]*fn[2]*inv, yz = fn[1]*fn[2]*inv;
for (int i : {i0, i1, i2}) {
double * Av = &A[(size_t) i * 6];
Av[0]+=xx; Av[1]+=yy; Av[2]+=zz; Av[3]+=xy; Av[4]+=xz; Av[5]+=yz;
float * sv = &seed[(size_t) i * 3];
sv[0]+=fn[0]; sv[1]+=fn[1]; sv[2]+=fn[2];
}
}
// (1) smooth, sign-ambiguous direction per vertex
std::vector<float> dir((size_t) nv * 3, 0.0f);
for (size_t v = 0; v < nv; ++v) {
const double * Av = &A[v * 6];
double x = seed[v*3], y = seed[v*3+1], z = seed[v*3+2];
double l = std::sqrt(x*x + y*y + z*z);
if (l < 1e-20) { x = Av[0]; y = Av[3]; z = Av[4]; l = std::sqrt(x*x+y*y+z*z); }
if (l < 1e-20) { dir[v*3+2] = 1.0f; continue; }
x/=l; y/=l; z/=l;
for (int it = 0; it < 8; ++it) {
const double nx = Av[0]*x + Av[3]*y + Av[4]*z;
const double ny = Av[3]*x + Av[1]*y + Av[5]*z;
const double nz = Av[4]*x + Av[5]*y + Av[2]*z;
const double nl = std::sqrt(nx*nx + ny*ny + nz*nz);
if (nl < 1e-20) break;
x = nx/nl; y = ny/nl; z = nz/nl;
}
dir[v*3] = (float) x; dir[v*3+1] = (float) y; dir[v*3+2] = (float) z;
}
// (2) Resolve the arbitrary per-vertex sign *consistently* via a parity
// union-find over mesh edges: two edge-adjacent vertices whose directions
// are anti-aligned must end up with opposite signs (and vice versa), so
// within each connected component neighbours share a hemisphere. Only
// genuinely frustrated (odd-cycle / non-manifold) edges are left flipped.
std::vector<int> ufp(nv), ufr(nv, 0);
std::vector<uint8_t> ufb(nv, 0); // parity of a vertex relative to its parent
for (size_t i = 0; i < nv; ++i) ufp[i] = (int) i;
auto find = [&](int v, int & parity) {
int p = 0;
while (ufp[v] != v) { p ^= ufb[v]; v = ufp[v]; }
parity = p; return v;
};
auto join = [&](int a, int b) {
const float * na = &dir[(size_t) a * 3];
const float * nb = &dir[(size_t) b * 3];
const int rel = (na[0]*nb[0] + na[1]*nb[1] + na[2]*nb[2]) < 0.0f ? 1 : 0;
int pa, pb, ra = find(a, pa), rb = find(b, pb);
if (ra == rb) return;
if (ufr[ra] < ufr[rb]) { std::swap(ra, rb); std::swap(pa, pb); }
ufp[rb] = ra; ufb[rb] = (uint8_t) (pa ^ pb ^ rel);
if (ufr[ra] == ufr[rb]) ufr[ra]++;
};
for (size_t t = 0; t < m.tris.size(); t += 3) {
const int a = m.tris[t], b = m.tris[t+1], c = m.tris[t+2];
join(a, b); join(b, c); join(c, a);
}
// (3) pick each component's global sign toward the signed-sum seed (so the
// result is deterministic and roughly outward), then emit oriented normals
std::unordered_map<int, double> comp_sign;
for (size_t v = 0; v < nv; ++v) {
int p, r = find((int) v, p);
const double s = p ? -1.0 : 1.0;
const float * sv = &seed[v*3];
comp_sign[r] += s * (dir[v*3]*sv[0] + dir[v*3+1]*sv[1] + dir[v*3+2]*sv[2]);
}
std::vector<float> nrm((size_t) nv * 3, 0.0f);
for (size_t v = 0; v < nv; ++v) {
int p, r = find((int) v, p);
double s = p ? -1.0 : 1.0;
if (comp_sign[r] < 0.0) s = -s;
nrm[v*3] = (float) (s * dir[v*3]);
nrm[v*3+1] = (float) (s * dir[v*3+1]);
nrm[v*3+2] = (float) (s * dir[v*3+2]);
}
return nrm;
}
// Remove triangles that belong to tiny disconnected islands (the floating
// specks that read as "blemishes"), keeping the vertex array and its indexing
// intact so a parallel per-vertex attribute array (e.g. baked PBR) stays
// aligned. Components are face groups connected through shared vertices; a
// component is dropped when its face count is below min_frac of the total.
inline void drop_small_components(Mesh & m, float min_frac = 0.0005f) {
const size_t nt = m.n_tris();
if (nt == 0) return;
const size_t nv = m.n_verts();
std::vector<int> p(nv);
for (size_t i = 0; i < nv; ++i) p[i] = (int) i;
auto find = [&](int x) { while (p[x] != x) { p[x] = p[p[x]]; x = p[x]; } return x; };
auto uni = [&](int a, int b) { a = find(a); b = find(b); if (a != b) p[a] = b; };
for (size_t t = 0; t < m.tris.size(); t += 3) {
uni(m.tris[t], m.tris[t+1]); uni(m.tris[t+1], m.tris[t+2]);
}
std::unordered_map<int, int> faces;
for (size_t t = 0; t < m.tris.size(); t += 3) faces[find(m.tris[t])]++;
const int min_faces = std::max(1, (int) (min_frac * (double) nt));
std::vector<int> keep;
keep.reserve(m.tris.size());
for (size_t t = 0; t < m.tris.size(); t += 3) {
if (faces[find(m.tris[t])] >= min_faces) {
keep.push_back(m.tris[t]); keep.push_back(m.tris[t+1]); keep.push_back(m.tris[t+2]);
}
}
m.tris.swap(keep);
}
// Fill the small holes extract() leaves where a dual-grid quad was skipped (a
// neighbour voxel was missing). A boundary edge is one used by exactly one
// triangle; boundary edges chain into loops around each hole. Each loop up to
// `max_loop` vertices is fan-triangulated from its first vertex. Only triangles
// are added — no new vertices — so a parallel per-vertex attribute array (baked
// PBR) stays aligned. Large loops (genuine openings) are left unfilled.
inline void fill_holes(Mesh & m, int max_loop = 64, int max_passes = 4) {
if (m.n_tris() == 0) return;
auto key = [](int a, int b) {
const uint32_t lo = a < b ? a : b, hi = a < b ? b : a;
return ((uint64_t) lo << 32) | hi;
};
// Iterate: the greedy loop walk misses some loops at non-manifold junctions,
// and each fill can expose newly closeable loops, so repeat until a pass
// adds nothing (or the pass cap is hit).
for (int pass = 0; pass < max_passes; ++pass) {
const size_t before = m.tris.size();
// run-length count of undirected edges via sort (lighter than a hashmap)
std::vector<uint64_t> ek;
ek.reserve(m.tris.size());
for (size_t t = 0; t < m.tris.size(); t += 3) {
ek.push_back(key(m.tris[t], m.tris[t+1]));
ek.push_back(key(m.tris[t+1], m.tris[t+2]));
ek.push_back(key(m.tris[t+2], m.tris[t]));
}
std::sort(ek.begin(), ek.end());
std::unordered_map<int, std::vector<int>> adj; // boundary-vertex adjacency
for (size_t i = 0; i < ek.size(); ) {
size_t j = i + 1;
while (j < ek.size() && ek[j] == ek[i]) ++j;
if (j - i == 1) { // used by exactly one triangle -> boundary edge
const int a = (int) (ek[i] >> 32), b = (int) (ek[i] & 0xffffffffu);
adj[a].push_back(b); adj[b].push_back(a);
}
i = j;
}
if (adj.empty()) break;
std::unordered_set<uint64_t> used; // consumed boundary edges
for (const auto & kv : adj) {
const int start = kv.first;
for (const int nb0 : kv.second) {
if (used.count(key(start, nb0))) continue;
std::vector<int> loop{start};
int prev = start, cur = nb0;
used.insert(key(prev, cur));
bool closed = false;
while ((int) loop.size() <= max_loop) {
loop.push_back(cur);
if (cur == start) { closed = true; break; }
int next = -1;
auto it = adj.find(cur);
if (it != adj.end())
for (const int c : it->second)
if (c != prev && !used.count(key(cur, c))) { next = c; break; }
if (next < 0) break; // open chain / dead end
used.insert(key(cur, next));
prev = cur; cur = next;
}
if (!closed) continue;
loop.pop_back(); // drop the repeated start vertex
const int k = (int) loop.size();
if (k < 3 || k > max_loop) continue;
for (int t = 1; t < k - 1; ++t) { // fan-triangulate from loop[0]
m.tris.push_back(loop[0]);
m.tris.push_back(loop[t]);
m.tris.push_back(loop[t + 1]);
}
}
}
if (m.tris.size() == before) break; // converged
}
}
} // namespace fdg
examples/marching_cubes.h
+243
View File
@@ -0,0 +1,243 @@
// marching_cubes.h — single-file, self-contained isosurface extraction.
//
// Extracts a triangle mesh for the level set {f = iso} of a scalar field
// sampled on a regular grid, and writes it as a Wavefront OBJ.
//
// It uses the TETRAHEDRAL variant of marching cubes (a.k.a. marching
// tetrahedra on the Freudenthal/Kuhn subdivision): each grid cube is split
// into 6 tetrahedra that all share the (0,0,0)-(1,1,1) main diagonal, and each
// tetrahedron is contoured with a tiny 16-case table. Compared to classic
// marching cubes this avoids the error-prone 256-row triangle table entirely,
// and — because every cube uses the same diagonal — the subdivision tiles space
// consistently, so the output is a watertight 2-manifold (each interior edge is
// shared by exactly two triangles). Vertices are de-duplicated by the global
// grid edge they sit on, so the mesh is properly indexed.
//
// Header-only and dependency-free (just <vector>/<cmath>/<unordered_map>).
//
// mc::Mesh m = mc::extract(field, nx, ny, nz, iso);
// mc::write_obj(m, "out.obj");
//
// `field` is indexed x + y*nx + z*nx*ny (x fastest). Vertex positions are in
// grid-index units (a voxel is one unit). The field is treated as `pad` (a
// large negative value, i.e. "outside") beyond the grid, so surfaces that reach
// the boundary are closed off.
#pragma once
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <unordered_map>
#include <vector>
namespace mc {
struct Mesh {
std::vector<float> verts; // 3 floats per vertex (x,y,z)
std::vector<float> normals; // 3 floats per vertex (unit, oriented outward)
std::vector<int> tris; // 3 vertex indices per triangle
size_t n_verts() const { return verts.size() / 3; }
size_t n_tris() const { return tris.size() / 3; }
};
namespace detail {
// Cube corner offsets, Bourke/Lorensen order.
static const int CORNER[8][3] = {
{0,0,0},{1,0,0},{1,1,0},{0,1,0},{0,0,1},{1,0,1},{1,1,1},{0,1,1}
};
// The 6 tetrahedra of the Freudenthal subdivision, all sharing the 0-6 diagonal.
static const int TETRA[6][4] = {
{0,1,2,6},{0,2,3,6},{0,3,7,6},{0,7,4,6},{0,4,5,6},{0,5,1,6}
};
// A tetrahedron's 6 edges, as (local-corner, local-corner) pairs.
static const int TET_EDGE[6][2] = {
{0,1},{1,2},{2,0},{0,3},{1,3},{2,3}
};
// Per-case triangle table for one tetrahedron. Index = bitmask of which of the
// 4 local corners are "inside" (value > iso); values are TET_EDGE indices,
// terminated by -1. Derived by hand (see marching_cubes.h history) and checked
// at runtime by extract()'s self-consistency asserts on small fields.
static const int TET_TRI[16][7] = {
{-1,-1,-1,-1,-1,-1,-1}, // 0000
{ 0, 2, 3,-1,-1,-1,-1}, // 0001 {0}
{ 0, 1, 4,-1,-1,-1,-1}, // 0010 {1}
{ 2, 1, 4, 2, 4, 3,-1}, // 0011 {0,1}
{ 2, 1, 5,-1,-1,-1,-1}, // 0100 {2}
{ 0, 1, 5, 0, 5, 3,-1}, // 0101 {0,2}
{ 0, 2, 5, 0, 5, 4,-1}, // 0110 {1,2}
{ 3, 4, 5,-1,-1,-1,-1}, // 0111 {0,1,2} (outside 3)
{ 3, 4, 5,-1,-1,-1,-1}, // 1000 {3}
{ 0, 4, 5, 0, 5, 2,-1}, // 1001 {0,3}
{ 0, 3, 5, 0, 5, 1,-1}, // 1010 {1,3}
{ 2, 1, 5,-1,-1,-1,-1}, // 1011 {0,1,3} (outside 2)
{ 2, 3, 4, 2, 4, 1,-1}, // 1100 {2,3}
{ 0, 1, 4,-1,-1,-1,-1}, // 1101 {0,2,3} (outside 1)
{ 0, 2, 3,-1,-1,-1,-1}, // 1110 {1,2,3} (outside 0)
{-1,-1,-1,-1,-1,-1,-1}, // 1111
};
} // namespace detail
inline Mesh extract(const float * field, int nx, int ny, int nz, float iso,
float pad = -1e30f) {
using namespace detail;
Mesh mesh;
// Tie-breaking jitter (Simulation-of-Simplicity style): a corner whose value
// is *exactly* iso makes the surface pass through that single point, which is
// shared by every incident tetrahedron — a pinch that no winding can make
// coherent. A tiny deterministic per-corner perturbation (consistent across
// cubes, so shared corners still agree -> mesh stays watertight) pushes every
// corner decisively on/off the surface. Scaled to the field so vertex motion
// is sub-micro-voxel; continuous fields (e.g. decoder logits) are unaffected.
double fmax = 0.0;
for (size_t i = 0; i < (size_t) nx * ny * nz; ++i) {
double a = std::fabs((double) field[i]); if (a > fmax) fmax = a;
}
const float eps = (float) (1e-6 * fmax) + 1e-20f;
auto jitter = [](int x, int y, int z) -> float {
uint32_t h = (uint32_t)(x * 73856093) ^ (uint32_t)(y * 19349663) ^ (uint32_t)(z * 83492791);
h ^= h >> 13; h *= 0x5bd1e995u; h ^= h >> 15;
return (float)(h & 0xffff) / 65535.0f - 0.5f; // [-0.5, 0.5)
};
auto at = [&](int x, int y, int z) -> float {
if (x < 0 || y < 0 || z < 0 || x >= nx || y >= ny || z >= nz) return pad;
return field[(size_t) x + (size_t) y * nx + (size_t) z * nx * ny] + eps * jitter(x, y, z);
};
// Central-difference gradient (for outward-oriented vertex normals).
auto grad = [&](int x, int y, int z, float g[3]) {
g[0] = at(x + 1, y, z) - at(x - 1, y, z);
g[1] = at(x, y + 1, z) - at(x, y - 1, z);
g[2] = at(x, y, z + 1) - at(x, y, z - 1);
};
std::unordered_map<int64_t, int> vmap; // quantized-position key -> vertex index
// Interpolate (and cache) the vertex on the grid edge cA->cB. Vertices are
// de-duplicated by QUANTIZED POSITION, not by edge: against the large pad,
// boundary-cap vertices interpolate to t~0 and collapse exactly onto grid
// corners, so several distinct edges land on the same point. Position keying
// welds those into one vertex (killing the degenerate zero-area cap triangles
// and keeping the boundary watertight), while shared edges across cubes —
// which already yield identical positions — still merge as before.
const double QS = 1024.0; // sub-milli-voxel quantization
auto edge_vertex = [&](const int ca[3], const int cb[3], float va, float vb) -> int {
float t = (vb != va) ? (iso - va) / (vb - va) : 0.5f;
if (t < 0.0f) t = 0.0f; else if (t > 1.0f) t = 1.0f;
float p[3] = { ca[0] + t * (cb[0] - ca[0]),
ca[1] + t * (cb[1] - ca[1]),
ca[2] + t * (cb[2] - ca[2]) };
int64_t qx = (int64_t) std::llround(p[0] * QS) + 0x200000;
int64_t qy = (int64_t) std::llround(p[1] * QS) + 0x200000;
int64_t qz = (int64_t) std::llround(p[2] * QS) + 0x200000;
int64_t key = (qx * 0x400000LL + qy) * 0x400000LL + qz;
auto it = vmap.find(key);
if (it != vmap.end()) return it->second;
float ga[3], gb[3];
grad(ca[0], ca[1], ca[2], ga);
grad(cb[0], cb[1], cb[2], gb);
float n[3];
for (int d = 0; d < 3; ++d) n[d] = -(ga[d] + t * (gb[d] - ga[d])); // outward = -grad
// Normalize in double: against the large negative pad the gradient can
// reach ~1e30, and 1e30^2 overflows float32 to inf (-> bogus normal).
double len = std::sqrt((double)n[0]*n[0] + (double)n[1]*n[1] + (double)n[2]*n[2]);
if (len > 1e-12) { n[0]=(float)(n[0]/len); n[1]=(float)(n[1]/len); n[2]=(float)(n[2]/len); }
else { n[0]=0; n[1]=0; n[2]=1; }
int idx = (int) mesh.n_verts();
mesh.verts.insert(mesh.verts.end(), { p[0], p[1], p[2] });
mesh.normals.insert(mesh.normals.end(), { n[0], n[1], n[2] });
vmap.emplace(key, idx);
return idx;
};
// Iterate cube origins over [-1 .. n-1] so boundary cubes (against the pad)
// close the surface.
for (int z = -1; z < nz; ++z)
for (int y = -1; y < ny; ++y)
for (int x = -1; x < nx; ++x) {
float cval[8];
int ccoord[8][3];
for (int c = 0; c < 8; ++c) {
ccoord[c][0] = x + CORNER[c][0];
ccoord[c][1] = y + CORNER[c][1];
ccoord[c][2] = z + CORNER[c][2];
cval[c] = at(ccoord[c][0], ccoord[c][1], ccoord[c][2]);
}
for (int t = 0; t < 6; ++t) {
const int * tc = TETRA[t];
int code = 0;
for (int i = 0; i < 4; ++i) if (cval[tc[i]] > iso) code |= (1 << i);
const int * tri = TET_TRI[code];
if (tri[0] < 0) continue;
// Gather this tetra's triangles (1 or 2), then decide their winding
// ONCE, together: flip so the summed geometric normal (snorm) agrees
// with the summed outward vertex normal (gsum = sum of -grad over the
// triangle vertices). gsum is PARALLEL to the surface normal, so the
// dot is never near zero — unlike a centroid-difference proxy, which
// can be in-plane at boundary/crease tetra and pick a random sign.
// The surface inside a tetra is planar, so a quad's two triangles are
// coplanar and flip as a unit (a per-triangle test would mis-flip a
// sliver triangle, whose own geometric normal is ~0).
int tribuf[2][3]; int ntri = 0;
double snorm[3] = {0, 0, 0}, gsum[3] = {0, 0, 0};
for (int e = 0; tri[e] >= 0; e += 3) {
int vi[3];
for (int k = 0; k < 3; ++k) {
const int * ed = TET_EDGE[tri[e + k]];
int la = tc[ed[0]], lb = tc[ed[1]];
vi[k] = edge_vertex(ccoord[la], ccoord[lb], cval[la], cval[lb]);
}
if (vi[0] == vi[1] || vi[1] == vi[2] || vi[0] == vi[2]) continue; // degenerate
const float * P0 = &mesh.verts[3*vi[0]];
const float * P1 = &mesh.verts[3*vi[1]];
const float * P2 = &mesh.verts[3*vi[2]];
float u[3] = { P1[0]-P0[0], P1[1]-P0[1], P1[2]-P0[2] };
float v[3] = { P2[0]-P0[0], P2[1]-P0[1], P2[2]-P0[2] };
snorm[0] += u[1]*v[2]-u[2]*v[1];
snorm[1] += u[2]*v[0]-u[0]*v[2];
snorm[2] += u[0]*v[1]-u[1]*v[0];
for (int k = 0; k < 3; ++k) {
gsum[0] += mesh.normals[3*vi[k]];
gsum[1] += mesh.normals[3*vi[k]+1];
gsum[2] += mesh.normals[3*vi[k]+2];
}
tribuf[ntri][0] = vi[0]; tribuf[ntri][1] = vi[1]; tribuf[ntri][2] = vi[2];
++ntri;
}
bool flip = (snorm[0]*gsum[0] + snorm[1]*gsum[1] + snorm[2]*gsum[2]) < 0.0;
for (int a = 0; a < ntri; ++a) {
int v0 = tribuf[a][0], v1 = tribuf[a][1], v2 = tribuf[a][2];
if (flip) { int tmp = v1; v1 = v2; v2 = tmp; }
mesh.tris.insert(mesh.tris.end(), { v0, v1, v2 });
}
}
}
return mesh;
}
inline bool write_obj(const Mesh & m, const char * path) {
FILE * f = std::fopen(path, "wb");
if (!f) return false;
std::fprintf(f, "# trellis2.cpp isosurface (%zu verts, %zu tris)\n",
m.n_verts(), m.n_tris());
for (size_t i = 0; i < m.n_verts(); ++i)
std::fprintf(f, "v %.6f %.6f %.6f\n", m.verts[3*i], m.verts[3*i+1], m.verts[3*i+2]);
for (size_t i = 0; i < m.n_verts(); ++i)
std::fprintf(f, "vn %.6f %.6f %.6f\n", m.normals[3*i], m.normals[3*i+1], m.normals[3*i+2]);
for (size_t i = 0; i < m.n_tris(); ++i) {
int a = m.tris[3*i] + 1, b = m.tris[3*i+1] + 1, c = m.tris[3*i+2] + 1;
std::fprintf(f, "f %d//%d %d//%d %d//%d\n", a, a, b, b, c, c);
}
std::fclose(f);
return true;
}
} // namespace mc
examples/mesh2glb.cpp
+130
View File
@@ -0,0 +1,130 @@
// mesh2glb — export a demo mesh (T2MESH01/T2MESH02/T2MESH03 wire format) to a
// portable vertex-coloured GLB. Exercises the CUDA-free component cleanup and
// glTF path offline, with no models or GPU. T2GLB_XATLAS opts into image baking.
//
// mesh2glb in.bin out.glb [texture_size] [--print [alpha_pct offset_pct]]
//
// The wire format is what the demo server emits at /api/mesh/{id}:
// magic[8] u32 nv u32 nt f32[3nv] verts f32[3nv] normals
// [T2MESH02: f32[5nv] legacy pbr]
// [T2MESH03: f32[6nv] pbr incl. alpha] i32[3nt] tris (little-endian)
#include "mesh_export.h"
#include <cstdint>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <vector>
template <class T>
static bool rd(FILE * f, std::vector<T> & v, size_t n) {
v.resize(n);
return n == 0 || std::fread(v.data(), sizeof(T), n, f) == n;
}
int main(int argc, char ** argv) {
if (argc < 3) {
std::fprintf(stderr,
"usage: %s in.bin out.glb [texture_size] [--print [alpha_pct offset_pct]]\n",
argv[0]);
return 2;
}
t2glb::MeshExportOptions opt;
opt.components = t2glb::ComponentFilter::KeepAll;
bool print_wrap = false;
float alpha_ratio = 0.01f, offset_ratio = 0.01f / 30.0f;
for (int i = 3; i < argc; ++i) {
if (std::strcmp(argv[i], "--print") == 0) {
print_wrap = true;
if (i + 1 < argc && argv[i+1][0] != '-') alpha_ratio = std::atof(argv[++i]) / 100.0f;
if (i + 1 < argc && argv[i+1][0] != '-') offset_ratio = std::atof(argv[++i]) / 100.0f;
} else {
opt.texture_size = std::atoi(argv[i]);
}
}
if (print_wrap && !t2glb::print_remesh_available()) {
std::fprintf(stderr, "print remeshing unavailable: rebuild with CGAL 5.5 or newer\n");
return 1;
}
FILE * f = std::fopen(argv[1], "rb");
if (!f) { std::fprintf(stderr, "open %s failed\n", argv[1]); return 1; }
char magic[9] = {0};
uint32_t nv = 0, nt = 0;
if (std::fread(magic, 1, 8, f) != 8 || std::fread(&nv, 4, 1, f) != 1 || std::fread(&nt, 4, 1, f) != 1) {
std::fprintf(stderr, "bad header\n"); return 1;
}
const bool legacy = std::memcmp(magic, "T2MESH02", 8) == 0;
const bool textured = legacy || std::memcmp(magic, "T2MESH03", 8) == 0;
if (!textured && std::memcmp(magic, "T2MESH01", 8) != 0) {
std::fprintf(stderr, "unknown magic\n"); return 1;
}
std::vector<float> verts, normals, pbr; std::vector<int32_t> tris;
bool ok = rd(f, verts, (size_t) nv * 3) && rd(f, normals, (size_t) nv * 3);
if (ok && textured) {
if (legacy) {
std::vector<float> old;
ok = rd(f, old, (size_t) nv * 5);
if (ok) {
pbr.resize((size_t) nv * 6);
for (uint32_t i = 0; i < nv; ++i) {
std::memcpy(pbr.data() + (size_t) i * 6,
old.data() + (size_t) i * 5, 5 * sizeof(float));
pbr[(size_t) i * 6 + 5] = 1.0f;
}
}
} else {
ok = rd(f, pbr, (size_t) nv * 6);
}
}
ok = ok && rd(f, tris, (size_t) nt * 3);
std::fclose(f);
if (!ok) { std::fprintf(stderr, "truncated mesh\n"); return 1; }
std::fprintf(stderr, "in: %s %u verts %u tris %s\n", magic, nv, nt,
textured ? "textured" : "geometry-only");
std::fprintf(stderr, "export: %s ...\n",
print_wrap ? "constructing watertight CGAL Alpha Wrap" : "preserving input topology");
std::vector<uint8_t> glb; std::string err;
t2glb::PreparedMesh wrapped;
const float * export_verts = verts.data();
const int32_t * export_tris = tris.data();
const float * export_pbr = textured ? pbr.data() : nullptr;
int export_nv = (int) nv, export_nt = (int) nt;
if (print_wrap) {
if (!t2glb::prepare_print_mesh(verts.data(), (int) nv, tris.data(), (int) nt,
export_pbr, opt, alpha_ratio, offset_ratio,
wrapped, err)) {
std::fprintf(stderr, "prepare_print_mesh: %s\n", err.c_str());
return 1;
}
export_verts = wrapped.verts.data(); export_nv = (int) wrapped.verts.size() / 3;
export_tris = wrapped.tris.data(); export_nt = (int) wrapped.tris.size() / 3;
export_pbr = nullptr;
opt.components = t2glb::ComponentFilter::KeepAll;
std::fprintf(stderr, "wrap: %d verts %d tris %s\n", export_nv, export_nt,
textured ? "rebaking source PBR atlas" : "geometry-only");
}
const bool projected = print_wrap && textured;
const bool baked = projected
? t2glb::mesh_to_projected_glb(
export_verts, export_nv, export_tris, export_nt,
verts.data(), (int) nv, tris.data(), (int) nt, pbr.data(),
opt, glb, err)
: t2glb::mesh_to_glb(export_verts, export_nv, export_tris, export_nt,
export_pbr, opt, glb, err);
if (!baked) {
std::fprintf(stderr, "%s: %s\n",
projected ? "mesh_to_projected_glb" : "mesh_to_glb", err.c_str());
return 1;
}
FILE * o = std::fopen(argv[2], "wb");
if (!o) { std::fprintf(stderr, "open %s failed\n", argv[2]); return 1; }
std::fwrite(glb.data(), 1, glb.size(), o);
std::fclose(o);
std::fprintf(stderr, "wrote %s (%.2f MB)\n", argv[2], glb.size() / 1048576.0);
return 0;
}
examples/ss_decode.cpp
+75
View File
@@ -0,0 +1,75 @@
// ss_decode — run the stage-1 SS decoder: a sparse-structure latent z_s
// (.latent from ss_sample) + the SS decoder GGUF -> an occupancy logit grid.
// The coarse voxel scaffold is logit > 0.
//
// usage: ss_decode <ss_dec.gguf> <z_s.latent> [out.occ]
//
// Writes the occupancy logit grid (channel-major [out_channels * R_out^3]
// float32) to out.occ if given. Also prints occupancy stats and, if all output
// channels are 1, the count of occupied voxels.
#include "trellis2.h"
#include <cstdio>
#include <cstring>
#include <fstream>
#include <string>
#include <vector>
int main(int argc, char ** argv) {
if (argc < 3) {
std::fprintf(stderr, "usage: %s <ss_dec.gguf> <z_s.latent> [out.occ]\n", argv[0]);
return 2;
}
const std::string gguf_path = argv[1];
const std::string lat_path = argv[2];
const std::string out_path = argc > 3 ? argv[3] : "";
std::printf("trellis2.cpp %s\n", trellis2_version());
std::string err;
trellis2_ss_dec_model * m = trellis2_ss_dec_load(gguf_path, true, &err);
if (!m) { std::fprintf(stderr, "model load error: %s\n", err.c_str()); return 1; }
std::printf("backend: %s\n", trellis2_ss_dec_backend_name(m));
const trellis2_ss_dec_hparams hp = trellis2_ss_dec_hparams_of(m); // copy (used after free)
const int Rin = hp.res_in();
const int Rout = hp.res_out();
const size_t n_in = (size_t) hp.latent_channels * Rin * Rin * Rin;
const size_t n_out = (size_t) hp.out_channels * Rout * Rout * Rout;
// load latent
std::ifstream f(lat_path, std::ios::binary);
if (!f) { std::fprintf(stderr, "cannot open latent %s\n", lat_path.c_str()); trellis2_ss_dec_free(m); return 1; }
std::vector<float> latent(n_in);
f.read(reinterpret_cast<char *>(latent.data()), (std::streamsize) (n_in * sizeof(float)));
if ((size_t) f.gcount() != n_in * sizeof(float)) {
std::fprintf(stderr, "latent size mismatch: got %zu floats, want %zu\n",
(size_t) f.gcount() / sizeof(float), n_in);
trellis2_ss_dec_free(m);
return 1;
}
std::printf("latent : [%d,%d,%d,%d]\n", hp.latent_channels, Rin, Rin, Rin);
std::vector<float> occ(n_out, 0.0f);
if (!trellis2_ss_dec_decode(m, latent.data(), occ.data(), &err)) {
std::fprintf(stderr, "decode error: %s\n", err.c_str());
trellis2_ss_dec_free(m);
return 1;
}
trellis2_ss_dec_free(m);
double mn = 1e30, mx = -1e30, sum = 0.0;
size_t occupied = 0;
for (float v : occ) { mn = v < mn ? v : mn; mx = v > mx ? v : mx; sum += v; if (v > 0.0f) ++occupied; }
std::printf("logits : [%d,%d,%d,%d] min=%.4f max=%.4f mean=%.5f occupied(>0)=%zu/%zu (%.2f%%)\n",
hp.out_channels, Rout, Rout, Rout, mn, mx, sum / (double) n_out,
occupied, n_out, 100.0 * (double) occupied / (double) n_out);
if (!out_path.empty()) {
std::ofstream o(out_path, std::ios::binary);
o.write(reinterpret_cast<const char *>(occ.data()), (std::streamsize) (n_out * sizeof(float)));
std::printf("wrote %s (%zu floats)\n", out_path.c_str(), n_out);
}
return 0;
}
examples/ss_flow_info.cpp
+94
View File
@@ -0,0 +1,94 @@
// ss_flow_info — load a converted SS-flow DiT GGUF and print its hyperparameters
// and tensor inventory. Validates that convert_ss_flow_to_gguf.py produced a file
// ggml can read, and that every weight is reachable by name.
//
// usage: ss_flow_info <path-to.gguf> [--load]
//
// By default only metadata is parsed (fast). Pass --load to also read all weight
// data into host memory (~2.6 GB for the f16 checkpoint).
#include "trellis2.h"
#include <cstdio>
#include <cstring>
#include <string>
int main(int argc, char ** argv) {
if (argc < 2) {
std::fprintf(stderr, "usage: %s <path-to.gguf> [--load]\n", argv[0]);
return 2;
}
const std::string path = argv[1];
const bool load_tensors = (argc > 2 && std::strcmp(argv[2], "--load") == 0);
std::printf("trellis2.cpp %s\n", trellis2_version());
std::string err;
trellis2_ss_flow_model * m = trellis2_ss_flow_load(path, load_tensors, &err);
if (!m) {
std::fprintf(stderr, "error: %s\n", err.c_str());
return 1;
}
const trellis2_ss_flow_hparams & hp = trellis2_ss_flow_hparams_of(m);
std::printf("file : %s (%s)\n", path.c_str(),
load_tensors ? "weights loaded" : "metadata only");
if (load_tensors) {
std::printf("backend : %s\n", trellis2_ss_flow_backend_name(m));
}
std::printf("hyperparameters:\n");
std::printf(" resolution : %d (grid %d^3 = %d tokens)\n",
hp.resolution, hp.resolution, hp.resolution * hp.resolution * hp.resolution);
std::printf(" in/out channels : %d / %d\n", hp.in_channels, hp.out_channels);
std::printf(" model_channels : %d\n", hp.model_channels);
std::printf(" cond_channels : %d\n", hp.cond_channels);
std::printf(" num_blocks : %d\n", hp.num_blocks);
std::printf(" num_heads : %d (head_dim %d)\n", hp.num_heads, hp.head_dim());
std::printf(" mlp_ratio : %.4f\n", hp.mlp_ratio);
std::printf(" pe_mode : %s (freq %.1f..%.1f)\n",
hp.pe_mode, hp.rope_freq_min, hp.rope_freq_base);
std::printf(" share_mod : %s\n", hp.share_mod ? "true" : "false");
std::printf(" qk_rms_norm : self=%s cross=%s\n",
hp.qk_rms_norm ? "true" : "false",
hp.qk_rms_norm_cross ? "true" : "false");
std::printf(" file_type : %d (0=f32,1=f16,2=bf16)\n", hp.file_type);
const int n = trellis2_ss_flow_n_tensors(m);
std::printf("tensors : %d\n", n);
// Print the global (non-block) tensors in full, then summarize block 0.
std::printf(" -- global + block 0 (sample) --\n");
size_t total_bytes = 0;
int shown = 0;
for (int i = 0; i < n; ++i) {
trellis2_tensor_info ti;
if (!trellis2_ss_flow_get_tensor_info(m, i, ti)) continue;
total_bytes += ti.n_bytes;
const bool is_global = ti.name.rfind("blocks.", 0) != 0;
const bool is_block0 = ti.name.rfind("blocks.0.", 0) == 0;
if ((is_global || is_block0) && shown < 40) {
std::printf(" %-40s %-4s [", ti.name.c_str(), ti.type_name.c_str());
for (int d = ti.n_dims - 1; d >= 0; --d) {
std::printf("%lld%s", (long long) ti.ne[d], d ? ", " : "");
}
std::printf("]\n");
++shown;
}
}
std::printf("total weight bytes: %.2f MB\n", total_bytes / (1024.0 * 1024.0));
// Spot-check a couple of expected names.
const char * probes[] = {
"input_layer.weight", "adaLN_modulation.1.weight",
"blocks.0.self_attn.to_qkv.weight", "blocks.29.cross_attn.to_kv.weight",
"out_layer.weight",
};
std::printf("name probes:\n");
for (const char * p : probes) {
std::printf(" %-40s %s\n", p,
trellis2_ss_flow_has_tensor(m, p) ? "present" : "MISSING");
}
trellis2_ss_flow_free(m);
return 0;
}
examples/ss_mesh.cpp
+95
View File
@@ -0,0 +1,95 @@
// ss_mesh — run the stage-1 SS decoder and export the occupancy isosurface as
// a Wavefront OBJ via (tetrahedral) marching cubes.
//
// usage: ss_mesh <ss_dec.gguf> <z_s.latent> [out.obj] [--iso V] [--normalize]
//
// Decodes z_s -> a 64^3 occupancy logit grid, extracts the {logit = iso} surface
// (default iso 0, i.e. the occupancy boundary), and writes it as out.obj
// (default ss_mesh.obj). With --normalize, vertices are mapped from grid-index
// units into the centered unit cube [-0.5, 0.5]^3.
#include "trellis2.h"
#include "marching_cubes.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <string>
#include <vector>
int main(int argc, char ** argv) {
if (argc < 3) {
std::fprintf(stderr, "usage: %s <ss_dec.gguf> <z_s.latent> [out.obj] [--iso V] [--normalize]\n", argv[0]);
return 2;
}
const std::string gguf_path = argv[1];
const std::string lat_path = argv[2];
std::string out_path = "ss_mesh.obj";
float iso = 0.0f;
bool normalize = false;
bool out_set = false;
for (int i = 3; i < argc; ++i) {
if (std::strcmp(argv[i], "--iso") == 0 && i + 1 < argc) iso = (float) std::atof(argv[++i]);
else if (std::strcmp(argv[i], "--normalize") == 0) normalize = true;
else if (!out_set) { out_path = argv[i]; out_set = true; }
}
std::printf("trellis2.cpp %s\n", trellis2_version());
std::string err;
trellis2_ss_dec_model * m = trellis2_ss_dec_load(gguf_path, true, &err);
if (!m) { std::fprintf(stderr, "model load error: %s\n", err.c_str()); return 1; }
std::printf("backend: %s\n", trellis2_ss_dec_backend_name(m));
const trellis2_ss_dec_hparams hp = trellis2_ss_dec_hparams_of(m);
const int Rin = hp.res_in();
const int Rout = hp.res_out();
const size_t n_in = (size_t) hp.latent_channels * Rin * Rin * Rin;
const size_t n_out = (size_t) hp.out_channels * Rout * Rout * Rout;
std::ifstream f(lat_path, std::ios::binary);
if (!f) { std::fprintf(stderr, "cannot open latent %s\n", lat_path.c_str()); trellis2_ss_dec_free(m); return 1; }
std::vector<float> latent(n_in);
f.read(reinterpret_cast<char *>(latent.data()), (std::streamsize) (n_in * sizeof(float)));
if ((size_t) f.gcount() != n_in * sizeof(float)) {
std::fprintf(stderr, "latent size mismatch: got %zu floats, want %zu\n",
(size_t) f.gcount() / sizeof(float), n_in);
trellis2_ss_dec_free(m); return 1;
}
std::vector<float> logits(n_out, 0.0f);
if (!trellis2_ss_dec_decode(m, latent.data(), logits.data(), &err)) {
std::fprintf(stderr, "decode error: %s\n", err.c_str());
trellis2_ss_dec_free(m); return 1;
}
trellis2_ss_dec_free(m);
size_t occ = 0;
for (float v : logits) if (v > iso) ++occ;
std::printf("logits : [%d^3] occupied(>%.2f)=%zu/%zu (%.2f%%)\n",
Rout, iso, occ, n_out, 100.0 * (double) occ / (double) n_out);
// Decoder output is channel-major [1, R, R, R] with linear index
// i*R^2 + j*R + k (k fastest) -> matches marching_cubes' x + y*R + z*R^2 with
// (x,y,z) = (k, j, i).
mc::Mesh mesh = mc::extract(logits.data(), Rout, Rout, Rout, iso);
std::printf("mesh : %zu verts, %zu tris\n", mesh.n_verts(), mesh.n_tris());
if (mesh.n_tris() == 0) {
std::fprintf(stderr, "warning: empty surface at iso=%.3f (nothing to write)\n", iso);
return 1;
}
if (normalize) {
for (size_t i = 0; i < mesh.verts.size(); ++i)
mesh.verts[i] = mesh.verts[i] / (float) Rout - 0.5f;
}
if (!mc::write_obj(mesh, out_path.c_str())) {
std::fprintf(stderr, "failed to write %s\n", out_path.c_str());
return 1;
}
std::printf("wrote %s\n", out_path.c_str());
return 0;
}
examples/ss_sample.cpp
+76
View File
@@ -0,0 +1,76 @@
// ss_sample — run stage-1 sampling: a DINOv3 .dinodata cond + the SS-flow DiT
// GGUF -> the sparse-structure latent z_s. The occupancy scaffold is z_s > 0.
//
// usage: ss_sample <ss_flow_dit.gguf> <cond.dinodata> [out.latent] [--seed N]
//
// Writes the z_s latent (channel-major [in_channels * R^3] float32) to out.latent
// if given — the input the SS decoder (next stage) will consume.
#include "trellis2.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <string>
#include <vector>
int main(int argc, char ** argv) {
if (argc < 3) {
std::fprintf(stderr, "usage: %s <ss_flow_dit.gguf> <cond.dinodata> [out.latent] [--seed N]\n", argv[0]);
return 2;
}
const std::string gguf_path = argv[1];
const std::string dino_path = argv[2];
std::string out_path;
uint64_t seed = 0;
for (int i = 3; i < argc; ++i) {
if (std::strcmp(argv[i], "--seed") == 0 && i + 1 < argc) seed = std::strtoull(argv[++i], nullptr, 10);
else if (out_path.empty()) out_path = argv[i];
}
std::printf("trellis2.cpp %s\n", trellis2_version());
trellis2_dino_cond cond;
std::string err;
if (!trellis2_load_dinodata(dino_path, cond, &err)) {
std::fprintf(stderr, "cond load error: %s\n", err.c_str());
return 1;
}
std::printf("cond : %lld tokens x %lld channels\n",
(long long) cond.tokens(), (long long) cond.channels());
trellis2_ss_flow_model * m = trellis2_ss_flow_load(gguf_path, true, &err);
if (!m) { std::fprintf(stderr, "model load error: %s\n", err.c_str()); return 1; }
std::printf("backend: %s\n", trellis2_ss_flow_backend_name(m));
const trellis2_ss_flow_hparams hp = trellis2_ss_flow_hparams_of(m); // copy (used after free)
const size_t N = (size_t) hp.resolution * hp.resolution * hp.resolution;
const size_t n = (size_t) hp.in_channels * N;
trellis2_ss_sampler_params P; // pipeline defaults (12 steps, gs 7.5, ...)
P.seed = seed;
std::vector<float> latent(n, 0.0f);
if (!trellis2_ss_flow_sample(m, cond.data.data(), (int) cond.tokens(), (int) cond.channels(),
&P, /*noise*/ nullptr, latent.data(), &err)) {
std::fprintf(stderr, "sample error: %s\n", err.c_str());
trellis2_ss_flow_free(m);
return 1;
}
trellis2_ss_flow_free(m);
double mn = 1e30, mx = -1e30, sum = 0.0;
size_t occ = 0;
for (float v : latent) { mn = v < mn ? v : mn; mx = v > mx ? v : mx; sum += v; if (v > 0.0f) ++occ; }
std::printf("z_s : [%d,%d,%d,%d] min=%.4f max=%.4f mean=%.5f occupancy(>0)=%.2f%%\n",
hp.in_channels, hp.resolution, hp.resolution, hp.resolution,
mn, mx, sum / (double) n, 100.0 * (double) occ / (double) n);
if (!out_path.empty()) {
std::ofstream f(out_path, std::ios::binary);
f.write(reinterpret_cast<const char *>(latent.data()), (std::streamsize) (n * sizeof(float)));
std::printf("wrote %s (%zu floats)\n", out_path.c_str(), n);
}
return 0;
}
fuzz/CMakeLists.txt
+18
View File
@@ -0,0 +1,18 @@
# libFuzzer harnesses for the untrusted (non-GGUF) input paths.
# Requires clang; enable with the fuzz build:
# CC=clang CXX=clang++ cmake -B build-fuzz -DTRELLIS2_FUZZ=ON
# cmake --build build-fuzz -j
# ./build-fuzz/fuzz/fuzz_image corpus_image/ -max_len=1048576
# ./build-fuzz/fuzz/fuzz_dinodata corpus_dinodata/ -max_len=1048576
if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang")
message(FATAL_ERROR "TRELLIS2_FUZZ requires clang (libFuzzer)")
endif()
foreach(target fuzz_image fuzz_dinodata)
add_executable(${target} ${target}.cpp)
target_link_libraries(${target} PRIVATE trellis2)
target_compile_options(${target} PRIVATE -fsanitize=fuzzer)
target_link_options(${target} PRIVATE -fsanitize=fuzzer)
target_compile_features(${target} PRIVATE cxx_std_14)
endforeach()
fuzz/crashes-fixed/README.md
+10
View File
@@ -0,0 +1,10 @@
# Fixed-crash regression seeds
Inputs that once crashed a fuzz harness and now must not. Re-run after changes:
./build-fuzz/fuzz/fuzz_dinodata fuzz/crashes-fixed/
- `dinodata-shape-overflow` — a `.dinodata` header whose shape dims multiply to
an enormous element count; the loader used to `vector::resize` it and throw
`std::length_error`. Fixed by an overflow-checked element-count cap in
`trellis2_load_dinodata`.
fuzz/crashes-fixed/dinodata-shape-overflow
BIN
View File
Binary file not shown.
fuzz/fuzz_dinodata.cpp
+57
View File
@@ -0,0 +1,57 @@
// libFuzzer harness for the .dinodata loader: arbitrary bytes -> temp file ->
// trellis2_load_dinodata -> (on success) fingerprints over the payload.
//
// .dinodata files typically come from this codebase itself, but the demo and
// CLI accept arbitrary paths, so the parser must be robust: reject bad magic /
// truncated headers / shape overflow cleanly, never over-read.
//
// Invariants checked with abort() (not just "no crash"):
// * on success, data.size() == product(shape) and shape is non-empty
// * fingerprints run over the whole payload without UB
//
// Run: ./build-fuzz/fuzz/fuzz_dinodata corpus_dinodata/ -max_len=1048576
#include "../trellis2.h"
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <string>
#include <unistd.h>
extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) {
if (size > (16u << 20)) {
return 0;
}
char path[] = "/tmp/fuzz_dinodata_XXXXXX";
const int fd = mkstemp(path);
if (fd < 0) {
return 0;
}
const ssize_t written = write(fd, data, size);
close(fd);
if (written != (ssize_t) size) {
unlink(path);
return 0;
}
trellis2_dino_cond cond;
std::string err;
if (trellis2_load_dinodata(path, cond, &err)) {
int64_t total = 1;
for (int64_t d : cond.shape) total *= d;
if (cond.shape.empty() || (size_t) total != cond.data.size()) {
abort(); // loader produced an inconsistent cond
}
const trellis2_dino_fingerprint fp = trellis2_dino_fingerprints(cond);
if (fp.count != cond.data.size()) {
abort();
}
} else if (err.empty()) {
abort(); // failure must come with a reason
}
unlink(path);
return 0;
}
fuzz/fuzz_image.cpp
+33
View File
@@ -0,0 +1,33 @@
// libFuzzer harness for the image upload path: arbitrary bytes ->
// t2_preprocess_image_bytes (stb_image decode -> alpha bbox crop ->
// premultiply -> PIL-compatible Lanczos resize).
//
// This is the demo server's untrusted-input surface: the Go server hands
// uploaded bytes straight to this function. GGUF model files are trusted
// assets and deliberately not fuzzed here (privacy-filter.cpp threat model).
//
// Build via -DTRELLIS2_FUZZ=ON (clang only); run:
// ./build-fuzz/fuzz/fuzz_image corpus_image/ -max_len=1048576
//
// Success invariant: the function either fails cleanly (nonzero + error
// message) or fills exactly out_size^2*3 bytes. Any crash/leak/UB is a bug.
#include "../trellis2_capi.h"
#include <cstdint>
#include <cstring>
#include <vector>
extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) {
if (size == 0 || size > (32u << 20)) {
return 0;
}
// Small output target keeps per-input cost low; the resampler code path
// is identical for any out_size.
static thread_local std::vector<unsigned char> out(64 * 64 * 3);
char err[256] = {0};
(void) t2_preprocess_image_bytes(data, (int) size, 64, out.data(), err, sizeof(err));
return 0;
}
ggml/.editorconfig
+22
View File
@@ -0,0 +1,22 @@
# https://EditorConfig.org
# Top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file, utf-8 charset
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 4
[*.md]
indent_size = 2
[Makefile]
indent_style = tab
[prompts/*.txt]
insert_final_newline = unset
ggml/.github/pull_request_template.md
Vendored
+1
View File
@@ -0,0 +1 @@
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
ggml/.github/workflows/ci.yml
Vendored
+272
View File
@@ -0,0 +1,272 @@
name: CI
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
libraries: [shared, static]
runs-on: ${{ matrix.os }}
steps:
- name: Clone
uses: actions/checkout@v6
- name: Dependencies for Ubuntu
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt-get install llvm
- name: Add msbuild to PATH
if: matrix.os == 'windows-latest'
uses: microsoft/setup-msbuild@v2
- name: Create Build Environment
run: mkdir build
- name: Configure CMake
working-directory: ./build
run: cmake ..
${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
${{ matrix.libraries == 'static' && '-DBUILD_SHARED_LIBS=OFF' || '-DBUILD_SHARED_LIBS=ON' }}
-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed
-DGGML_METAL=OFF
- name: Build
working-directory: ./build
run: cmake --build . ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
- name: Test
working-directory: ./build
run: ctest --verbose --timeout 900 ${{ contains(matrix.os, 'windows') && '--build-config Release' || '' }}
- name: Install
working-directory: ./build
run: cmake --build . --target install ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
- name: Test CMake config
run: |
mkdir test-cmake
cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
# TODO: simplify the following workflows using a matrix
ggml-ci-x64-cpu-low-perf:
runs-on: ubuntu-22.04
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-x64-cpu-low-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-low-perf:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-low-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-x64-cpu-high-perf:
runs-on: ubuntu-22.04
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-x64-cpu-high-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-high-perf:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-high-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-high-perf-sve:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-high-perf-sve
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-x64-nvidia-cuda:
runs-on: [self-hosted, Linux, X64, NVIDIA]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
nvidia-smi
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-x64-nvidia-vulkan-cm:
runs-on: [self-hosted, Linux, X64, NVIDIA]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-x64-nvidia-vulkan-cm2:
runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
# TODO: provision AMX-compatible machine
#ggml-ci-x64-cpu-amx:
# runs-on: [self-hosted, Linux, X64, CPU, AMX]
# steps:
# - name: Clone
# id: checkout
# uses: actions/checkout@v6
# - name: Test
# id: ggml-ci
# run: |
# bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-mac-metal:
runs-on: [self-hosted, macOS, ARM64]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
ggml-ci-mac-vulkan:
runs-on: [self-hosted, macOS, ARM64]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
ggml/.github/workflows/release.yml
Vendored
+27
View File
@@ -0,0 +1,27 @@
name: Release
on:
push:
tags:
- 'v*'
jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Create Release
id: create_release
uses: ggml-org/action-create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref_name }}
release_name: ${{ github.ref }}
draft: false
prerelease: false
ggml/.gitignore
+39
View File
@@ -0,0 +1,39 @@
build/
build-*/
out/
tmp/
models/
models-mnt
compile_commands.json
CMakeSettings.json
.vs/
.vscode/
.idea/
.clangd
.venv/
ggml_env/
.exrc
.cache
.DS_Store
.stablelm
.gpt-2
src/arm_neon.h
tests/arm_neon.h
zig-out/
zig-cache/
*.o
*.d
*.dot
*.sw?
__pycache__/
# Model files
ggml-model-f16.bin
*.bat
ggml/.gitmodules
View File
ggml/AUTHORS
+335
View File
@@ -0,0 +1,335 @@
# date: Tue Feb 4 13:03:51 EET 2025
# this file is auto-generated by scripts/gen-authors.sh
0cc4m <picard12@live.de>
65a <10104049+65a@users.noreply.github.com>
AT <manyoso@users.noreply.github.com>
Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
Adam Tazi <52357206+ad1tazi@users.noreply.github.com>
Adrien Gallouët <adrien@gallouet.fr>
Adrien Gallouët <angt@huggingface.co>
Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>
AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>
AidanBeltonS <aidan.belton@codeplay.com>
Akarshan Biswas <akarshan.biswas@gmail.com>
Akarshan Biswas <akarshanbiswas@fedoraproject.org>
Albert Jin <albert.jin@gmail.com>
Alberto Cabrera Pérez <alberto.cabrera@codeplay.com>
Alberto Cabrera Pérez <alberto.cabrera@intel.com>
Alex Azarov <alex@azarov.by>
Alex O'Connell <35843486+acon96@users.noreply.github.com>
Alex von Gluck IV <kallisti5@unixzen.com>
AmbientL <107641468+AmbientL@users.noreply.github.com>
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
Ananta Bastola <anantarajbastola@gmail.com>
Andreas (Andi) Kunar <andreask@msn.com>
Andreas Kieslinger <47689530+aendk@users.noreply.github.com>
Andrei <abetlen@gmail.com>
Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>
Andrii Ryzhkov <andriiryzhkov@users.noreply.github.com>
Arjun <ccldarjun@icloud.com>
Ashraful Islam <ashraful.meche@gmail.com>
Astariul <43774355+astariul@users.noreply.github.com>
AsukaMinato <asukaminato@nyan.eu.org>
Avi Lumelsky <avilume@gmail.com>
Bart Pelle <3662930+Velocity-@users.noreply.github.com>
Ben Ashbaugh <ben.ashbaugh@intel.com>
Bernhard M. Wiedemann <githubbmwprimary@lsmod.de>
Borislav Stanimirov <b.stanimirov@abv.bg>
Brad Ito <phlogisticfugu@users.noreply.github.com>
Brad Murray <59848399+bradmurray-dt@users.noreply.github.com>
Brian <mofosyne@gmail.com>
Bryan Lozano <b.lozano.havoc@gmail.com>
Carolinabanana <140120812+Carolinabanana@users.noreply.github.com>
CarterLi999 <664681047@qq.com>
Cebtenzzre <cebtenzzre@gmail.com>
Changyeon Kim <cyzero.kim@samsung.com>
Charles Xu <63788048+chaxu01@users.noreply.github.com>
Charles Xu <charles.xu@arm.com>
Chen Xi <xi2.chen@intel.com>
Chen Xi <xixichen08@foxmail.com>
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
Chris Elrod <elrodc@gmail.com>
Christian Kastner <ckk@kvr.at>
Clint Herron <hanclinto@gmail.com>
Conrad Kramer <conrad@conradkramer.com>
Cordeiro <1471463+ocordeiro@users.noreply.github.com>
Cristiano Calcagno <cristianoc@users.noreply.github.com>
DAN™ <dranger003@gmail.com>
Dan Forbes <dan@danforbes.dev>
Dan Johansson <164997844+eddnjjn@users.noreply.github.com>
Dan Johansson <dan.johansson@arm.com>
Daniel Bevenius <daniel.bevenius@gmail.com>
Daniel Ziegenberg <daniel@ziegenberg.at>
Daniele <57776841+daniandtheweb@users.noreply.github.com>
Daulet Zhanguzin <daulet@users.noreply.github.com>
Dave <dave-fl@users.noreply.github.com>
Dave Airlie <airlied@gmail.com>
Dave Airlie <airlied@redhat.com>
David Miller <david@patagona.ca>
DavidKorczynski <david@adalogics.com>
Davidson Francis <davidsondfgl@gmail.com>
Dibakar Gope <dibakar.gope@arm.com>
Didzis Gosko <didzis@users.noreply.github.com>
Diego Devesa <slarengh@gmail.com>
Diogo <dgcruz983@gmail.com>
Djip007 <3705339+Djip007@users.noreply.github.com>
Djip007 <djip.perois@free.fr>
Dou Xinpeng <15529241576@163.com>
Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com>
Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>
Ebey Abraham <ebey97@gmail.com>
Eldar Yusupov <eyusupov@gmail.com>
Emmanuel Durand <emmanueldurand@protonmail.com>
Engininja2 <139037756+Engininja2@users.noreply.github.com>
Eric Zhang <34133756+EZForever@users.noreply.github.com>
Erik Scholz <Green-Sky@users.noreply.github.com>
Ettore Di Giacinto <mudler@users.noreply.github.com>
Eve <139727413+netrunnereve@users.noreply.github.com>
F1L1P <78918286+F1L1Pv2@users.noreply.github.com>
Faisal Zaghloul <quic_fzaghlou@quicinc.com>
FantasyGmm <16450052+FantasyGmm@users.noreply.github.com>
Felix <stenbackfelix@gmail.com>
Finn Voorhees <finnvoorhees@gmail.com>
FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>
Frankie Robertson <frankier@users.noreply.github.com>
GainLee <perfecter.gen@gmail.com>
George Hindle <george@georgehindle.com>
Georgi Gerganov <ggerganov@gmail.com>
Gilad S <7817232+giladgd@users.noreply.github.com>
Gilad S <giladgd@users.noreply.github.com>
Gilad S. <7817232+giladgd@users.noreply.github.com>
Guillaume Wenzek <gwenzek@users.noreply.github.com>
Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>
Haus1 <haus.xda@gmail.com>
Herman Semenov <GermanAizek@yandex.ru>
HimariO <dsfhe49854@gmail.com>
Hirochika Matsumoto <git@hkmatsumoto.com>
Hong Bo PENG <penghb@cn.ibm.com>
Hugo Rosenkranz-Costa <hugo.rosenkranz@gmail.com>
Hyunsung Lee <ita9naiwa@gmail.com>
IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com>
Ian Bull <irbull@eclipsesource.com>
Ihar Hrachyshka <ihrachys@redhat.com>
Ikko Eltociear Ashimine <eltociear@gmail.com>
Ivan <nekotekina@gmail.com>
Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
Ivan Stepanov <ivanstepanovftw@gmail.com>
Ivan Zdane <accounts@ivanzdane.com>
Jack Mousseau <jmousseau@users.noreply.github.com>
Jack Vial <vialjack@gmail.com>
JacobLinCool <jacoblincool@gmail.com>
Jakob Frick <jakob.maria.frick@gmail.com>
Jan Ploski <jpl@plosquare.com>
Jared Van Bortel <jared@nomic.ai>
Jeff Bolz <jbolz@nvidia.com>
Jeffrey Quesnelle <jquesnelle@gmail.com>
Jeroen Mostert <jeroen.mostert@cm.com>
Jiahao Li <liplus17@163.com>
JidongZhang-THU <1119708529@qq.com>
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
Jo Liss <joliss42@gmail.com>
Joe Todd <joe.todd@codeplay.com>
Johannes Gäßler <johannesg@5d6.de>
John Balis <phobossystems@gmail.com>
Josh Bleecher Snyder <josharian@gmail.com>
Judd <foldl@users.noreply.github.com>
Jun Hee Yoo <contact.jhyoo@gmail.com>
Junil Kim <logyourself@gmail.com>
Justina Cho <justcho5@gmail.com>
Justine Tunney <jtunney@gmail.com>
Justine Tunney <jtunney@mozilla.com>
Karol Kontny <82021046+kkontny@users.noreply.github.com>
Kawrakow <48489457+ikawrakow@users.noreply.github.com>
Kevin Gibbons <bakkot@gmail.com>
Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
Kylin <56434533+KyL0N@users.noreply.github.com>
LoganDark <git@logandark.mozmail.com>
LoganDark <github@logandark.mozmail.com>
LostRuins <39025047+LostRuins@users.noreply.github.com>
Lukas Möller <mail@lukas-moeller.ch>
M Refi D.A <24388107+refinism@users.noreply.github.com>
M. Yusuf Sarıgöz <yusufsarigoz@gmail.com>
Ma Mingfei <mingfei.ma@intel.com>
Mahesh Madhav <67384846+heshpdx@users.noreply.github.com>
MaiHD <maihd.dev@gmail.com>
Mark Zhuang <zhuangqiubin@gmail.com>
Markus Tavenrath <mtavenrath@users.noreply.github.com>
Masaya, Kato <62578291+msy-kato@users.noreply.github.com>
Mathieu Baudier <mbaudier@argeo.org>
Mathijs de Bruin <mathijs@mathijsfietst.nl>
Matt Stephenson <mstephenson6@users.noreply.github.com>
Max Krasnyansky <max.krasnyansky@gmail.com>
Max Krasnyansky <quic_maxk@quicinc.com>
Mayank Kumar Pal <mynkpl1998@gmail.com>
Meng, Hengyu <hengyu.meng@intel.com>
Mengqing Cao <cmq0113@163.com>
Metal Whale <45712559+metalwhale@users.noreply.github.com>
Michael Klimenko <mklimenko29@gmail.com>
Michael Podvitskiy <podvitskiymichael@gmail.com>
Michael Verrilli <msv@pobox.com>
Molly Sophia <mollysophia379@gmail.com>
Natsu <chino@hotococoa.moe>
Neo Zhang <14088817+arthw@users.noreply.github.com>
Neo Zhang Jianyu <jianyu.zhang@intel.com>
Neuman Vong <neuman.vong@gmail.com>
Nevin <nevinpuri1901@gmail.com>
Nicholai Tukanov <nicholaitukanov@gmail.com>
Nico Bosshard <nico@bosshome.ch>
Nicolò Scipione <nicolo.scipione@codeplay.com>
Nikita Sarychev <42014488+sARY77@users.noreply.github.com>
Nouamane Tazi <nouamane98@gmail.com>
Olivier Chafik <ochafik@google.com>
Olivier Chafik <ochafik@users.noreply.github.com>
Ondřej Čertík <ondrej@certik.us>
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
PAB <pierreantoine.bannier@gmail.com>
Paul Tsochantaris <ptsochantaris@icloud.com>
Peter <peter277@users.noreply.github.com>
Philpax <me@philpax.me>
Pierre Alexandre SCHEMBRI <pa.schembri@gmail.com>
Plamen Minev <pacominev@gmail.com>
Playdev <josang1204@gmail.com>
Prashant Vithule <119530321+Vithulep@users.noreply.github.com>
Przemysław Pawełczyk <przemoc@gmail.com>
R0CKSTAR <xiaodong.ye@mthreads.com>
R0CKSTAR <yeahdongcn@gmail.com>
Radoslav Gerganov <rgerganov@gmail.com>
Radosław Gryta <radek.gryta@gmail.com>
Ravindra Marella <marella@users.noreply.github.com>
Ray Cromwell <cromwellian@gmail.com>
Reinforce-II <fate@eastal.com>
Rémy Oudompheng <oudomphe@phare.normalesup.org>
Reza Rezvan <reza@rezvan.xyz>
Rick G <26732651+TheFlipbook@users.noreply.github.com>
RiverZhou <riverzhou2000@gmail.com>
Robert Ormandi <52251610+ormandi@users.noreply.github.com>
Romain Biessy <romain.biessy@codeplay.com>
Ronsor <ronsor@ronsor.pw>
Rotem Dan <rotemdan@gmail.com>
Ryan Hitchman <hitchmanr@gmail.com>
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
SXX <sxx1136965276@gmail.com>
Salvatore Mesoraca <s.mesoraca16@gmail.com>
Sam Spilsbury <smspillaz@gmail.com>
Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>
Santtu Keskinen <santtu.keskinen@gmail.com>
Sergio López <slp@redhat.com>
Sergio López <slp@sinrega.org>
Shanshan Shen <467638484@qq.com>
Shijie <821898965@qq.com>
Shupei Fan <dymarkfan@outlook.com>
Siddharth Ramakrishnan <srr2141@columbia.edu>
Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
Skyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com>
Slava Primenko <primenko.s@gmail.com>
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
Supreet Sethi <supreet.sethi@gmail.com>
Takuya Takeuchi <takuya.takeuchi.dev@gmail.com>
Tamotsu Takahashi <ttakah+github@gmail.com>
Tanmay <tnmysachan@gmail.com>
Tanmay Sachan <tnmysachan@gmail.com>
Timothy Cronin <40186632+4imothy@users.noreply.github.com>
Tom Bailey <tombailey@users.noreply.github.com>
Tom Jobbins <784313+TheBloke@users.noreply.github.com>
Tony Wasserka <4840017+neobrain@users.noreply.github.com>
Tristan Druyen <tristan@vault81.mozmail.com>
Tyé singwa <92231658+tye-singwa@users.noreply.github.com>
UEXTM.com <84163508+uextm@users.noreply.github.com>
WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com>
William Tambellini <william.tambellini@gmail.com>
William Tambellini <wtambellini@sdl.com>
XiaotaoChen <chenxiaotao1234@gmail.com>
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
Xuan Son Nguyen <thichthat@gmail.com>
Yavor Ivanov <yivanov@viewray.com>
YavorGIvanov <yivanov@viewray.com>
Yilong Guo <vfirst218@gmail.com>
Yilong Guo <yilong.guo@intel.com>
Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
Zhiyuan Li <lizhiyuan@uniartisan.com>
Zhiyuan Li <uniartisan2017@gmail.com>
a3sh <38979186+A3shTnT@users.noreply.github.com>
ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>
agray3 <agray3@users.noreply.github.com>
amd-dwang <dong.wang@amd.com>
amritahs-ibm <amritahs@linux.vnet.ibm.com>
apcameron <37645737+apcameron@users.noreply.github.com>
appvoid <78444142+appvoid@users.noreply.github.com>
ariez-xyz <41232910+ariez-xyz@users.noreply.github.com>
automaticcat <daogiatuank54@gmail.com>
bandoti <141645996+bandoti@users.noreply.github.com>
bmwl <brian.marshall@tolko.com>
bobqianic <129547291+bobqianic@users.noreply.github.com>
bssrdf <merlintiger@hotmail.com>
chengchi <davesjoewang@gmail.com>
compilade <113953597+compilade@users.noreply.github.com>
compilade <git@compilade.net>
ddpasa <112642920+ddpasa@users.noreply.github.com>
denersc <denerstassun@gmail.com>
dscripka <dscripka@users.noreply.github.com>
fitzsim <fitzsim@fitzsim.org>
fj-y-saito <85871716+fj-y-saito@users.noreply.github.com>
fraxy-v <65565042+fraxy-v@users.noreply.github.com>
gn64 <yukikaze.jp@gmail.com>
goerch <jhr.walter@t-online.de>
goldwaving <77494627+goldwaving@users.noreply.github.com>
haopeng <657407891@qq.com>
hidenorly <hidenorly@users.noreply.github.com>
hipudding <huafengchun@gmail.com>
hydai <z54981220@gmail.com>
issixx <46835150+issixx@users.noreply.github.com>
jaeminSon <woalsdnd@gmail.com>
jdomke <28772296+jdomke@users.noreply.github.com>
jiez <373447296@qq.com>
johnson442 <56517414+johnson442@users.noreply.github.com>
junchao-loongson <68935141+junchao-loongson@users.noreply.github.com>
k.h.lai <adrian.k.h.lai@outlook.com>
katsu560 <118887472+katsu560@users.noreply.github.com>
klosax <131523366+klosax@users.noreply.github.com>
kunnis <kunnis@users.noreply.github.com>
l3utterfly <gc.pthzfoldr@gmail.com>
le.chang <cljs118@126.com>
leejet <31925346+leejet@users.noreply.github.com>
leejet <leejet714@gmail.com>
leo-pony <nengjunma@outlook.com>
lhez <quic_lih@quicinc.com>
liuwei-git <14815172+liuwei-git@users.noreply.github.com>
luoyu-intel <yu.luo@intel.com>
magicse <magicse@users.noreply.github.com>
mahorozte <41834471+mahorozte@users.noreply.github.com>
mashizora <30516315+mashizora@users.noreply.github.com>
matt23654 <matthew.webber@protonmail.com>
matteo <matteogeniaccio@yahoo.it>
ochafik <ochafik@google.com>
otaGran <ujt2h8@gmail.com>
pengxin99 <pengxin.yuan@intel.com>
pikalover6 <49179590+pikalover6@users.noreply.github.com>
postmasters <namnguyen@google.com>
sjinzh <sjinzh@gmail.com>
skirodev <57715494+skirodev@users.noreply.github.com>
slaren <slarengh@gmail.com>
snadampal <87143774+snadampal@users.noreply.github.com>
someone13574 <81528246+someone13574@users.noreply.github.com>
stduhpf <stephduh@live.fr>
taher <8665427+nullhook@users.noreply.github.com>
texmex76 <40733439+texmex76@users.noreply.github.com>
the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com>
thewh1teagle <61390950+thewh1teagle@users.noreply.github.com>
ucag.li <ucag@qq.com>
ulatekh <ulatekh@yahoo.com>
uvos <devnull@uvos.xyz>
uvos <philipp@uvos.xyz>
wangshuai09 <391746016@qq.com>
woachk <24752637+woachk@users.noreply.github.com>
xctan <axunlei@gmail.com>
yangyaofei <yangyaofei@gmail.com>
yuri@FreeBSD <yuri@FreeBSD>
zhentaoyu <zhentao.yu@intel.com>
zhouwg <6889919+zhouwg@users.noreply.github.com>
zhouwg <zhouwg2000@gmail.com>
谢乃闻 <sienaiwun@users.noreply.github.com>
布客飞龙 <562826179@qq.com>
旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com>
ggml/CMakeLists.txt
+497
View File
@@ -0,0 +1,497 @@
cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.
project("ggml" C CXX ASM)
### GGML Version
set(GGML_VERSION_MAJOR 0)
set(GGML_VERSION_MINOR 9)
set(GGML_VERSION_PATCH 9)
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
if(GIT_EXE)
# Get current git commit hash
execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE GGML_BUILD_COMMIT
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
)
# Check if the working directory is dirty (i.e., has uncommitted changes)
execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- .
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE GGML_GIT_DIRTY
ERROR_QUIET
)
endif()
set(GGML_VERSION "${GGML_VERSION_BASE}")
if(NOT GGML_BUILD_COMMIT)
set(GGML_BUILD_COMMIT "unknown")
endif()
# Build the commit string with optional dirty flag
if(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1)
set(GGML_BUILD_COMMIT "${GGML_BUILD_COMMIT}-dirty")
endif()
include(CheckIncludeFileCXX)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(GGML_STANDALONE ON)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
# configure project version
# TODO
else()
set(GGML_STANDALONE OFF)
if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
endif()
endif()
if (EMSCRIPTEN)
set(BUILD_SHARED_LIBS_DEFAULT OFF)
option(GGML_WASM_SINGLE_FILE "ggml: embed WASM inside the generated ggml.js" ON)
else()
if (MINGW)
set(BUILD_SHARED_LIBS_DEFAULT OFF)
else()
set(BUILD_SHARED_LIBS_DEFAULT ON)
endif()
endif()
# remove the lib prefix on win32 mingw
if (WIN32)
set(CMAKE_STATIC_LIBRARY_PREFIX "")
set(CMAKE_SHARED_LIBRARY_PREFIX "")
set(CMAKE_SHARED_MODULE_PREFIX "")
endif()
option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
option(GGML_BACKEND_DL "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
set(GGML_BACKEND_DIR "" CACHE PATH "ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL")
#
# option list
#
# TODO: mark all options as advanced when not GGML_STANDALONE
if (APPLE)
set(GGML_METAL_DEFAULT ON)
set(GGML_BLAS_DEFAULT ON)
set(GGML_BLAS_VENDOR_DEFAULT "Apple")
else()
set(GGML_METAL_DEFAULT OFF)
set(GGML_BLAS_DEFAULT OFF)
set(GGML_BLAS_VENDOR_DEFAULT "Generic")
endif()
if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH})
message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF")
set(GGML_NATIVE_DEFAULT OFF)
else()
set(GGML_NATIVE_DEFAULT ON)
endif()
# defaults
if (NOT GGML_LLAMAFILE_DEFAULT)
set(GGML_LLAMAFILE_DEFAULT OFF)
endif()
if (NOT GGML_CUDA_GRAPHS_DEFAULT)
set(GGML_CUDA_GRAPHS_DEFAULT OFF)
endif()
# general
option(GGML_STATIC "ggml: static link libraries" OFF)
option(GGML_NATIVE "ggml: optimize the build for the current system" ${GGML_NATIVE_DEFAULT})
option(GGML_LTO "ggml: enable link time optimization" OFF)
option(GGML_CCACHE "ggml: use ccache if available" ON)
# debug
option(GGML_ALL_WARNINGS "ggml: enable all compiler warnings" ON)
option(GGML_ALL_WARNINGS_3RD_PARTY "ggml: enable all compiler warnings in 3rd party libs" OFF)
option(GGML_GPROF "ggml: enable gprof" OFF)
# build
option(GGML_FATAL_WARNINGS "ggml: enable -Werror flag" OFF)
# sanitizers
option(GGML_SANITIZE_THREAD "ggml: enable thread sanitizer" OFF)
option(GGML_SANITIZE_ADDRESS "ggml: enable address sanitizer" OFF)
option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF)
# instruction set specific
if (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)
set(INS_ENB OFF)
else()
set(INS_ENB ON)
endif()
message(DEBUG "GGML_NATIVE : ${GGML_NATIVE}")
message(DEBUG "GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}")
message(DEBUG "INS_ENB : ${INS_ENB}")
option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF)
option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF)
option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB})
option(GGML_AVX "ggml: enable AVX" ${INS_ENB})
option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF)
option(GGML_AVX2 "ggml: enable AVX2" ${INS_ENB})
option(GGML_BMI2 "ggml: enable BMI2" ${INS_ENB})
option(GGML_AVX512 "ggml: enable AVX512F" OFF)
option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI" OFF)
option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI" OFF)
option(GGML_AVX512_BF16 "ggml: enable AVX512-BF16" OFF)
if (NOT MSVC)
# in MSVC F16C and FMA is implied with AVX2/AVX512
option(GGML_FMA "ggml: enable FMA" ${INS_ENB})
option(GGML_F16C "ggml: enable F16C" ${INS_ENB})
# MSVC does not seem to support AMX
option(GGML_AMX_TILE "ggml: enable AMX-TILE" OFF)
option(GGML_AMX_INT8 "ggml: enable AMX-INT8" OFF)
option(GGML_AMX_BF16 "ggml: enable AMX-BF16" OFF)
endif()
option(GGML_LASX "ggml: enable lasx" ON)
option(GGML_LSX "ggml: enable lsx" ON)
option(GGML_RVV "ggml: enable rvv" ON)
option(GGML_RV_ZFH "ggml: enable riscv zfh" ON)
option(GGML_RV_ZVFH "ggml: enable riscv zvfh" ON)
option(GGML_RV_ZICBOP "ggml: enable riscv zicbop" ON)
option(GGML_RV_ZIHINTPAUSE "ggml: enable riscv zihintpause " ON)
option(GGML_XTHEADVECTOR "ggml: enable xtheadvector" OFF)
option(GGML_VXE "ggml: enable vxe" ${GGML_NATIVE})
option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM")
set(GGML_CPU_POWERPC_CPUTYPE "" CACHE STRING "ggml: CPU type for PowerPC")
# ggml core
set(GGML_SCHED_MAX_COPIES "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
option(GGML_CPU "ggml: enable CPU backend" ON)
option(GGML_SCHED_NO_REALLOC "ggml: disallow reallocations in ggml-alloc (for debugging)" OFF)
# 3rd party libs / backends
option(GGML_ACCELERATE "ggml: enable Accelerate framework" ON)
option(GGML_BLAS "ggml: use BLAS" ${GGML_BLAS_DEFAULT})
set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING
"ggml: BLAS library vendor")
option(GGML_LLAMAFILE "ggml: use LLAMAFILE" ${GGML_LLAMAFILE_DEFAULT})
option(GGML_CUDA "ggml: use CUDA" OFF)
option(GGML_MUSA "ggml: use MUSA" OFF)
option(GGML_CUDA_FORCE_MMQ "ggml: use mmq kernels instead of cuBLAS" OFF)
option(GGML_CUDA_FORCE_CUBLAS "ggml: always use cuBLAS instead of mmq kernels" OFF)
set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
"ggml: max. batch size for using peer access")
option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF)
option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF)
option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON)
option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF)
option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT})
set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING
"ggml: cuda link binary compression mode; requires cuda 12.8+")
set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size")
option(GGML_HIP "ggml: use HIP" OFF)
option(GGML_HIP_GRAPHS "ggml: use HIP graph, experimental, slow" OFF)
option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON)
option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF)
option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON)
option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF)
option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF)
option(GGML_MUSA_MUDNN_COPY "ggml: enable muDNN for accelerated copy" OFF)
option(GGML_VULKAN "ggml: use Vulkan" OFF)
option(GGML_VULKAN_CHECK_RESULTS "ggml: run Vulkan op checks" OFF)
option(GGML_VULKAN_DEBUG "ggml: enable Vulkan debug output" OFF)
option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug output" OFF)
option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF)
option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF)
option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF)
option(GGML_WEBGPU "ggml: use WebGPU" OFF)
option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF)
option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF)
option(GGML_WEBGPU_GPU_PROFILE "ggml: enable WebGPU profiling (GPU)" OFF)
option(GGML_WEBGPU_JSPI "ggml: use JSPI for WebGPU" ON)
option(GGML_ZDNN "ggml: use zDNN" OFF)
option(GGML_VIRTGPU "ggml: use the VirtGPU/Virglrenderer API Remoting frontend" OFF)
option(GGML_VIRTGPU_BACKEND "ggml: build the VirtGPU/Virglrenderer API Remoting backend" OFF)
option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT})
option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF)
option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF)
option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" ${GGML_METAL})
set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
"ggml: metal minimum macOS version")
set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)")
option(GGML_OPENMP "ggml: use OpenMP" ON)
option(GGML_RPC "ggml: use RPC" OFF)
option(GGML_SYCL "ggml: use SYCL" OFF)
option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF)
option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON)
option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON)
set (GGML_SYCL_TARGET "INTEL" CACHE STRING
"ggml: sycl target device")
set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
"ggml: sycl device architecture")
option(GGML_OPENVINO "ggml: use OPENVINO" OFF)
option(GGML_OPENCL "ggml: use OpenCL" OFF)
option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF)
option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON)
option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON)
set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
"ggml: OpenCL API version to target")
option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF)
set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)")
# toolchain for vulkan-shaders-gen
set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
option(GGML_ZENDNN "ggml: use ZenDNN" OFF)
option(ZENDNN_ROOT "ggml: path to ZenDNN installation" "")
# extra artifacts
option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE})
option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
#
# dependencies
#
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED true)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED true)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include(GNUInstallDirs)
#
# build the library
#
add_subdirectory(src)
#
# tests and examples
#
if (GGML_BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
endif ()
if (GGML_BUILD_EXAMPLES)
add_subdirectory(examples)
endif ()
#
# install
#
include(CMakePackageConfigHelpers)
# all public headers
set(GGML_PUBLIC_HEADERS
include/ggml.h
include/ggml-cpu.h
include/ggml-alloc.h
include/ggml-backend.h
include/ggml-blas.h
include/ggml-cann.h
include/ggml-cpp.h
include/ggml-cuda.h
include/ggml-opt.h
include/ggml-metal.h
include/ggml-rpc.h
include/ggml-virtgpu.h
include/ggml-sycl.h
include/ggml-vulkan.h
include/ggml-webgpu.h
include/ggml-zendnn.h
include/ggml-openvino.h
include/gguf.h)
set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
#if (GGML_METAL)
# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
#endif()
install(TARGETS ggml LIBRARY PUBLIC_HEADER)
install(TARGETS ggml-base LIBRARY)
if (GGML_STANDALONE)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in
${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
@ONLY)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
DESTINATION share/pkgconfig)
endif()
#
# Create CMake package
#
# Capture variables prefixed with GGML_.
set(variable_set_statements
"
####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() #######
####### Any changes to this file will be overwritten by the next CMake run #######
")
set(GGML_SHARED_LIB ${BUILD_SHARED_LIBS})
get_cmake_property(all_variables VARIABLES)
foreach(variable_name IN LISTS all_variables)
if(variable_name MATCHES "^GGML_")
string(REPLACE ";" "\\;"
variable_value "${${variable_name}}")
set(variable_set_statements
"${variable_set_statements}set(${variable_name} \"${variable_value}\")\n")
endif()
endforeach()
set(GGML_VARIABLES_EXPANDED ${variable_set_statements})
# Create the CMake package and set install location.
set(GGML_INSTALL_VERSION ${GGML_VERSION})
set(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files")
set(GGML_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files")
set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files")
configure_package_config_file(
${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml
PATH_VARS GGML_INCLUDE_INSTALL_DIR
GGML_LIB_INSTALL_DIR
GGML_BIN_INSTALL_DIR)
write_basic_package_version_file(
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
VERSION ${GGML_INSTALL_VERSION}
COMPATIBILITY SameMajorVersion)
target_compile_definitions(ggml-base PRIVATE
GGML_VERSION="${GGML_INSTALL_VERSION}"
GGML_COMMIT="${GGML_BUILD_COMMIT}"
)
message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
if (MSVC)
set(MSVC_WARNING_FLAGS
/wd4005 # Macro redefinition
/wd4244 # Conversion from one type to another type, possible loss of data
/wd4267 # Conversion from 'size_t' to a smaller type, possible loss of data
/wd4305 # Conversion from 'type1' to 'type2', possible loss of data
/wd4566 # Conversion from 'char' to 'wchar_t', possible loss of data
/wd4996 # Disable POSIX deprecation warnings
/wd4702 # Unreachable code warnings
)
set(MSVC_COMPILE_OPTIONS
"$<$<COMPILE_LANGUAGE:C>:/utf-8>"
"$<$<COMPILE_LANGUAGE:CXX>:/utf-8>"
)
function(configure_msvc_target target_name)
if(TARGET ${target_name})
target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})
target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS})
endif()
endfunction()
configure_msvc_target(ggml-base)
configure_msvc_target(ggml)
configure_msvc_target(ggml-cpu)
configure_msvc_target(ggml-cpu-x64)
configure_msvc_target(ggml-cpu-sse42)
configure_msvc_target(ggml-cpu-sandybridge)
# __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
# skipping ggml-cpu-ivybridge
# skipping ggml-cpu-piledriver
configure_msvc_target(ggml-cpu-haswell)
configure_msvc_target(ggml-cpu-skylakex)
configure_msvc_target(ggml-cpu-cannonlake)
configure_msvc_target(ggml-cpu-cascadelake)
configure_msvc_target(ggml-cpu-icelake)
# MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!
# https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170
# skipping ggml-cpu-cooperlake
# skipping ggml-cpu-zen4
configure_msvc_target(ggml-cpu-alderlake)
# MSVC doesn't support AMX
# skipping ggml-cpu-sapphirerapids
if (GGML_BUILD_EXAMPLES)
configure_msvc_target(common-ggml)
configure_msvc_target(common)
configure_msvc_target(mnist-common)
configure_msvc_target(mnist-eval)
configure_msvc_target(mnist-train)
configure_msvc_target(gpt-2-ctx)
configure_msvc_target(gpt-2-alloc)
configure_msvc_target(gpt-2-backend)
configure_msvc_target(gpt-2-sched)
configure_msvc_target(gpt-2-quantize)
configure_msvc_target(gpt-2-batched)
configure_msvc_target(gpt-j)
configure_msvc_target(gpt-j-quantize)
configure_msvc_target(magika)
configure_msvc_target(yolov3-tiny)
configure_msvc_target(sam)
configure_msvc_target(simple-ctx)
configure_msvc_target(simple-backend)
endif()
if (GGML_BUILD_TESTS)
configure_msvc_target(test-mul-mat)
configure_msvc_target(test-arange)
configure_msvc_target(test-backend-ops)
configure_msvc_target(test-cont)
configure_msvc_target(test-conv-transpose)
configure_msvc_target(test-conv-transpose-1d)
configure_msvc_target(test-conv1d)
configure_msvc_target(test-conv2d)
configure_msvc_target(test-conv2d-dw)
configure_msvc_target(test-customop)
configure_msvc_target(test-dup)
configure_msvc_target(test-opt)
configure_msvc_target(test-pool)
endif ()
endif()
ggml/CONTRIBUTING.md
+3
View File
@@ -0,0 +1,3 @@
Please use [llama.cpp's contribution guidelines](https://github.com/ggml-org/llama.cpp/blob/master/CONTRIBUTING.md) for this project.
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
ggml/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023-2026 The ggml authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
ggml/README.md
+50
View File
@@ -0,0 +1,50 @@
# ggml
[Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205)
Tensor library for machine learning
***Note that this project is under active development. \
Some of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos***
## Features
- Low-level cross-platform implementation
- Integer quantization support
- Broad hardware support
- Automatic differentiation
- ADAM and L-BFGS optimizers
- No third-party dependencies
- Zero memory allocations during runtime
## Build
```bash
git clone https://github.com/ggml-org/ggml
cd ggml
# install python dependencies in a virtual environment
python3.10 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
# build the examples
mkdir build && cd build
cmake ..
cmake --build . --config Release -j 8
```
## GPT inference (example)
```bash
# run the GPT-2 small 117M model
../examples/gpt-2/download-ggml-model.sh 117M
./bin/gpt-2-backend -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
```
For more information, checkout the corresponding programs in the [examples](examples) folder.
## Resources
- [Introduction to ggml](https://huggingface.co/blog/introduction-to-ggml)
- [The GGUF file format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md)
ggml/ci/run.sh
+395
View File
@@ -0,0 +1,395 @@
#/bin/bash
#
# sample usage:
#
# mkdir tmp
#
# # CPU-only build
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
# # with CUDA support
# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
# # With SYCL support
# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
if [ -z "$2" ]; then
echo "usage: $0 <output-dir> <mnt-dir>"
exit 1
fi
mkdir -p "$1"
mkdir -p "$2"
OUT=$(realpath "$1")
MNT=$(realpath "$2")
rm -v $OUT/*.log
rm -v $OUT/*.exit
rm -v $OUT/*.md
sd=`dirname $0`
cd $sd/../
SRC=`pwd`
CMAKE_EXTRA=""
CTEST_EXTRA=""
if [ ! -z ${GG_BUILD_METAL} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
fi
if [ ! -z ${GG_BUILD_CUDA} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON"
if command -v nvidia-smi >/dev/null 2>&1; then
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
else
echo "Warning: Using fallback CUDA architectures"
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
fi
else
echo "Error: nvidia-smi not found, cannot build with CUDA"
exit 1
fi
fi
if [ ! -z ${GG_BUILD_ROCM} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
exit 1
fi
CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
fi
if [ ! -z ${GG_BUILD_SYCL} ]; then
if [ -z ${ONEAPI_ROOT} ]; then
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
echo "source /opt/intel/oneapi/setvars.sh"
exit 1
fi
# Use only main GPU
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
# Enable sysman for correct memory reporting
export ZES_ENABLE_SYSMAN=1
# to circumvent precision issues on CPY operations
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
fi
if [ ! -z ${GG_BUILD_VULKAN} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
# if on Mac, disable METAL
if [[ "$OSTYPE" == "darwin"* ]]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF"
fi
fi
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
fi
if [ ! -z ${GG_BUILD_MUSA} ]; then
# Use qy1 by default (MTT S80)
MUSA_ARCH=${MUSA_ARCH:-21}
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
fi
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
fi
## helpers
# download a file if it does not exist or if it is outdated
function gg_wget {
local out=$1
local url=$2
local cwd=`pwd`
mkdir -p $out
cd $out
# should not re-download if file is the same
wget -nv -N $url
cd $cwd
}
function gg_printf {
printf -- "$@" >> $OUT/README.md
}
function gg_run {
ci=$1
set -o pipefail
set -x
gg_run_$ci | tee $OUT/$ci.log
cur=$?
echo "$cur" > $OUT/$ci.exit
set +x
set +o pipefail
gg_sum_$ci
ret=$((ret | cur))
}
## ci
# ctest_debug
function gg_run_ctest_debug {
cd ${SRC}
rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
set -e
(time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
(time ctest ${CTEST_EXTRA} --output-on-failure -E "test-opt|test-backend-ops" ) 2>&1 | tee -a $OUT/${ci}-ctest.log
set +e
}
function gg_sum_ctest_debug {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs ctest in debug mode\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
gg_printf '```\n'
gg_printf '\n'
}
# ctest_release
function gg_run_ctest_release {
cd ${SRC}
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
set -e
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
if [ -z $GG_BUILD_LOW_PERF ]; then
(time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
else
(time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
fi
set +e
}
function gg_sum_ctest_release {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs ctest in release mode\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
gg_printf '```\n'
}
# gpt_2
function gg_run_gpt_2 {
cd ${SRC}
gg_wget models-mnt/gpt-2 https://huggingface.co/ggerganov/ggml/resolve/main/ggml-model-gpt-2-117M.bin
cd build-ci-release
set -e
model="../models-mnt/gpt-2/ggml-model-gpt-2-117M.bin"
prompts="../examples/prompts/gpt-2.txt"
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -tt ${prompts} ) 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-sched --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-batched --model ${model} -s 1234 -n 64 -np 8 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
set +e
}
function gg_sum_gpt_2 {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs short GPT-2 text generation\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-tg.log)"
gg_printf '```\n'
}
# TODO: update
## mnist
#
#function gg_run_mnist {
# cd ${SRC}
#
# cd build-ci-release
#
# set -e
#
# mkdir -p models/mnist
# python3 ../examples/mnist/convert-h5-to-ggml.py ../examples/mnist/models/mnist/mnist_model.state_dict
#
# model_f32="./models/mnist/ggml-model-f32.bin"
# samples="../examples/mnist/models/mnist/t10k-images.idx3-ubyte"
#
# # first command runs and exports "mnist.ggml", the second command runs the exported model
#
# (time ./bin/mnist ${model_f32} ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
# (time ./bin/mnist-cpu ./mnist.ggml ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
#
# set +e
#}
#
#function gg_sum_mnist {
# gg_printf '### %s\n\n' "${ci}"
#
# gg_printf 'MNIST\n'
# gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
# gg_printf '```\n'
# gg_printf '%s\n' "$(cat $OUT/${ci}-mnist.log)"
# gg_printf '```\n'
#}
# sam
function gg_run_sam {
cd ${SRC}
gg_wget models-mnt/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth
gg_wget models-mnt/sam/ https://raw.githubusercontent.com/YavorGIvanov/sam.cpp/ceafb7467bff7ec98e0c4f952e58a9eb8fd0238b/img.jpg
cd build-ci-release
set -e
path_models="../models-mnt/sam/"
model_f16="${path_models}/ggml-model-f16.bin"
img_0="${path_models}/img.jpg"
python3 ../examples/sam/convert-pth-to-ggml.py ${path_models}/sam_vit_b_01ec64.pth ${path_models}/ 1
# Test default parameters
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 ) 2>&1 | tee -a $OUT/${ci}-main.log
grep -q "point prompt" $OUT/${ci}-main.log
grep -q "bbox (371, 436), (144, 168)" $OUT/${ci}-main.log ||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
# Test box prompt and single mask output
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 -b 368,144,441,173 -sm) 2>&1 | tee -a $OUT/${ci}-main.log
grep -q "box prompt" $OUT/${ci}-main.log
grep -q "bbox (370, 439), (144, 169)" $OUT/${ci}-main.log ||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
set +e
}
function gg_sum_sam {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Run SAM\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
gg_printf '```\n'
}
# yolo
function gg_run_yolo {
cd ${SRC}
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/yolov3-tiny.weights
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/dog.jpg
cd build-ci-release
cp -r ../examples/yolo/data .
set -e
path_models="../models-mnt/yolo/"
python3 ../examples/yolo/convert-yolov3-tiny.py ${path_models}/yolov3-tiny.weights
(time ./bin/yolov3-tiny -m yolov3-tiny.gguf -i ${path_models}/dog.jpg ) 2>&1 | tee -a $OUT/${ci}-main.log
grep -qE "dog: (55|56|57|58|59)%" $OUT/${ci}-main.log
grep -qE "car: (50|51|52|53|54)%" $OUT/${ci}-main.log
grep -qE "truck: (54|55|56|57|58)%" $OUT/${ci}-main.log
grep -qE "bicycle: (57|58|59|60|61)%" $OUT/${ci}-main.log
set +e
}
function gg_sum_yolo {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Run YOLO\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
gg_printf '```\n'
}
## main
if true ; then
# Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt
rm -rf ${SRC}/models-mnt
mnt_models=${MNT}/models
mkdir -p ${mnt_models}
ln -sfn ${mnt_models} ${SRC}/models-mnt
# Create a fresh python3 venv and enter it
if ! python3 -m venv "$MNT/venv"; then
echo "Error: Failed to create Python virtual environment at $MNT/venv."
exit 1
fi
source "$MNT/venv/bin/activate"
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
fi
ret=0
test $ret -eq 0 && gg_run ctest_debug
test $ret -eq 0 && gg_run ctest_release
test $ret -eq 0 && gg_run gpt_2
#test $ret -eq 0 && gg_run mnist
test $ret -eq 0 && gg_run sam
test $ret -eq 0 && gg_run yolo
if [ -z $GG_BUILD_LOW_PERF ]; then
# run tests meant for low-perf runners
date
fi
cat $OUT/README.md
exit $ret
ggml/cmake/GitVars.cmake
+22
View File
@@ -0,0 +1,22 @@
find_package(Git)
# the commit's SHA1
execute_process(COMMAND
"${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_SHA1
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
# the date of the commit
execute_process(COMMAND
"${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_DATE
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
# the subject of the commit
execute_process(COMMAND
"${GIT_EXECUTABLE}" log -1 --format=%s
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
ggml/cmake/common.cmake
+50
View File
@@ -0,0 +1,50 @@
function(ggml_get_flags CCID CCVER)
set(C_FLAGS "")
set(CXX_FLAGS "")
if (CCID MATCHES "Clang")
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
if (
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
)
list(APPEND C_FLAGS -Wdouble-promotion)
endif()
elseif (CCID STREQUAL "GNU")
set(C_FLAGS -Wdouble-promotion)
set(CXX_FLAGS -Wno-array-bounds)
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
list(APPEND CXX_FLAGS -Wextra-semi)
endif()
endif()
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
endfunction()
function(ggml_get_system_arch)
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
else()
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
endif()
endfunction()
ggml/cmake/ggml-config.cmake.in
+191
View File
@@ -0,0 +1,191 @@
@PACKAGE_INIT@
@GGML_VARIABLES_EXPANDED@
# Find all dependencies before creating any target.
include(CMakeFindDependencyMacro)
find_dependency(Threads)
if (NOT GGML_SHARED_LIB)
set(GGML_CPU_INTERFACE_LINK_LIBRARIES "")
set(GGML_CPU_INTERFACE_LINK_OPTIONS "")
if (APPLE AND GGML_ACCELERATE)
find_library(ACCELERATE_FRAMEWORK Accelerate)
if(NOT ACCELERATE_FRAMEWORK)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK})
endif()
if (GGML_OPENMP_ENABLED)
find_dependency(OpenMP)
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
endif()
if (GGML_CPU_HBM)
find_library(memkind memkind)
if(NOT memkind)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind)
endif()
if (GGML_BLAS)
find_dependency(BLAS)
list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES})
list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS ${BLAS_LINKER_FLAGS})
endif()
if (GGML_CUDA)
set(GGML_CUDA_INTERFACE_LINK_LIBRARIES "")
find_dependency(CUDAToolkit)
if (GGML_STATIC)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cudart_static>)
if (WIN32)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas> $<LINK_ONLY:CUDA::cublasLt>)
else()
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas_static> $<LINK_ONLY:CUDA::cublasLt_static>)
endif()
endif()
if (NOT GGML_CUDA_NO_VMM)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cuda_driver>)
endif()
endif()
if (GGML_METAL)
find_library(FOUNDATION_LIBRARY Foundation)
find_library(METAL_FRAMEWORK Metal)
find_library(METALKIT_FRAMEWORK MetalKit)
if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
set(GGML_METAL_INTERFACE_LINK_LIBRARIES
${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK})
endif()
if (GGML_OPENCL)
find_dependency(OpenCL)
set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:OpenCL::OpenCL>)
endif()
if (GGML_VULKAN)
find_dependency(Vulkan)
set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:Vulkan::Vulkan>)
endif()
if (GGML_HIP)
find_dependency(hip)
find_dependency(hipblas)
find_dependency(rocblas)
set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas)
endif()
if (GGML_SYCL)
set(GGML_SYCL_INTERFACE_LINK_LIBRARIES "")
find_package(DNNL)
if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL")
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl)
endif()
if (WIN32)
find_dependency(IntelSYCL)
find_dependency(MKL)
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
endif()
endif()
endif()
set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@")
set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@")
if(NOT TARGET ggml::ggml)
find_package(Threads REQUIRED)
find_library(GGML_LIBRARY ggml
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
add_library(ggml::ggml UNKNOWN IMPORTED)
set_target_properties(ggml::ggml
PROPERTIES
IMPORTED_LOCATION "${GGML_LIBRARY}")
find_library(GGML_BASE_LIBRARY ggml-base
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
add_library(ggml::ggml-base UNKNOWN IMPORTED)
set_target_properties(ggml::ggml-base
PROPERTIES
IMPORTED_LOCATION "${GGML_BASE_LIBRARY}")
set(_ggml_all_targets "")
if (NOT GGML_BACKEND_DL)
foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS})
string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}")
string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx)
find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
message(STATUS "Found ${${_ggml_backend_pfx}_LIBRARY}")
add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${${_ggml_backend_pfx}_LIBRARY}"
INTERFACE_COMPILE_FEATURES c_std_90
POSITION_INDEPENDENT_CODE ON)
string(REGEX MATCH "^ggml-cpu" is_cpu_variant "${_ggml_backend}")
if(is_cpu_variant)
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_LIBRARIES "${GGML_CPU_INTERFACE_LINK_LIBRARIES}")
if(GGML_CPU_INTERFACE_LINK_OPTIONS)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_OPTIONS "${GGML_CPU_INTERFACE_LINK_OPTIONS}")
endif()
else()
list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_LIBRARIES "${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}")
if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_OPTIONS "${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}")
endif()
endif()
list(APPEND _ggml_all_targets ggml::${_ggml_backend})
endforeach()
endif()
list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base "${_ggml_all_targets}")
set_target_properties(ggml::ggml
PROPERTIES
INTERFACE_LINK_LIBRARIES "${GGML_INTERFACE_LINK_LIBRARIES}")
add_library(ggml::all INTERFACE IMPORTED)
set_target_properties(ggml::all
PROPERTIES
INTERFACE_LINK_LIBRARIES "${_ggml_all_targets}")
endif()
check_required_components(ggml)
ggml/docs/gguf.md
+807
View File
@@ -0,0 +1,807 @@
# GGUF
GGUF is a file format for storing models for inference with GGML and executors based on GGML. GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML.
It is a successor file format to GGML, GGMF and GGJT, and is designed to be unambiguous by containing all the information needed to load a model. It is also designed to be extensible, so that new information can be added to models without breaking compatibility.
For more information about the motivation behind GGUF, see [Historical State of Affairs](#historical-state-of-affairs).
## Specification
GGUF is a format based on the existing GGJT, but makes a few changes to the format to make it more extensible and easier to use. The following features are desired:
- Single-file deployment: they can be easily distributed and loaded, and do not require any external files for additional information.
- Extensible: new features can be added to GGML-based executors/new information can be added to GGUF models without breaking compatibility with existing models.
- `mmap` compatibility: models can be loaded using `mmap` for fast loading and saving.
- Easy to use: models can be easily loaded and saved using a small amount of code, with no need for external libraries, regardless of the language used.
- Full information: all information needed to load a model is contained in the model file, and no additional information needs to be provided by the user.
The key difference between GGJT and GGUF is the use of a key-value structure for the hyperparameters (now referred to as metadata), rather than a list of untyped values. This allows for new metadata to be added without breaking compatibility with existing models, and to annotate the model with additional information that may be useful for inference or for identifying the model.
### GGUF Naming Convention
GGUF follow a naming convention of `<BaseName><SizeLabel><FineTune><Version><Encoding><Type><Shard>.gguf` where each component is delimitated by a `-` if present. Ultimately this is intended to make it easier for humans to at a glance get the most important details of a model. It is not intended to be perfectly parsable in the field due to the diversity of existing gguf filenames.
The components are:
1. **BaseName**: A descriptive name for the model base type or architecture.
- This can be derived from gguf metadata `general.basename` substituting spaces for dashes.
1. **SizeLabel**: Parameter weight class (useful for leader boards) represented as `<expertCount>x<count><scale-prefix>`
- This can be derived from gguf metadata `general.size_label` if available or calculated if missing.
- Rounded decimal point is supported in count with a single letter scale prefix to assist in floating point exponent shown below
- `Q`: Quadrillion parameters.
- `T`: Trillion parameters.
- `B`: Billion parameters.
- `M`: Million parameters.
- `K`: Thousand parameters.
- Additional `-<attributes><count><scale-prefix>` can be appended as needed to indicate other attributes of interest
1. **FineTune**: A descriptive name for the model fine tuning goal (e.g. Chat, Instruct, etc...)
- This can be derived from gguf metadata `general.finetune` substituting spaces for dashes.
1. **Version**: (Optional) Denotes the model version number, formatted as `v<Major>.<Minor>`
- If model is missing a version number then assume `v1.0` (First Public Release)
- This can be derived from gguf metadata `general.version`
1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs.
1. **Type**: Indicates the kind of gguf file and the intended purpose for it
- If missing, then file is by default a typical gguf tensor model file
- `LoRA` : GGUF file is a LoRA adapter
- `vocab` : GGUF file with only vocab data and metadata
1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `<ShardNum>-of-<ShardTotal>`.
- *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros.
- Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`).
- *ShardTotal* : Total number of shards in this model. Must be 5 digits padded by zeros.
#### Validating Above Naming Convention
At a minimum all model files should have at least BaseName, SizeLabel, Version, in order to be easily validated as a file that is keeping with the GGUF Naming Convention. An example of this issue is that it is easy for Encoding to be mistaken as a FineTune if Version is omitted.
To validate you can use this regular expression `^(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$` which will check that you got the minimum BaseName, SizeLabel and Version present in the correct order.
For example:
* `Mixtral-8x7B-v0.1-KQ2.gguf`:
- Model Name: Mixtral
- Expert Count: 8
- Parameter Count: 7B
- Version Number: v0.1
- Weight Encoding Scheme: KQ2
* `Hermes-2-Pro-Llama-3-8B-F16.gguf`:
- Model Name: Hermes 2 Pro Llama 3
- Expert Count: 0
- Parameter Count: 8B
- Version Number: v1.0
- Weight Encoding Scheme: F16
- Shard: N/A
* `Grok-100B-v1.0-Q4_0-00003-of-00009.gguf`
- Model Name: Grok
- Expert Count: 0
- Parameter Count: 100B
- Version Number: v1.0
- Weight Encoding Scheme: Q4_0
- Shard: 3 out of 9 total shards
<details><summary>Example Node.js Regex Function</summary>
```js
#!/usr/bin/env node
const ggufRegex = /^(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$/;
function parseGGUFFilename(filename) {
const match = ggufRegex.exec(filename);
if (!match)
return null;
const {BaseName = null, SizeLabel = null, FineTune = null, Version = "v1.0", Encoding = null, Type = null, Shard = null} = match.groups;
return {BaseName: BaseName, SizeLabel: SizeLabel, FineTune: FineTune, Version: Version, Encoding: Encoding, Type: Type, Shard: Shard};
}
const testCases = [
{filename: 'Mixtral-8x7B-v0.1-KQ2.gguf', expected: { BaseName: 'Mixtral', SizeLabel: '8x7B', FineTune: null, Version: 'v0.1', Encoding: 'KQ2', Type: null, Shard: null}},
{filename: 'Grok-100B-v1.0-Q4_0-00003-of-00009.gguf', expected: { BaseName: 'Grok', SizeLabel: '100B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_0', Type: null, Shard: "00003-of-00009"}},
{filename: 'Hermes-2-Pro-Llama-3-8B-v1.0-F16.gguf', expected: { BaseName: 'Hermes-2-Pro-Llama-3', SizeLabel: '8B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},
{filename: 'Phi-3-mini-3.8B-ContextLength4k-instruct-v1.0.gguf', expected: { BaseName: 'Phi-3-mini', SizeLabel: '3.8B-ContextLength4k', FineTune: 'instruct', Version: 'v1.0', Encoding: null, Type: null, Shard: null}},
{filename: 'not-a-known-arrangement.gguf', expected: null},
];
testCases.forEach(({ filename, expected }) => {
const result = parseGGUFFilename(filename);
const passed = JSON.stringify(result) === JSON.stringify(expected);
console.log(`${filename}: ${passed ? "PASS" : "FAIL"}`);
if (!passed) {
console.log(result);
console.log(expected);
}
});
```
</details>
### File Structure
![image](https://github.com/ggerganov/ggml/assets/1991296/c3623641-3a1d-408e-bfaf-1b7c4e16aa63)
*diagram by [@mishig25](https://github.com/mishig25) (GGUF v3)*
GGUF files are structured as follows. They use a global alignment specified in the `general.alignment` metadata field, referred to as `ALIGNMENT` below. Where required, the file is padded with `0x00` bytes to the next multiple of `general.alignment`.
Fields, including arrays, are written sequentially without alignment unless otherwise specified.
Models are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian.
```c
enum ggml_type: uint32_t {
GGML_TYPE_F32 = 0,
GGML_TYPE_F16 = 1,
GGML_TYPE_Q4_0 = 2,
GGML_TYPE_Q4_1 = 3,
// GGML_TYPE_Q4_2 = 4, support has been removed
// GGML_TYPE_Q4_3 = 5, support has been removed
GGML_TYPE_Q5_0 = 6,
GGML_TYPE_Q5_1 = 7,
GGML_TYPE_Q8_0 = 8,
GGML_TYPE_Q8_1 = 9,
GGML_TYPE_Q2_K = 10,
GGML_TYPE_Q3_K = 11,
GGML_TYPE_Q4_K = 12,
GGML_TYPE_Q5_K = 13,
GGML_TYPE_Q6_K = 14,
GGML_TYPE_Q8_K = 15,
GGML_TYPE_IQ2_XXS = 16,
GGML_TYPE_IQ2_XS = 17,
GGML_TYPE_IQ3_XXS = 18,
GGML_TYPE_IQ1_S = 19,
GGML_TYPE_IQ4_NL = 20,
GGML_TYPE_IQ3_S = 21,
GGML_TYPE_IQ2_S = 22,
GGML_TYPE_IQ4_XS = 23,
GGML_TYPE_I8 = 24,
GGML_TYPE_I16 = 25,
GGML_TYPE_I32 = 26,
GGML_TYPE_I64 = 27,
GGML_TYPE_F64 = 28,
GGML_TYPE_IQ1_M = 29,
GGML_TYPE_BF16 = 30,
// GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
// GGML_TYPE_Q4_0_4_8 = 32,
// GGML_TYPE_Q4_0_8_8 = 33,
GGML_TYPE_TQ1_0 = 34,
GGML_TYPE_TQ2_0 = 35,
// GGML_TYPE_IQ4_NL_4_4 = 36,
// GGML_TYPE_IQ4_NL_4_8 = 37,
// GGML_TYPE_IQ4_NL_8_8 = 38,
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
GGML_TYPE_COUNT = 40,
};
enum gguf_metadata_value_type: uint32_t {
// The value is a 8-bit unsigned integer.
GGUF_METADATA_VALUE_TYPE_UINT8 = 0,
// The value is a 8-bit signed integer.
GGUF_METADATA_VALUE_TYPE_INT8 = 1,
// The value is a 16-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT16 = 2,
// The value is a 16-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT16 = 3,
// The value is a 32-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT32 = 4,
// The value is a 32-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT32 = 5,
// The value is a 32-bit IEEE754 floating point number.
GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6,
// The value is a boolean.
// 1-byte value where 0 is false and 1 is true.
// Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.
GGUF_METADATA_VALUE_TYPE_BOOL = 7,
// The value is a UTF-8 non-null-terminated string, with length prepended.
GGUF_METADATA_VALUE_TYPE_STRING = 8,
// The value is an array of other values, with the length and type prepended.
///
// Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.
GGUF_METADATA_VALUE_TYPE_ARRAY = 9,
// The value is a 64-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT64 = 10,
// The value is a 64-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT64 = 11,
// The value is a 64-bit IEEE754 floating point number.
GGUF_METADATA_VALUE_TYPE_FLOAT64 = 12,
};
// A string in GGUF.
struct gguf_string_t {
// The length of the string, in bytes.
uint64_t len;
// The string as a UTF-8 non-null-terminated string.
char string[len];
};
union gguf_metadata_value_t {
uint8_t uint8;
int8_t int8;
uint16_t uint16;
int16_t int16;
uint32_t uint32;
int32_t int32;
float float32;
uint64_t uint64;
int64_t int64;
double float64;
bool bool_;
gguf_string_t string;
struct {
// Any value type is valid, including arrays.
gguf_metadata_value_type type;
// Number of elements, not bytes
uint64_t len;
// The array of values.
gguf_metadata_value_t array[len];
} array;
};
struct gguf_metadata_kv_t {
// The key of the metadata. It is a standard GGUF string, with the following caveats:
// - It must be a valid ASCII string.
// - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`.
// - It must be at most 2^16-1/65535 bytes long.
// Any keys that do not follow these rules are invalid.
gguf_string_t key;
// The type of the value.
// Must be one of the `gguf_metadata_value_type` values.
gguf_metadata_value_type value_type;
// The value.
gguf_metadata_value_t value;
};
struct gguf_header_t {
// Magic number to announce that this is a GGUF file.
// Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`.
// Your executor might do little-endian byte order, so it might be
// check for 0x46554747 and letting the endianness cancel out.
// Consider being *very* explicit about the byte order here.
uint32_t magic;
// The version of the format implemented.
// Must be `3` for version described in this spec, which introduces big-endian support.
//
// This version should only be increased for structural changes to the format.
// Changes that do not affect the structure of the file should instead update the metadata
// to signify the change.
uint32_t version;
// The number of tensors in the file.
// This is explicit, instead of being included in the metadata, to ensure it is always present
// for loading the tensors.
uint64_t tensor_count;
// The number of metadata key-value pairs.
uint64_t metadata_kv_count;
// The metadata key-value pairs.
gguf_metadata_kv_t metadata_kv[metadata_kv_count];
};
uint64_t align_offset(uint64_t offset) {
return offset + (ALIGNMENT - (offset % ALIGNMENT)) % ALIGNMENT;
}
struct gguf_tensor_info_t {
// The name of the tensor. It is a standard GGUF string, with the caveat that
// it must be at most 64 bytes long.
gguf_string_t name;
// The number of dimensions in the tensor.
// Currently at most 4, but this may change in the future.
uint32_t n_dimensions;
// The dimensions of the tensor.
uint64_t dimensions[n_dimensions];
// The type of the tensor.
ggml_type type;
// The offset of the tensor's data in this file in bytes.
//
// This offset is relative to `tensor_data`, not to the start
// of the file, to make it easier for writers to write the file.
// Readers should consider exposing this offset relative to the
// file to make it easier to read the data.
//
// Must be a multiple of `ALIGNMENT`. That is, `align_offset(offset) == offset`.
uint64_t offset;
};
struct gguf_file_t {
// The header of the file.
gguf_header_t header;
// Tensor infos, which can be used to locate the tensor data.
gguf_tensor_info_t tensor_infos[header.tensor_count];
// Padding to the nearest multiple of `ALIGNMENT`.
//
// That is, if `sizeof(header) + sizeof(tensor_infos)` is not a multiple of `ALIGNMENT`,
// this padding is added to make it so.
//
// This can be calculated as `align_offset(position) - position`, where `position` is
// the position of the end of `tensor_infos` (i.e. `sizeof(header) + sizeof(tensor_infos)`).
uint8_t _padding[];
// Tensor data.
//
// This is arbitrary binary data corresponding to the weights of the model. This data should be close
// or identical to the data in the original model file, but may be different due to quantization or
// other optimizations for inference. Any such deviations should be recorded in the metadata or as
// part of the architecture definition.
//
// Each tensor's data must be stored within this array, and located through its `tensor_infos` entry.
// The offset of each tensor's data must be a multiple of `ALIGNMENT`, and the space between tensors
// should be padded to `ALIGNMENT` bytes.
uint8_t tensor_data[];
};
```
## Standardized key-value pairs
The following key-value pairs are standardized. This list may grow in the future as more use cases are discovered. Where possible, names are shared with the original model definitions to make it easier to map between the two.
Not all of these are required, but they are all recommended. Keys that are required are bolded. For omitted pairs, the reader should assume that the value is unknown and either default or error as appropriate.
The community can develop their own key-value pairs to carry additional data. However, these should be namespaced with the relevant community name to avoid collisions. For example, the `rustformers` community might use `rustformers.` as a prefix for all of their keys.
If a particular community key is widely used, it may be promoted to a standardized key.
By convention, most counts/lengths/etc are `uint64` unless otherwise specified. This is to allow for larger models to be supported in the future. Some models may use `uint32` for their values; it is recommended that readers support both.
### General
#### Required
- **`general.architecture: string`**: describes what architecture this model implements. All lowercase ASCII, with only `[a-z0-9]+` characters allowed. Known values include:
- `llama`
- `mpt`
- `gptneox`
- `gptj`
- `gpt2`
- `bloom`
- `falcon`
- `mamba`
- `rwkv`
- **`general.quantization_version: uint32`**: The version of the quantization format. Not required if the model is not quantized (i.e. no tensors are quantized). If any tensors are quantized, this _must_ be present. This is separate to the quantization scheme of the tensors itself; the quantization version may change without changing the scheme's name (e.g. the quantization scheme is Q5_K, and the quantization version is 4).
- **`general.alignment: uint32`**: the global alignment to use, as described above. This can vary to allow for different alignment schemes, but it must be a multiple of 8. Some writers may not write the alignment. If the alignment is **not** specified, assume it is `32`.
#### General metadata
- `general.name: string`: The name of the model. This should be a human-readable name that can be used to identify the model. It should be unique within the community that the model is defined in.
- `general.author: string`: The author of the model.
- `general.version: string`: The version of the model.
- `general.organization: string`: The organization of the model.
- `general.basename: string`: The base model name / architecture of the model
- `general.finetune: string`: What has the base model been optimized toward.
- `general.description: string`: free-form description of the model including anything that isn't covered by the other fields
- `general.quantized_by: string`: The name of the individual who quantized the model
- `general.size_label: string`: Size class of the model, such as number of weights and experts. (Useful for leader boards)
- `general.license: string`: License of the model, expressed as a [SPDX license expression](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/) (e.g. `"MIT OR Apache-2.0`). Do not include any other information, such as the license text or the URL to the license.
- `general.license.name: string`: Human friendly license name
- `general.license.link: string`: URL to the license.
- `general.url: string`: URL to the model's homepage. This can be a GitHub repo, a paper, etc.
- `general.doi: string`: Digital Object Identifier (DOI) https://www.doi.org/
- `general.uuid: string`: [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.repo_url: string`: URL to the model's repository such as a GitHub repo or HuggingFace repo
- `general.tags: string[]`: List of tags that can be used as search terms for a search engine or social media
- `general.languages: string[]`: What languages can the model speak. Encoded as [ISO 639](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) two letter codes
- `general.datasets: string[]`: Links or references to datasets that the model was trained upon
- `general.file_type: uint32`: An enumerated value describing the type of the majority of the tensors in the file. Optional; can be inferred from the tensor types.
- `ALL_F32 = 0`
- `MOSTLY_F16 = 1`
- `MOSTLY_Q4_0 = 2`
- `MOSTLY_Q4_1 = 3`
- `MOSTLY_Q4_1_SOME_F16 = 4`
- `MOSTLY_Q4_2 = 5` (support removed)
- `MOSTLY_Q4_3 = 6` (support removed)
- `MOSTLY_Q8_0 = 7`
- `MOSTLY_Q5_0 = 8`
- `MOSTLY_Q5_1 = 9`
- `MOSTLY_Q2_K = 10`
- `MOSTLY_Q3_K_S = 11`
- `MOSTLY_Q3_K_M = 12`
- `MOSTLY_Q3_K_L = 13`
- `MOSTLY_Q4_K_S = 14`
- `MOSTLY_Q4_K_M = 15`
- `MOSTLY_Q5_K_S = 16`
- `MOSTLY_Q5_K_M = 17`
- `MOSTLY_Q6_K = 18`
#### Source metadata
Information about where this model came from. This is useful for tracking the provenance of the model, and for finding the original source if the model is modified. For a model that was converted from GGML, for example, these keys would point to the model that was converted from.
- `general.source.url: string`: URL to the source of the model's homepage. This can be a GitHub repo, a paper, etc.
- `general.source.doi: string`: Source Digital Object Identifier (DOI) https://www.doi.org/
- `general.source.uuid: string`: Source [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.source.repo_url: string`: URL to the source of the model's repository such as a GitHub repo or HuggingFace repo
- `general.base_model.count: uint32`: Number of parent models
- `general.base_model.{id}.name: string`: The name of the parent model.
- `general.base_model.{id}.author: string`: The author of the parent model.
- `general.base_model.{id}.version: string`: The version of the parent model.
- `general.base_model.{id}.organization: string`: The organization of the parent model.
- `general.base_model.{id}.url: string`: URL to the source of the parent model's homepage. This can be a GitHub repo, a paper, etc.
- `general.base_model.{id}.doi: string`: Parent Digital Object Identifier (DOI) https://www.doi.org/
- `general.base_model.{id}.uuid: string`: Parent [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.base_model.{id}.repo_url: string`: URL to the source of the parent model's repository such as a GitHub repo or HuggingFace repo
### LLM
In the following, `[llm]` is used to fill in for the name of a specific LLM architecture. For example, `llama` for LLaMA, `mpt` for MPT, etc. If mentioned in an architecture's section, it is required for that architecture, but not all keys are required for all architectures. Consult the relevant section for more information.
- `[llm].context_length: uint64`: Also known as `n_ctx`. length of the context (in tokens) that the model was trained on. For most architectures, this is the hard limit on the length of the input. Architectures, like RWKV, that are not reliant on transformer-style attention may be able to handle larger inputs, but this is not guaranteed.
- `[llm].embedding_length: uint64`: Also known as `n_embd`. Embedding layer size.
- `[llm].block_count: uint64`: The number of blocks of attention+feed-forward layers (i.e. the bulk of the LLM). Does not include the input or embedding layers.
- `[llm].feed_forward_length: uint64`: Also known as `n_ff`. The length of the feed-forward layer.
- `[llm].use_parallel_residual: bool`: Whether or not the parallel residual logic should be used.
- `[llm].tensor_data_layout: string`: When a model is converted to GGUF, tensors may be rearranged to improve performance. This key describes the layout of the tensor data. This is not required; if not present, it is assumed to be `reference`.
- `reference`: tensors are laid out in the same order as the original model
- further options can be found for each architecture in their respective sections
- `[llm].expert_count: uint32`: Number of experts in MoE models (optional for non-MoE arches).
- `[llm].expert_used_count: uint32`: Number of experts used during each token token evaluation (optional for non-MoE arches).
#### Attention
- `[llm].attention.head_count: uint64`: Also known as `n_head`. Number of attention heads.
- `[llm].attention.head_count_kv: uint64`: The number of heads per group used in Grouped-Query-Attention. If not present or if present and equal to `[llm].attention.head_count`, the model does not use GQA.
- `[llm].attention.max_alibi_bias: float32`: The maximum bias to use for ALiBI.
- `[llm].attention.clamp_kqv: float32`: Value (`C`) to clamp the values of the `Q`, `K`, and `V` tensors between (`[-C, C]`).
- `[llm].attention.layer_norm_epsilon: float32`: Layer normalization epsilon.
- `[llm].attention.layer_norm_rms_epsilon: float32`: Layer RMS normalization epsilon.
- `[llm].attention.key_length: uint32`: The optional size of a key head, $d_k$. If not specified, it will be `n_embd / n_head`.
- `[llm].attention.value_length: uint32`: The optional size of a value head, $d_v$. If not specified, it will be `n_embd / n_head`.
#### RoPE
- `[llm].rope.dimension_count: uint64`: The number of rotary dimensions for RoPE.
- `[llm].rope.freq_base: float32`: The base frequency for RoPE.
##### Scaling
The following keys describe RoPE scaling parameters:
- `[llm].rope.scaling.type: string`: Can be `none`, `linear`, or `yarn`.
- `[llm].rope.scaling.factor: float32`: A scale factor for RoPE to adjust the context length.
- `[llm].rope.scaling.original_context_length: uint32_t`: The original context length of the base model.
- `[llm].rope.scaling.finetuned: bool`: True if model has been finetuned with RoPE scaling.
Note that older models may not have these keys, and may instead use the following key:
- `[llm].rope.scale_linear: float32`: A linear scale factor for RoPE to adjust the context length.
It is recommended that models use the newer keys if possible, as they are more flexible and allow for more complex scaling schemes. Executors will need to support both indefinitely.
#### SSM
- `[llm].ssm.conv_kernel: uint32`: The size of the rolling/shift state.
- `[llm].ssm.inner_size: uint32`: The embedding size of the states.
- `[llm].ssm.state_size: uint32`: The size of the recurrent state.
- `[llm].ssm.time_step_rank: uint32`: The rank of time steps.
#### Models
The following sections describe the metadata for each model architecture. Each key specified _must_ be present.
##### LLaMA
- `llama.context_length`
- `llama.embedding_length`
- `llama.block_count`
- `llama.feed_forward_length`
- `llama.rope.dimension_count`
- `llama.attention.head_count`
- `llama.attention.layer_norm_rms_epsilon`
###### Optional
- `llama.rope.scale`
- `llama.attention.head_count_kv`
- `llama.tensor_data_layout`:
- `Meta AI original pth`:
```python
def permute(weights: NDArray, n_head: int) -> NDArray:
return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
.swapaxes(1, 2)
.reshape(weights.shape))
```
- `llama.expert_count`
- `llama.expert_used_count`
##### MPT
- `mpt.context_length`
- `mpt.embedding_length`
- `mpt.block_count`
- `mpt.attention.head_count`
- `mpt.attention.alibi_bias_max`
- `mpt.attention.clip_kqv`
- `mpt.attention.layer_norm_epsilon`
##### GPT-NeoX
- `gptneox.context_length`
- `gptneox.embedding_length`
- `gptneox.block_count`
- `gptneox.use_parallel_residual`
- `gptneox.rope.dimension_count`
- `gptneox.attention.head_count`
- `gptneox.attention.layer_norm_epsilon`
###### Optional
- `gptneox.rope.scale`
##### GPT-J
- `gptj.context_length`
- `gptj.embedding_length`
- `gptj.block_count`
- `gptj.rope.dimension_count`
- `gptj.attention.head_count`
- `gptj.attention.layer_norm_epsilon`
###### Optional
- `gptj.rope.scale`
##### GPT-2
- `gpt2.context_length`
- `gpt2.embedding_length`
- `gpt2.block_count`
- `gpt2.attention.head_count`
- `gpt2.attention.layer_norm_epsilon`
##### BLOOM
- `bloom.context_length`
- `bloom.embedding_length`
- `bloom.block_count`
- `bloom.feed_forward_length`
- `bloom.attention.head_count`
- `bloom.attention.layer_norm_epsilon`
##### Falcon
- `falcon.context_length`
- `falcon.embedding_length`
- `falcon.block_count`
- `falcon.attention.head_count`
- `falcon.attention.head_count_kv`
- `falcon.attention.use_norm`
- `falcon.attention.layer_norm_epsilon`
###### Optional
- `falcon.tensor_data_layout`:
- `jploski` (author of the original GGML implementation of Falcon):
```python
# The original query_key_value tensor contains n_head_kv "kv groups",
# each consisting of n_head/n_head_kv query weights followed by one key
# and one value weight (shared by all query heads in the kv group).
# This layout makes it a big pain to work with in GGML.
# So we rearrange them here,, so that we have n_head query weights
# followed by n_head_kv key weights followed by n_head_kv value weights,
# in contiguous fashion.
if "query_key_value" in src:
qkv = model[src].view(
n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
model[src] = torch.cat((q,k,v)).reshape_as(model[src])
```
##### Mamba
- `mamba.context_length`
- `mamba.embedding_length`
- `mamba.block_count`
- `mamba.ssm.conv_kernel`
- `mamba.ssm.inner_size`
- `mamba.ssm.state_size`
- `mamba.ssm.time_step_rank`
- `mamba.attention.layer_norm_rms_epsilon`
##### RWKV
The vocabulary size is the same as the number of rows in the `head` matrix.
- `rwkv.architecture_version: uint32`: The only allowed value currently is 4. Version 5 is expected to appear some time in the future.
- `rwkv.context_length: uint64`: Length of the context used during training or fine-tuning. RWKV is able to handle larger context than this limit, but the output quality may suffer.
- `rwkv.block_count: uint64`
- `rwkv.embedding_length: uint64`
- `rwkv.feed_forward_length: uint64`
##### Whisper
Keys that do not have types defined should be assumed to share definitions with `llm.` keys.
(For example, `whisper.context_length` is equivalent to `llm.context_length`.)
This is because they are both transformer models.
- `whisper.encoder.context_length`
- `whisper.encoder.embedding_length`
- `whisper.encoder.block_count`
- `whisper.encoder.mels_count: uint64`
- `whisper.encoder.attention.head_count`
- `whisper.decoder.context_length`
- `whisper.decoder.embedding_length`
- `whisper.decoder.block_count`
- `whisper.decoder.attention.head_count`
#### Prompting
**TODO**: Include prompt format, and/or metadata about how it should be used (instruction, conversation, autocomplete, etc).
### LoRA
**TODO**: Figure out what metadata is needed for LoRA. Probably desired features:
- match an existing model exactly, so that it can't be misapplied
- be marked as a LoRA so executors won't try to run it by itself
Should this be an architecture, or should it share the details of the original model with additional fields to mark it as a LoRA?
### Tokenizer
The following keys are used to describe the tokenizer of the model. It is recommended that model authors support as many of these as possible, as it will allow for better tokenization quality with supported executors.
#### GGML
GGML supports an embedded vocabulary that enables inference of the model, but implementations of tokenization using this vocabulary (i.e. `llama.cpp`'s tokenizer) may have lower accuracy than the original tokenizer used for the model. When a more accurate tokenizer is available and supported, it should be used instead.
It is not guaranteed to be standardized across models, and may change in the future. It is recommended that model authors use a more standardized tokenizer if possible.
- `tokenizer.ggml.model: string`: The name of the tokenizer model.
- `llama`: Llama style SentencePiece (tokens and scores extracted from HF `tokenizer.model`)
- `replit`: Replit style SentencePiece (tokens and scores extracted from HF `spiece.model`)
- `gpt2`: GPT-2 / GPT-NeoX style BPE (tokens extracted from HF `tokenizer.json`)
- `rwkv`: RWKV tokenizer
- `tokenizer.ggml.tokens: array[string]`: A list of tokens indexed by the token ID used by the model.
- `tokenizer.ggml.scores: array[float32]`: If present, the score/probability of each token. If not present, all tokens are assumed to have equal probability. If present, it must have the same length and index as `tokens`.
- `tokenizer.ggml.token_type: array[int32]`: The token type (1=normal, 2=unknown, 3=control, 4=user defined, 5=unused, 6=byte). If present, it must have the same length and index as `tokens`.
- `tokenizer.ggml.merges: array[string]`: If present, the merges of the tokenizer. If not present, the tokens are assumed to be atomic.
- `tokenizer.ggml.added_tokens: array[string]`: If present, tokens that were added after training.
##### Special tokens
- `tokenizer.ggml.bos_token_id: uint32`: Beginning of sequence marker
- `tokenizer.ggml.eos_token_id: uint32`: End of sequence marker
- `tokenizer.ggml.unknown_token_id: uint32`: Unknown token
- `tokenizer.ggml.separator_token_id: uint32`: Separator token
- `tokenizer.ggml.padding_token_id: uint32`: Padding token
#### Hugging Face
Hugging Face maintains their own `tokenizers` library that supports a wide variety of tokenizers. If your executor uses this library, it may be able to use the model's tokenizer directly.
- `tokenizer.huggingface.json: string`: the entirety of the HF `tokenizer.json` for a given model (e.g. <https://huggingface.co/mosaicml/mpt-7b-instruct/blob/main/tokenizer.json>). Included for compatibility with executors that support HF tokenizers directly.
#### Other
Other tokenizers may be used, but are not necessarily standardized. They may be executor-specific. They will be documented here as they are discovered/further developed.
- `tokenizer.rwkv.world: string`: a RWKV World tokenizer, like [this](https://github.com/BlinkDL/ChatRWKV/blob/main/tokenizer/rwkv_vocab_v20230424.txt). This text file should be included verbatim.
- `tokenizer.chat_template : string`: a Jinja template that specifies the input format expected by the model. For more details see: <https://huggingface.co/docs/transformers/main/en/chat_templating>
### Computation graph
This is a future extension and still needs to be discussed, and may necessitate a new GGUF version. At the time of writing, the primary blocker is the stabilization of the computation graph format.
A sample computation graph of GGML nodes could be included in the model itself, allowing an executor to run the model without providing its own implementation of the architecture. This would allow for a more consistent experience across executors, and would allow for more complex architectures to be supported without requiring the executor to implement them.
## Standardized tensor names
To minimize complexity and maximize compatibility, it is recommended that models using the transformer architecture use the following naming convention for their tensors:
### Base layers
`AA.weight` `AA.bias`
where `AA` can be:
- `token_embd`: Token embedding layer
- `pos_embd`: Position embedding layer
- `output_norm`: Output normalization layer
- `output`: Output layer
### Attention and feed-forward layer blocks
`blk.N.BB.weight` `blk.N.BB.bias`
where N signifies the block number a layer belongs to, and where `BB` could be:
- `attn_norm`: Attention normalization layer
- `attn_norm_2`: Attention normalization layer
- `attn_qkv`: Attention query-key-value layer
- `attn_q`: Attention query layer
- `attn_k`: Attention key layer
- `attn_v`: Attention value layer
- `attn_output`: Attention output layer
- `ffn_norm`: Feed-forward network normalization layer
- `ffn_up`: Feed-forward network "up" layer
- `ffn_gate`: Feed-forward network "gate" layer
- `ffn_down`: Feed-forward network "down" layer
- `ffn_gate_inp`: Expert-routing layer for the Feed-forward network in MoE models
- `ffn_gate_exp`: Feed-forward network "gate" layer per expert in MoE models
- `ffn_down_exp`: Feed-forward network "down" layer per expert in MoE models
- `ffn_up_exp`: Feed-forward network "up" layer per expert in MoE models
- `ssm_in`: State space model input projections layer
- `ssm_conv1d`: State space model rolling/shift layer
- `ssm_x`: State space model selective parametrization layer
- `ssm_a`: State space model state compression layer
- `ssm_d`: State space model skip connection layer
- `ssm_dt`: State space model time step layer
- `ssm_out`: State space model output projection layer
## Version History
This document is actively updated to describe the current state of the metadata, and these changes are not tracked outside of the commits.
However, the format _itself_ has changed. The following sections describe the changes to the format itself.
### v3
Adds big-endian support.
### v2
Most countable values (lengths, etc) were changed from `uint32` to `uint64` to allow for larger models to be supported in the future.
### v1
Initial version.
## Historical State of Affairs
The following information is provided for context, but is not necessary to understand the rest of this document.
### Overview
At present, there are three GGML file formats floating around for LLMs:
- **GGML** (unversioned): baseline format, with no versioning or alignment.
- **GGMF** (versioned): the same as GGML, but with versioning. Only one version exists.
- **GGJT**: Aligns the tensors to allow for use with `mmap`, which requires alignment. v1, v2 and v3 are identical, but the latter versions use a different quantization scheme that is incompatible with previous versions.
GGML is primarily used by the examples in `ggml`, while GGJT is used by `llama.cpp` models. Other executors may use any of the three formats, but this is not 'officially' supported.
These formats share the same fundamental structure:
- a magic number with an optional version number
- model-specific hyperparameters, including
- metadata about the model, such as the number of layers, the number of heads, etc.
- a `ftype` that describes the type of the majority of the tensors,
- for GGML files, the quantization version is encoded in the `ftype` divided by 1000
- an embedded vocabulary, which is a list of strings with length prepended. The GGMF/GGJT formats embed a float32 score next to the strings.
- finally, a list of tensors with their length-prepended name, type, and (aligned, in the case of GGJT) tensor data
Notably, this structure does not identify what model architecture the model belongs to, nor does it offer any flexibility for changing the structure of the hyperparameters. This means that the only way to add new hyperparameters is to add them to the end of the list, which is a breaking change for existing models.
### Drawbacks
Unfortunately, over the last few months, there are a few issues that have become apparent with the existing models:
- There's no way to identify which model architecture a given model is for, because that information isn't present
- Similarly, existing programs cannot intelligently fail upon encountering new architectures
- Adding or removing any new hyperparameters is a breaking change, which is impossible for a reader to detect without using heuristics
- Each model architecture requires its own conversion script to their architecture's variant of GGML
- Maintaining backwards compatibility without breaking the structure of the format requires clever tricks, like packing the quantization version into the ftype, which are not guaranteed to be picked up by readers/writers, and are not consistent between the two formats
### Why not other formats?
There are a few other formats that could be used, but issues include:
- requiring additional dependencies to load or save the model, which is complicated in a C environment
- limited or no support for 4-bit quantization
- existing cultural expectations (e.g. whether or not the model is a directory or a file)
- lack of support for embedded vocabularies
- lack of control over direction of future development
Ultimately, it is likely that GGUF will remain necessary for the foreseeable future, and it is better to have a single format that is well-documented and supported by all executors than to contort an existing format to fit the needs of GGML.
ggml/docs/metal-conv-transpose-2d.md
+62
View File
@@ -0,0 +1,62 @@
# ggml `CONV_TRANSPOSE_2D` semantics
Source of truth: [`ggml_compute_forward_conv_transpose_2d()`](../src/ggml-cpu/ops.cpp).
This note documents the exact contract the Metal kernel mirrors.
## Tensor layouts
- `src0` weights: `(KW, KH, Cout, Cin)`, contiguous, `F16` in the CPU implementation used by SAM.
- `src1` input: `(IW, IH, Cin, N)`, contiguous, `F32`.
- `dst` output: `(OW, OH, Cout, N)`, `F32`.
For `ggml_conv_transpose_2d_p0()` the output size is:
- `OW = (IW - 1) * stride + KW`
- `OH = (IH - 1) * stride + KH`
The `p0` variant used here has:
- zero padding only
- one shared spatial stride `s0`
- no dilation
- no output padding
- no bias term
## CPU computation
The CPU implementation permutes weights and inputs into temporary buffers, zeros `dst`,
then performs a scatter-add over output channel, input spatial position, and kernel tap:
```text
for out_c in [0, Cout):
for in_y in [0, IH):
for in_x in [0, IW):
for kh in [0, KH):
for kw in [0, KW):
dst[in_x * stride + kw, in_y * stride + kh, out_c] +=
dot(input[in_x, in_y, :, 0], weight[kw, kh, out_c, :, 0])
```
The Metal reference kernel uses the equivalent gather form for one output element:
```text
dst[ow, oh, out_c] =
sum over kh, kw, in_c where
ow = in_x * stride + kw
oh = in_y * stride + kh
of weight[kw, kh, out_c, in_c] * input[in_x, in_y, in_c]
```
## Numeric behavior
- input source type: `F32`, but the CPU implementation first packs it to `F16`
- weight type: `F16` or `F32` on Metal, `F16` in the CPU path used here
- multiply inputs: `F16 x F16` in the reference CPU path
- accumulation type: `float`
- output type: `F32`
## Scope
The current CPU implementation only uses batch `N = 1` in practice for this path, and
the Metal implementation matches that contract explicitly.
ggml/examples/CMakeLists.txt
+34
View File
@@ -0,0 +1,34 @@
if (GGML_ALL_WARNINGS)
if (NOT MSVC)
set(cxx_flags
# TODO(marella): Add other warnings.
-Wpedantic
-Wunused-variable
-Wno-unused-function
-Wno-multichar
)
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${cxx_flags}>")
endif()
endif()
add_library(common STATIC common.cpp)
target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_library(common-ggml STATIC common-ggml.cpp)
target_link_libraries(common-ggml PRIVATE ggml)
target_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(yolo)
if (NOT GGML_BACKEND_DL)
add_subdirectory(gpt-2)
add_subdirectory(gpt-j)
add_subdirectory(mnist)
add_subdirectory(sam)
add_subdirectory(simple)
add_subdirectory(magika)
endif()
if (GGML_METAL)
add_subdirectory(perf-metal)
endif()
ggml/examples/common-ggml.cpp
+242
View File
@@ -0,0 +1,242 @@
#include "common-ggml.h"
#include <regex>
#include <map>
static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {
{"q4_0", GGML_FTYPE_MOSTLY_Q4_0},
{"q4_1", GGML_FTYPE_MOSTLY_Q4_1},
{"q5_0", GGML_FTYPE_MOSTLY_Q5_0},
{"q5_1", GGML_FTYPE_MOSTLY_Q5_1},
{"q8_0", GGML_FTYPE_MOSTLY_Q8_0},
{"q2_k", GGML_FTYPE_MOSTLY_Q2_K},
{"q3_k", GGML_FTYPE_MOSTLY_Q3_K},
{"q4_k", GGML_FTYPE_MOSTLY_Q4_K},
{"q5_k", GGML_FTYPE_MOSTLY_Q5_K},
{"q6_k", GGML_FTYPE_MOSTLY_Q6_K},
};
void ggml_print_ftypes(FILE * fp) {
for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {
fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second);
}
}
enum ggml_ftype ggml_parse_ftype(const char * str) {
enum ggml_ftype ftype;
if (str[0] == 'q') {
const auto it = GGML_FTYPE_MAP.find(str);
if (it == GGML_FTYPE_MAP.end()) {
fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);
return GGML_FTYPE_UNKNOWN;
}
ftype = it->second;
} else {
ftype = (enum ggml_ftype) atoi(str);
}
return ftype;
}
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip) {
ggml_type qtype = GGML_TYPE_F32;
switch (ftype) {
case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;
case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;
case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;
case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;
case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;
case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;
case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;
case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;
case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;
case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;
case GGML_FTYPE_UNKNOWN:
case GGML_FTYPE_ALL_F32:
case GGML_FTYPE_MOSTLY_F16:
case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:
case GGML_FTYPE_MOSTLY_IQ2_XXS:
case GGML_FTYPE_MOSTLY_IQ2_XS:
case GGML_FTYPE_MOSTLY_IQ2_S:
case GGML_FTYPE_MOSTLY_IQ3_XXS:
case GGML_FTYPE_MOSTLY_IQ3_S:
case GGML_FTYPE_MOSTLY_IQ1_S:
case GGML_FTYPE_MOSTLY_IQ4_NL:
case GGML_FTYPE_MOSTLY_IQ4_XS:
case GGML_FTYPE_MOSTLY_IQ1_M:
case GGML_FTYPE_MOSTLY_BF16:
case GGML_FTYPE_MOSTLY_MXFP4:
case GGML_FTYPE_MOSTLY_NVFP4:
{
fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
return false;
}
};
if (!ggml_is_quantized(qtype)) {
fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));
return false;
}
size_t total_size_org = 0;
size_t total_size_new = 0;
std::vector<float> work;
std::vector<uint8_t> data_u8;
std::vector<ggml_fp16_t> data_f16;
std::vector<float> data_f32;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
finp.read(reinterpret_cast<char *>(&length), sizeof(length));
finp.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (finp.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[4] = { 1, 1, 1, 1 };
for (int i = 0; i < n_dims; ++i) {
finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
finp.read (&name[0], length);
printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));
bool quantize = false;
// check if we should quantize this tensor
for (const auto & s : to_quant) {
if (std::regex_match(name, std::regex(s))) {
quantize = true;
break;
}
}
// check if we should skip this tensor
for (const auto & s : to_skip) {
if (std::regex_match(name, std::regex(s))) {
quantize = false;
break;
}
}
// quantize only 2D tensors
quantize &= (n_dims == 2);
if (quantize) {
if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {
fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
if (ttype == GGML_TYPE_F16) {
data_f16.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));
data_f32.resize(nelements);
for (int i = 0; i < nelements; ++i) {
data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);
}
} else {
data_f32.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));
}
ttype = qtype;
} else {
const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);
data_u8.resize(nelements*bpe);
finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);
}
fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fout.write(reinterpret_cast<char *>(&length), sizeof(length));
fout.write(reinterpret_cast<char *>(&ttype), sizeof(ttype));
for (int i = 0; i < n_dims; ++i) {
fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
}
fout.write(&name[0], length);
if (quantize) {
work.resize(nelements); // for quantization
size_t cur_size = 0;
switch ((ggml_type) ttype) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
{
cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);
} break;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
case GGML_TYPE_I8:
case GGML_TYPE_I16:
case GGML_TYPE_I32:
case GGML_TYPE_I64:
case GGML_TYPE_F64:
case GGML_TYPE_Q8_1:
case GGML_TYPE_Q8_K:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ1_M:
case GGML_TYPE_BF16:
case GGML_TYPE_TQ1_0:
case GGML_TYPE_TQ2_0:
case GGML_TYPE_MXFP4:
case GGML_TYPE_NVFP4:
case GGML_TYPE_COUNT:
{
fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
}
fout.write(reinterpret_cast<char *>(work.data()), cur_size);
total_size_new += cur_size;
printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);
} else {
printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);
fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());
total_size_new += data_u8.size();
}
total_size_org += nelements * sizeof(float);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));
return true;
}
ggml/examples/common-ggml.h
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "ggml.h"
#include <fstream>
#include <vector>
#include <string>
enum ggml_ftype ggml_parse_ftype(const char * str);
void ggml_print_ftypes(FILE * fp = stderr);
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip);
ggml/examples/common.cpp
+675
View File
@@ -0,0 +1,675 @@
#define _USE_MATH_DEFINES // for M_PI
#include "common.h"
#include <cmath>
#include <codecvt>
#include <cstring>
#include <fstream>
#include <locale>
#include <regex>
#include <sstream>
// Function to check if the next argument exists
static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
if (i + 1 < argc && argv[i + 1][0] != '-') {
return argv[++i];
} else {
fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-s" || arg == "--seed") {
params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-t" || arg == "--threads") {
params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-p" || arg == "--prompt") {
params.prompt = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-n" || arg == "--n_predict") {
params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-np" || arg == "--n_parallel") {
params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_k") {
params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_p") {
params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--temp") {
params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-last-n") {
params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-penalty") {
params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-b" || arg == "--batch_size") {
params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-c" || arg == "--context") {
params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--ignore-eos") {
params.ignore_eos = true;
} else if (arg == "-m" || arg == "--model") {
params.model = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-i" || arg == "--interactive") {
params.interactive = true;
} else if (arg == "-ip" || arg == "--interactive-port") {
params.interactive = true;
params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-h" || arg == "--help") {
gpt_print_usage(argc, argv, params);
exit(0);
} else if (arg == "-f" || arg == "--file") {
get_next_arg(i, argc, argv, arg, params);
std::ifstream file(argv[i]);
if (!file) {
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
break;
}
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
if (params.prompt.back() == '\n') {
params.prompt.pop_back();
}
} else if (arg == "-tt" || arg == "--token_test") {
params.token_test = get_next_arg(i, argc, argv, arg, params);
}
else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
return true;
}
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
fprintf(stderr, " prompt to start generation with (default: random)\n");
fprintf(stderr, " -f FNAME, --file FNAME\n");
fprintf(stderr, " load prompt from a file\n");
fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n");
fprintf(stderr, " test tokenization\n");
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict);
fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k);
fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p);
fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp);
fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n);
fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty);
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx);
fprintf(stderr, " --ignore-eos ignore EOS token during generation\n");
fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
fprintf(stderr, " -m FNAME, --model FNAME\n");
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
fprintf(stderr, "\n");
}
std::string gpt_random_prompt(std::mt19937 & rng) {
const int r = rng() % 10;
switch (r) {
case 0: return "So";
case 1: return "Once upon a time";
case 2: return "When";
case 3: return "The";
case 4: return "After";
case 5: return "If";
case 6: return "import";
case 7: return "He";
case 8: return "She";
case 9: return "They";
}
return "The";
}
std::string trim(const std::string & s) {
std::regex e("^\\s+|\\s+$");
return std::regex_replace(s, e, "");
}
std::string replace(const std::string & s, const std::string & from, const std::string & to) {
std::string result = s;
size_t pos = 0;
while ((pos = result.find(from, pos)) != std::string::npos) {
result.replace(pos, from.length(), to);
pos += to.length();
}
return result;
}
void gpt_vocab::add_special_token(const std::string & token) {
special_tokens.push_back(token);
}
std::map<std::string, int32_t> json_parse(const std::string & fname) {
std::map<std::string, int32_t> result;
// read file into string
std::string json;
{
std::ifstream ifs(fname);
if (!ifs) {
fprintf(stderr, "Failed to open %s\n", fname.c_str());
exit(1);
}
json = std::string((std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()));
}
if (json[0] != '{') {
return result;
}
// parse json
{
bool has_key = false;
bool in_token = false;
std::string str_key = "";
std::string str_val = "";
int n = json.size();
for (int i = 1; i < n; ++i) {
if (!in_token) {
if (json[i] == ' ') continue;
if (json[i] == '"') {
in_token = true;
continue;
}
} else {
if (json[i] == '\\' && i+1 < n) {
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
++i;
} else if (json[i] == '"') {
if (has_key == false) {
has_key = true;
++i;
while (json[i] == ' ') ++i;
++i; // :
while (json[i] == ' ') ++i;
if (json[i] != '\"') {
while (json[i] != ',' && json[i] != '}') {
str_val += json[i++];
}
has_key = false;
} else {
in_token = true;
continue;
}
} else {
has_key = false;
}
str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> "
try {
result[str_key] = std::stoi(str_val);
} catch (...) {
//fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
}
str_key = "";
str_val = "";
in_token = false;
continue;
}
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
}
}
}
return result;
}
void gpt_split_words(std::string str, std::vector<std::string>& words) {
const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
const std::regex re(pattern);
std::smatch m;
while (std::regex_search(str, m, re)) {
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
}
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
std::vector<std::string> words;
// first split the text into words
{
std::string str = text;
// Generate the subpattern from the special_tokens vector if it's not empty
if (!vocab.special_tokens.empty()) {
const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])");
std::string special_tokens_subpattern;
for (const auto & token : vocab.special_tokens) {
if (!special_tokens_subpattern.empty()) {
special_tokens_subpattern += "|";
}
special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)");
}
std::regex re(special_tokens_subpattern);
std::smatch m;
// Split the text by special tokens.
while (std::regex_search(str, m, re)) {
// Split the substrings in-between special tokens into words.
gpt_split_words(m.prefix(), words);
// Add matched special tokens as words.
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
// Remaining text without special tokens will be handled below.
}
gpt_split_words(str, words);
}
// find the longest token that forms each word in words:
std::vector<gpt_vocab::id> tokens;
for (const auto & word : words) {
for (int i = 0; i < (int) word.size(); ){
for (int j = word.size() - 1; j >= i; j--){
auto cand = word.substr(i, j-i+1);
auto it = vocab.token_to_id.find(cand);
if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
tokens.push_back(it->second);
i = j + 1;
break;
}
else if (j == i){ // word.substr(i, 1) has no matching
fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
i++;
}
}
}
}
return tokens;
}
static std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {
std::vector<gpt_vocab::id> output;
std::stringstream ss(input);
std::string token;
while (std::getline(ss, token, delimiter)) {
output.push_back(std::stoi(token));
}
return output;
}
static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){
if (fpath_test.empty()){
fprintf(stderr, "%s : No test file found.\n", __func__);
return std::map<std::string, std::vector<gpt_vocab::id>>();
}
std::map<std::string, std::vector<gpt_vocab::id>> tests;
auto fin = std::ifstream(fpath_test, std::ios_base::in);
const char * delimeter = " => ";
const char del_tok = ',';
std::string line;
while (std::getline(fin, line)) {
size_t delimiterPos = line.find(delimeter);
if (delimiterPos != std::string::npos) {
std::string text = line.substr(0, delimiterPos);
std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));
tests[text] = parse_tokens_from_string(s_tokens, del_tok);
}
}
return tests;
}
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){
std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);
size_t n_fails = 0;
for (const auto & test : tests) {
std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);
if (tokens != test.second){
n_fails++;
// print out failure cases
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str());
fprintf(stderr, "%s : tokens in hf: ", __func__);
for (const auto & t : test.second) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : tokens in ggml: ", __func__);
for (const auto & t : tokens) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
}
}
fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size());
}
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
vocab.token_to_id = ::json_parse(fname);
for (const auto & kv : vocab.token_to_id) {
vocab.id_to_token[kv.second] = kv.first;
}
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
// print the vocabulary
//for (auto kv : vocab.token_to_id) {
// printf("'%s' -> %d\n", kv.first.data(), kv.second);
//}
return true;
}
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const double scale = 1.0/temp;
for (int i = 0; i < n_logits; ++i) {
logits_id.push_back(std::make_pair(logits[i]*scale, i));
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
//printf("\n");
//for (int i = 0; i < (int) probs.size(); i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
//}
//exit(0);
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
const auto * plogits = logits;
const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);
if (temp <= 0) {
// select the token with the highest logit directly
float max_logit = plogits[0];
gpt_vocab::id max_id = 0;
for (int i = 1; i < n_logits; ++i) {
if (plogits[i] > max_logit) {
max_logit = plogits[i];
max_id = i;
}
}
return max_id;
}
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const float scale = 1.0f/temp;
for (int i = 0; i < n_logits; ++i) {
// repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
// credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {
// if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
if (plogits[i] < 0.0f) {
logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
}
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale, i));
}
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
// printf("\n");
// for (int i = 0; i < (int) probs.size(); i++) {
// for (int i = 0; i < 10; i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
// }
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
const float rc = 1.0f / (2.0f * M_PI * cutoff);
const float dt = 1.0f / sample_rate;
const float alpha = dt / (rc + dt);
float y = data[0];
for (size_t i = 1; i < data.size(); i++) {
y = alpha * (y + data[i] - data[i - 1]);
data[i] = y;
}
}
bool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {
const int n_samples = pcmf32.size();
const int n_samples_last = (sample_rate * last_ms) / 1000;
if (n_samples_last >= n_samples) {
// not enough samples - assume no speech
return false;
}
if (freq_thold > 0.0f) {
high_pass_filter(pcmf32, freq_thold, sample_rate);
}
float energy_all = 0.0f;
float energy_last = 0.0f;
for (int i = 0; i < n_samples; i++) {
energy_all += fabsf(pcmf32[i]);
if (i >= n_samples - n_samples_last) {
energy_last += fabsf(pcmf32[i]);
}
}
energy_all /= n_samples;
energy_last /= n_samples_last;
if (verbose) {
fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold);
}
if (energy_last > vad_thold*energy_all) {
return false;
}
return true;
}
float similarity(const std::string & s0, const std::string & s1) {
const size_t len0 = s0.size() + 1;
const size_t len1 = s1.size() + 1;
std::vector<int> col(len1, 0);
std::vector<int> prevCol(len1, 0);
for (size_t i = 0; i < len1; i++) {
prevCol[i] = i;
}
for (size_t i = 0; i < len0; i++) {
col[0] = i;
for (size_t j = 1; j < len1; j++) {
col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));
}
col.swap(prevCol);
}
const float dist = prevCol[len1 - 1];
return 1.0f - (dist / std::max(s0.size(), s1.size()));
}
bool is_file_exist(const char * filename) {
std::ifstream infile(filename);
return infile.good();
}
ggml/examples/common.h
+322
View File
@@ -0,0 +1,322 @@
// Various helper functions and utilities
#pragma once
#include <string>
#include <map>
#include <vector>
#include <random>
#include <thread>
#include <ctime>
#include <fstream>
#include <sstream>
//
// GPT CLI argument parsing
//
struct gpt_params {
int32_t seed = -1; // RNG seed
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_predict = 200; // new tokens to predict
int32_t n_parallel = 1; // number of parallel streams
int32_t n_batch = 32; // batch size for prompt processing
int32_t n_ctx = 2048; // context size (this is the KV cache max size)
int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU
bool ignore_eos = false; // ignore EOS token when generating text
// sampling parameters
int32_t top_k = 40;
float top_p = 0.9f;
float temp = 0.9f;
int32_t repeat_last_n = 64;
float repeat_penalty = 1.00f;
std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
std::string prompt = "";
std::string token_test = "";
bool interactive = false;
int32_t interactive_port = -1;
};
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng);
//
// Vocab utils
//
std::string trim(const std::string & s);
std::string replace(
const std::string & s,
const std::string & from,
const std::string & to);
struct gpt_vocab {
using id = int32_t;
using token = std::string;
std::map<token, id> token_to_id;
std::map<id, token> id_to_token;
std::vector<std::string> special_tokens;
void add_special_token(const std::string & token);
};
// poor-man's JSON parsing
std::map<std::string, int32_t> json_parse(const std::string & fname);
std::string convert_to_utf8(const std::wstring & input);
std::wstring convert_to_wstring(const std::string & input);
void gpt_split_words(std::string str, std::vector<std::string>& words);
// split text into tokens
//
// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
//
// Regex (Python):
// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
//
// Regex (C++):
// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
//
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
// test outputs of gpt_tokenize
//
// - compare with tokens generated by the huggingface tokenizer
// - test cases are chosen based on the model's main language (under 'prompt' directory)
// - if all sentences are tokenized identically, print 'All tests passed.'
// - otherwise, print sentence, huggingface tokens, ggml tokens
//
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);
// load the tokens from encoder.json
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
// sample next token given probabilities for each embedding
//
// - consider only the top K tokens
// - from them, consider only the top tokens with cumulative probability > P
//
// TODO: not sure if this implementation is correct
// TODO: temperature is not implemented
//
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng);
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng);
//
// Audio utils
//
// Write PCM data into WAV audio file
class wav_writer {
private:
std::ofstream file;
uint32_t dataSize = 0;
std::string wav_filename;
bool write_header(const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
file.write("RIFF", 4);
file.write("\0\0\0\0", 4); // Placeholder for file size
file.write("WAVE", 4);
file.write("fmt ", 4);
const uint32_t sub_chunk_size = 16;
const uint16_t audio_format = 1; // PCM format
const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;
const uint16_t block_align = channels * bits_per_sample / 8;
file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);
file.write(reinterpret_cast<const char *>(&audio_format), 2);
file.write(reinterpret_cast<const char *>(&channels), 2);
file.write(reinterpret_cast<const char *>(&sample_rate), 4);
file.write(reinterpret_cast<const char *>(&byte_rate), 4);
file.write(reinterpret_cast<const char *>(&block_align), 2);
file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);
file.write("data", 4);
file.write("\0\0\0\0", 4); // Placeholder for data size
return true;
}
// It is assumed that PCM data is normalized to a range from -1 to 1
bool write_audio(const float * data, size_t length) {
for (size_t i = 0; i < length; ++i) {
const int16_t intSample = int16_t(data[i] * 32767);
file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));
dataSize += sizeof(int16_t);
}
if (file.is_open()) {
file.seekp(4, std::ios::beg);
uint32_t fileSize = 36 + dataSize;
file.write(reinterpret_cast<char *>(&fileSize), 4);
file.seekp(40, std::ios::beg);
file.write(reinterpret_cast<char *>(&dataSize), 4);
file.seekp(0, std::ios::end);
}
return true;
}
bool open_wav(const std::string & filename) {
if (filename != wav_filename) {
if (file.is_open()) {
file.close();
}
}
if (!file.is_open()) {
file.open(filename, std::ios::binary);
wav_filename = filename;
dataSize = 0;
}
return file.is_open();
}
public:
bool open(const std::string & filename,
const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
if (open_wav(filename)) {
write_header(sample_rate, bits_per_sample, channels);
} else {
return false;
}
return true;
}
bool close() {
file.close();
return true;
}
bool write(const float * data, size_t length) {
return write_audio(data, length);
}
~wav_writer() {
if (file.is_open()) {
file.close();
}
}
};
// Apply a high-pass frequency filter to PCM audio
// Suppresses frequencies below cutoff Hz
void high_pass_filter(
std::vector<float> & data,
float cutoff,
float sample_rate);
// Basic voice activity detection (VAD) using audio energy adaptive threshold
bool vad_simple(
std::vector<float> & pcmf32,
int sample_rate,
int last_ms,
float vad_thold,
float freq_thold,
bool verbose);
// compute similarity between two strings using Levenshtein distance
float similarity(const std::string & s0, const std::string & s1);
//
// Terminal utils
//
#define SQR(X) ((X) * (X))
#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
/**
* Quantizes 24-bit RGB to xterm256 code range [16,256).
*/
static int rgb2xterm256(int r, int g, int b) {
unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
int av, ir, ig, ib, il, qr, qg, qb, ql;
av = r * .299 + g * .587 + b * .114 + .5;
ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
qr = cube[(ir = UNCUBE(r))];
qg = cube[(ig = UNCUBE(g))];
qb = cube[(ib = UNCUBE(b))];
if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
return ir * 36 + ig * 6 + ib + 020;
return il + 0350;
}
static std::string set_xterm256_foreground(int r, int g, int b) {
int x = rgb2xterm256(r, g, b);
std::ostringstream oss;
oss << "\033[38;5;" << x << "m";
return oss.str();
}
// Lowest is red, middle is yellow, highest is green. Color scheme from
// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault
const std::vector<std::string> k_colors = {
set_xterm256_foreground(220, 5, 12),
set_xterm256_foreground(232, 96, 28),
set_xterm256_foreground(241, 147, 45),
set_xterm256_foreground(246, 193, 65),
set_xterm256_foreground(247, 240, 86),
set_xterm256_foreground(144, 201, 135),
set_xterm256_foreground( 78, 178, 101),
};
// ANSI formatting codes
static std::string set_inverse() {
return "\033[7m";
}
static std::string set_underline() {
return "\033[4m";
}
static std::string set_dim() {
return "\033[2m";
}
// Style scheme for different confidence levels
const std::vector<std::string> k_styles = {
set_inverse(), // Low confidence - inverse (highlighted)
set_underline(), // Medium confidence - underlined
set_dim(), // High confidence - dim
};
//
// Other utils
//
// check if file exists using ifstream
bool is_file_exist(const char * filename);
ggml/examples/gpt-2/CMakeLists.txt
+32
View File
@@ -0,0 +1,32 @@
#
# gpt-2
set(TEST_TARGET gpt-2-ctx)
add_executable(${TEST_TARGET} main-ctx.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-alloc)
add_executable(${TEST_TARGET} main-alloc.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-backend)
add_executable(${TEST_TARGET} main-backend.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-sched)
add_executable(${TEST_TARGET} main-sched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-quantize
set(TEST_TARGET gpt-2-quantize)
add_executable(${TEST_TARGET} quantize.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-batched
set(TEST_TARGET gpt-2-batched)
add_executable(${TEST_TARGET} main-batched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
ggml/examples/gpt-2/README.md
+225
View File
@@ -0,0 +1,225 @@
# gpt-2
This is a C++ example running GPT-2 inference using the [ggml](https://github.com/ggerganov/ggml) library.
The program runs on the CPU - no video card is required.
The [Cerebras-GPT](https://huggingface.co/cerebras) models are also supported.
The example supports the following GPT-2 models:
| Model | Description | Disk Size |
| --- | --- | --- |
| 117M | Small model | 240 MB |
| 345M | Medium model | 680 MB |
| 774M | Large model | 1.5 GB |
| 1558M | XL model | 3.0 GB |
Sample performance on MacBook M1 Pro:
| Model | Size | Time / Token |
| --- | --- | --- |
| GPT-2 | 117M | 5 ms |
| GPT-2 | 345M | 12 ms |
| GPT-2 | 774M | 23 ms |
| GPT-2 | 1558M | 42 ms |
*TODO: add tables for Cerebras-GPT models*
Sample output:
```bash
$ ./bin/gpt-2 -h
usage: ./bin/gpt-2 [options]
options:
-h, --help show this help message and exit
-s SEED, --seed SEED RNG seed (default: -1)
-t N, --threads N number of threads to use during computation (default: 8)
-p PROMPT, --prompt PROMPT
prompt to start generation with (default: random)
-n N, --n_predict N number of tokens to predict (default: 200)
--top_k N top-k sampling (default: 40)
--top_p N top-p sampling (default: 0.9)
--temp N temperature (default: 1.0)
-b N, --batch_size N batch size for prompt processing (default: 8)
-m FNAME, --model FNAME
model path (default: models/gpt-2-117M/ggml-model.bin)
$ ./bin/gpt-2
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
gpt2_model_load: n_vocab = 50257
gpt2_model_load: n_ctx = 1024
gpt2_model_load: n_embd = 768
gpt2_model_load: n_head = 12
gpt2_model_load: n_layer = 12
gpt2_model_load: f16 = 1
gpt2_model_load: ggml ctx size = 311.12 MB
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
gpt2_model_load: model size = 239.08 MB
main: number of tokens in prompt = 1
So this is going to be the end of the line for us.
If the Dolphins continue to do their business, it's possible that the team could make a bid to bring in new defensive coordinator Scott Linehan.
Linehan's job is a little daunting, but he's a great coach and an excellent coach. I don't believe we're going to make the playoffs.
We're going to have to work hard to keep our heads down and get ready to go.<|endoftext|>
main: mem per token = 2048612 bytes
main: load time = 106.32 ms
main: sample time = 7.10 ms
main: predict time = 506.40 ms / 5.06 ms per token
main: total time = 629.84 ms
```
## Downloading and converting the original models (GPT-2)
You can download the original model files using the [download-model.sh](download-model.sh) Bash script. The models are
in Tensorflow format, so in order to use them with ggml, you need to convert them to appropriate format. This is done
via the [convert-ckpt-to-ggml.py](convert-ckpt-to-ggml.py) python script.
Here is the entire process for the GPT-2 117M model (download from official site + conversion):
```bash
cd ggml/build
../examples/gpt-2/download-model.sh 117M
Downloading model 117M ...
models/gpt-2-117M/checkpoint 100%[=============================>] 77 --.-KB/s in 0s
models/gpt-2-117M/encoder.json 100%[=============================>] 1018K 1.20MB/s in 0.8s
models/gpt-2-117M/hparams.json 100%[=============================>] 90 --.-KB/s in 0s
models/gpt-2-117M/model.ckpt.data-00000-of-00001 100%[=============================>] 474.70M 1.21MB/s in 8m 39s
models/gpt-2-117M/model.ckpt.index 100%[=============================>] 5.09K --.-KB/s in 0s
models/gpt-2-117M/model.ckpt.meta 100%[=============================>] 460.11K 806KB/s in 0.6s
models/gpt-2-117M/vocab.bpe 100%[=============================>] 445.62K 799KB/s in 0.6s
Done! Model '117M' saved in 'models/gpt-2-117M/'
Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.
python /Users/john/ggml/examples/gpt-2/convert-ckpt-to-ggml.py models/gpt-2-117M/ 1
```
This conversion requires that you have python and Tensorflow installed on your computer. Still, if you want to avoid
this, you can download the already converted ggml models as described below.
## Downloading and converting the original models (Cerebras-GPT)
Clone the respective repository from here: https://huggingface.co/cerebras
Use the [convert-cerebras-to-ggml.py](convert-cerebras-to-ggml.py) script to convert the model to `ggml` format:
```bash
cd ggml/build
git clone https://huggingface.co/cerebras/Cerebras-GPT-111M models/
python ../examples/gpt-2/convert-cerebras-to-ggml.py models/Cerebras-GPT-111M/
```
## Downloading the ggml model directly (GPT-2)
For convenience, I will be hosting the converted ggml model files in order to make it easier to run the examples. This
way, you can directly download a single binary file and start using it. No python or Tensorflow is required.
Here is how to get the 117M ggml model:
```bash
cd ggml/build
../examples/gpt-2/download-ggml-model.sh 117M
Downloading ggml model 117M ...
models/gpt-2-117M/ggml-model.bin 100%[===============================>] 239.58M 8.52MB/s in 28s
Done! Model '117M' saved in 'models/gpt-2-117M/ggml-model.bin'
You can now use it like this:
$ ./bin/gpt-2 -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
```
At some point, I might decide to stop hosting these models. So in that case, simply revert to the manual process above.
## Quantizing the models
You can also try to quantize the `ggml` models via 4-bit integer quantization.
Keep in mind that for smaller models, this will render them completely useless.
You generally want to quantize larger models.
```bash
# quantize GPT-2 F16 to Q4_0 (faster but less precise)
./bin/gpt-2-quantize models/gpt-2-1558M/ggml-model-f16.bin models/gpt-2-1558M/ggml-model-q4_0.bin 2
./bin/gpt-2 -m models/gpt-2-1558M/ggml-model-q4_0.bin -p "This is an example"
# quantize Cerebras F16 to Q4_1 (slower but more precise)
./bin/gpt-2-quantize models/Cerebras-GPT-6.7B/ggml-model-f16.bin models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin 3
./bin/gpt-2 -m models/Cerebras-GPT-6.7B/ggml-model-q4_1.bin -p "This is an example"
```
## Batched generation example
You can try the batched generation from a given prompt using the gpt-2-batched binary.
Sample output:
```bash
$ gpt-2-batched -np 5 -m models/gpt-2-117M/ggml-model.bin -p "Hello my name is" -n 50
main: seed = 1697037431
gpt2_model_load: loading model from 'models/gpt-2-117M/ggml-model.bin'
gpt2_model_load: n_vocab = 50257
gpt2_model_load: n_ctx = 1024
gpt2_model_load: n_embd = 768
gpt2_model_load: n_head = 12
gpt2_model_load: n_layer = 12
gpt2_model_load: ftype = 1
gpt2_model_load: qntvr = 0
gpt2_model_load: ggml tensor size = 320 bytes
gpt2_model_load: backend buffer size = 312.72 MB
ggml_init_cublas: found 1 CUDA devices:
Device 0: NVIDIA GeForce GTX 1660, compute capability 7.5
gpt2_model_load: using CPU backend
gpt2_model_load: memory size = 72.00 MB, n_mem = 12288
gpt2_model_load: model size = 239.08 MB
extract_tests_from_file : No test file found.
test_gpt_tokenizer : 0 tests failed out of 0 tests.
main: compute buffer size: 3.26 MB
main: generating 5 sequences ...
main: prompt: 'Hello my name is'
main: number of tokens in prompt = 4, first 8 tokens: 15496 616 1438 318
sequence 0:
Hello my name is John. You can call me any way you want, if you want, but for my very first date, I will be on the phone with you. We're both in our early 20s, but I feel like it's all
sequence 1:
Hello my name is Robert, and I want to say that we're proud to have your company here on the world's largest platform for sharing your stories with us. This is a huge opportunity for our community. We have hundreds of people on this team and
sequence 2:
Hello my name is Jack. I'm the one who created you.
Jack is a boy with a big smile and a big heart. He is a handsome guy. He loves the outdoors and loves the people he meets. He wants to be a
sequence 3:
Hello my name is John. I am a Canadian citizen with a large number of family in Quebec and I am interested in studying. My aim is to take up a post in the Journal of the International Academy of Sciences of Canada which I am currently finishing.
sequence 4:
Hello my name is Dan. I am an entrepreneur. I am a great father. I am a great husband. I am a great husband. I am a great dad. And I am a great husband.
I love my life. I love
main: load time = 880.80 ms
main: sample time = 91.43 ms
main: predict time = 2518.29 ms
main: total time = 3544.32 ms
```
ggml/examples/gpt-2/convert-cerebras-to-ggml.py
+183
View File
@@ -0,0 +1,183 @@
# Convert Cerebras models to ggml format
#
# ref: https://www.cerebras.net/blog/cerebras-gpt-a-family-of-open-compute-efficient-large-language-models/
#
import sys
import struct
import json
import torch
import numpy as np
import re
from transformers import AutoModelForCausalLM
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 2:
print("Usage: convert-cerebras-to-ggml.py dir-model [use-f32]\n")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model-f16.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# use 16-bit or 32-bit floats
use_f16 = True
if len(sys.argv) > 2:
use_f16 = False
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
model = AutoModelForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
print(hparams)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", use_f16))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# rename headers to keep compatibility
if name == "transformer.ln_f.weight":
name = "model/ln_f/g"
elif name == "transformer.ln_f.bias":
name = "model/ln_f/b"
elif name == "transformer.wte.weight":
name = "model/wte"
elif name == "transformer.wpe.weight":
name = "model/wpe"
elif name == "lm_head.weight":
name = "model/lm_head"
elif re.match(r"transformer.h\.\d+\.ln_1\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/g"
elif re.match(r"transformer.h\.\d+\.ln_1\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/b"
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/w"
elif re.match(r"transformer.h\.\d+\.attn\.c_attn\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/b"
elif re.match(r"transformer.h\.\d+\.attn\.c_proj\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/w"
elif re.match(r"transformer.h.\d+.attn.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/b"
elif re.match(r"transformer.h.\d+.ln_2.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/g"
elif re.match(r"transformer.h.\d+.ln_2.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/b"
elif re.match(r"transformer.h.\d+.mlp.c_fc.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/w"
elif re.match(r"transformer.h.\d+.mlp.c_fc.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/b"
elif re.match(r"transformer.h.\d+.mlp.c_proj.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/w"
elif re.match(r"transformer.h.\d+.mlp.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/b"
else:
print("Unrecognized variable name. %s", name)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype = 0;
if use_f16:
if (name == "model/wte" or name == "model/lm_head" or name[-2:] == "/g" or name[-2:] == "/w") and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype = 0
# for efficiency - transpose the projection matrices
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name[-14:] == "/attn/c_attn/w" or \
name[-14:] == "/attn/c_proj/w" or \
name[-11:] == "/mlp/c_fc/w" or \
name[-13:] == "/mlp/c_proj/w":
print(" Transposing")
data = data.transpose()
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/convert-ckpt-to-ggml.py
+159
View File
@@ -0,0 +1,159 @@
# Convert a model checkpoint to a ggml compatible file
#
# Load the model using TensorFlow.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import json
import struct
import numpy as np
import tensorflow as tf
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
# helper method to convert a numpy array to different float types
def convert_to_ftype(data, ftype):
# fp16
if ftype == 1:
return data.astype(np.float16)
assert False, "Invalid ftype: " + str(ftype)
if len(sys.argv) < 3:
print("Usage: convert-ckpt-to-ggml.py dir-model ftype\n")
print(" ftype == 0 -> float32")
print(" ftype == 1 -> float16")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/encoder.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/hparams.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# possible data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#
# map from ftype to string
ftype_str = ["f32", "f16"]
ftype = 1
if len(sys.argv) > 2:
ftype = int(sys.argv[2])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))
sys.exit(1)
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
list_vars = tf.train.list_variables(dir_model)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["n_vocab"]))
fout.write(struct.pack("i", hparams["n_ctx"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", ftype))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name, shape in list_vars:
print("Processing variable: " + name + " with shape: ", shape)
data = tf.train.load_variable(dir_model, name).squeeze()
n_dims = len(data.shape);
# for efficiency - transpose the projection matrices
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name[-14:] == "/attn/c_attn/w" or \
name[-14:] == "/attn/c_proj/w" or \
name[-11:] == "/mlp/c_fc/w" or \
name[-13:] == "/mlp/c_proj/w":
print(" Transposing")
data = data.transpose()
dshape = data.shape
ftype_cur = 0
if ftype != 0:
# match name:
# "model/wte"
# "model/h.*/attn/c_attn/w"
# "model/h.*/attn/c_proj/w"
# "model/h.*/mlp/c_fc/w"
# "model/h.*/mlp/c_proj/w"
if name == "model/wte" or name[-2:] == "/w":
print(" Converting to " + ftype_str[ftype])
data = convert_to_ftype(data, ftype)
ftype_cur = ftype
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
for i in range(n_dims):
fout.write(struct.pack("i", dshape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/convert-h5-to-ggml.py
+195
View File
@@ -0,0 +1,195 @@
# Convert GPT-2 h5 transformer model to ggml format
#
# Load the model using GPT2Model.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import struct
import json
import numpy as np
import re
from transformers import GPT2Model
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 2:
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
encoder_added = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# use 16-bit or 32-bit floats
use_f16 = True
if len(sys.argv) > 2:
use_f16 = False
fname_out = sys.argv[1] + "/ggml-model-f32.bin"
model = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
#fout.write(struct.pack("i", hparams["rotary_dim"]))
fout.write(struct.pack("i", use_f16))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for key in encoder_added:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype = 0;
if use_f16:
if name[-7:] == ".weight" and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype = 0
# for efficiency - transpose these matrices:
# "transformer.h.*.mlp.c_proj.weight
if name.endswith(".mlp.c_proj.weight"):
print(" Transposing")
data = data.transpose()
# rename headers to keep compatibility
if name == "ln_f.weight":
name = "model/ln_f/g"
elif name == "ln_f.bias":
name = "model/ln_f/b"
elif name == "wte.weight":
name = "model/wte"
elif name == "wpe.weight":
name = "model/wpe"
elif re.match(r"h\.\d+\.ln_1\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/g"
elif re.match(r"h\.\d+\.ln_1\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_1/b"
elif re.match(r"h\.\d+\.attn\.c_attn\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/w"
elif re.match(r"h\.\d+\.attn\.c_attn\.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_attn/b"
elif re.match(r"h\.\d+\.attn\.c_proj\.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/w"
elif re.match(r"h.\d+.attn.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/attn/c_proj/b"
elif re.match(r"h.\d+.ln_2.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/g"
elif re.match(r"h.\d+.ln_2.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/ln_2/b"
elif re.match(r"h.\d+.mlp.c_fc.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/w"
elif re.match(r"h.\d+.mlp.c_fc.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_fc/b"
elif re.match(r"h.\d+.mlp.c_proj.weight", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/w"
elif re.match(r"h.\d+.mlp.c_proj.bias", name):
i = re.findall("\d+", name)[0]
name = f"model/h{i}/mlp/c_proj/b"
else:
print("Unrecognized variable name. %s", name)
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-2/download-ggml-model.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/bash
# This script downloads GPT-2 model files that have already been converted to ggml format.
# This way you don't have to convert them yourself.
#
# If you want to download the original GPT-2 model files, use the "download-model.sh" script instead.
#src="https://ggml.ggerganov.com"
#pfx="ggml-model-gpt-2"
src="https://huggingface.co/ggerganov/ggml"
pfx="resolve/main/ggml-model-gpt-2"
ggml_path=$(dirname $(realpath $0))
# GPT-2 models
models=( "117M" "345M" "774M" "1558M" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download ggml model
printf "Downloading ggml model $model ...\n"
mkdir -p models/gpt-2-$model
if [ -x "$(command -v wget)" ]; then
wget --quiet --show-progress -O models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
elif [ -x "$(command -v curl)" ]; then
curl -L --output models/gpt-2-$model/ggml-model.bin $src/$pfx-$model.bin
else
printf "Either wget or curl is required to download models.\n"
exit 1
fi
if [ $? -ne 0 ]; then
printf "Failed to download ggml model $model \n"
printf "Please try again later or download the original GPT-2 model files and convert them yourself.\n"
exit 1
fi
printf "Done! Model '$model' saved in 'models/gpt-2-$model/ggml-model.bin'\n"
printf "You can now use it like this:\n\n"
printf " $ ./bin/gpt-2 -m models/gpt-2-$model/ggml-model.bin -p \"This is an example\"\n"
printf "\n"
ggml/examples/gpt-2/download-model.sh
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/bin/bash
ggml_path=$(dirname $(realpath $0))
# GPT-2 models
models=( "117M" "345M" "774M" "1558M" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download model
printf "Downloading model $model ...\n"
mkdir -p models/gpt-2-$model
for file in checkpoint encoder.json hparams.json model.ckpt.data-00000-of-00001 model.ckpt.index model.ckpt.meta vocab.bpe; do
wget --quiet --show-progress -O models/gpt-2-$model/$file https://openaipublic.blob.core.windows.net/gpt-2/models/$model/$file
done
printf "Done! Model '$model' saved in 'models/gpt-2-$model/'\n\n"
printf "Run the convert-ckpt-to-ggml.py script to convert the model to ggml format.\n"
printf "\n"
printf " python $ggml_path/convert-ckpt-to-ggml.py models/gpt-2-$model/\n"
printf "\n"
ggml/examples/gpt-2/main-alloc.cpp
+880
View File
@@ -0,0 +1,880 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx_w;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
ctx_size += ggml_row_size(GGML_TYPE_F32 , n_ctx*n_embd); // wpe
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
ctx_size += (6 + 12*n_layer)*512; // object overhead
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx_w = ggml_init(params);
if (!model.ctx_w) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// build the computation graph
struct ggml_cgraph * gpt2_graph(
const gpt2_model & model,
const int n_past,
const int n_tokens) {
const int N = n_tokens;
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
// at this point, the tensor data is not allocated yet and cannot be set
// we will find the tensor after the graph is allocated by its name, and set the data then
ggml_set_name(embd, "embd");
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
// this is important to ensure that the input tensors are not overwritten before they are used
ggml_set_input(embd);
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
ggml_set_name(position, "position");
ggml_set_input(position);
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx,
ggml_get_rows(ctx, model.wte, embd),
ggml_get_rows(ctx, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx, model.layers[il].ln_1_b, cur));
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_attn_attn_b, cur),
cur);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx,
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled =
ggml_scale(ctx,
KQ,
1.0f/sqrtf(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cont_3d(ctx,
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
n_past + N, n_embd/n_head, n_head);
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_attn_proj_b, cur),
cur);
}
// add the input
cur = ggml_add(ctx, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.layers[il].ln_2_g, cur),
cur),
ggml_repeat(ctx, model.layers[il].ln_2_b, cur));
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx,
ggml_repeat(ctx, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// input for next layer
inpL = ggml_add(ctx, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx,
ggml_mul(ctx,
ggml_repeat(ctx, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx, model.ln_f_b, inpL));
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
ggml_set_name(inpL, "logits");
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
ggml_set_output(inpL);
// logits -> probs
//inpL = ggml_soft_max(ctx0, inpL);
ggml_build_forward_expand(gf, inpL);
ggml_free(ctx);
return gf;
}
// evaluate the transformer
//
// - model: the model
// - allocr: ggml_gallocr to use to allocate the compute buffer
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
ggml_gallocr_t allocr,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_vocab = hparams.n_vocab;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
// allocate the graph tensors
ggml_gallocr_alloc_graph(allocr, gf);
// set the graph inputs
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
for (int i = 0; i < N; ++i) {
((int32_t *) position->data)[i] = n_past + i;
}
// run the computation
struct ggml_cplan plan = ggml_graph_plan(gf, n_threads, nullptr);
static std::vector<uint8_t> work_buffer;
work_buffer.resize(plan.work_size);
plan.work_data = work_buffer.data();
ggml_graph_compute(gf, &plan);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
// get the graph outputs
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(logits), sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(logits) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
ggml_gallocr_t allocr = NULL;
// allocate the compute buffer
{
allocr = ggml_gallocr_new(ggml_backend_cpu_buffer_type());
// create the worst case graph for memory usage estimation
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
int n_past = model.hparams.n_ctx - n_tokens;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
// pre-allocate the compute buffer for the worst case (optional)
ggml_gallocr_reserve(allocr, gf);
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
return 0;
}
ggml/examples/gpt-2/main-backend.cpp
+946
View File
@@ -0,0 +1,946 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#ifdef GGML_USE_CUDA
#include "ggml-cuda.h"
#endif
#ifdef GGML_USE_METAL
#include "ggml-metal.h"
#endif
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
#define GPT2_MAX_NODES 4096
static void ggml_log_callback_default(ggml_log_level level, const char * text, void * user_data) {
(void) level;
(void) user_data;
fputs(text, stderr);
fflush(stderr);
}
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
struct ggml_context * ctx_kv;
ggml_backend_t backend = NULL;
ggml_backend_buffer_t buffer_w;
ggml_backend_buffer_t buffer_kv;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab, int n_ctx, int n_gpu_layers) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
ggml_log_set(ggml_log_callback_default, nullptr);
auto & ctx = model.ctx_w;
// create the ggml context
{
size_t n_tensors = 2 + 6 + 12*model.hparams.n_layer;
struct ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx = ggml_init(params);
if (!ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// initialize the backend
#ifdef GGML_USE_CUDA
if (n_gpu_layers > 0) {
fprintf(stderr, "%s: using CUDA backend\n", __func__);
model.backend = ggml_backend_cuda_init(0);
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);
}
}
#endif
#ifdef GGML_USE_METAL
if (n_gpu_layers > 0) {
fprintf(stderr, "%s: using Metal backend\n", __func__);
model.backend = ggml_backend_metal_init();
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
}
}
#endif
if (!model.backend) {
// fallback to CPU backend
fprintf(stderr, "%s: using CPU backend\n", __func__);
model.backend = ggml_backend_cpu_init();
}
if (!model.backend) {
fprintf(stderr, "%s: ggml_backend_cpu_init() failed\n", __func__);
return false;
}
// create the tensors for the model
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// allocate the model tensors in a backend buffer
model.buffer_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: backend buffer size = %6.2f MB\n", __func__, ggml_backend_buffer_get_size(model.buffer_w)/(1024.0*1024.0));
// override the default training context with the user-provided
model.hparams.n_ctx = n_ctx;
// key + value memory
{
auto * ctx = model.ctx_kv;
// create the ggml context
{
size_t n_tensors = 2;
struct ggml_init_params params = {
/*.mem_size =*/ ggml_tensor_overhead() * n_tensors,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx = ggml_init(params);
if (!ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
// k and v here can also be GGML_TYPE_F16 to save memory and speed up the computation
// if backend supports it
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
// allocate the KV memory in a backend buffer
model.buffer_kv = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
const size_t memory_size = ggml_backend_buffer_get_size(model.buffer_kv);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
std::vector<char> read_buf;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
ggml_set_name(tensor, name.c_str());
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
if (ggml_backend_buffer_is_host(model.buffer_w)) {
// for some backends such as CPU and Metal, the tensor data is in system memory and we can read directly into it
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
} else {
// read into a temporary buffer first, then copy to device memory
read_buf.resize(ggml_nbytes(tensor));
fin.read(read_buf.data(), ggml_nbytes(tensor));
ggml_backend_tensor_set(tensor, read_buf.data(), 0, ggml_nbytes(tensor));
}
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
//ggml_backend_tensor_copy(tensor, model.lm_head);
model.lm_head = tensor;
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// build the computation graph
struct ggml_cgraph * gpt2_graph(
const gpt2_model & model,
const int n_past,
const int n_tokens) {
const int N = n_tokens;
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
// since we are using ggml-alloc, this buffer only needs enough space to hold the ggml_tensor and ggml_cgraph structs, but not the tensor data
static size_t buf_size = ggml_tensor_overhead()*GPT2_MAX_NODES + ggml_graph_overhead_custom(GPT2_MAX_NODES, false);
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, GPT2_MAX_NODES, false);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
// at this point, the tensor data is not allocated yet and cannot be set
// we will find the tensor after the graph is allocated by its name, and set the data then
ggml_set_name(embd, "embd");
// setting a tensor as an input will ensure that it is allocated at the beginning of the graph
// this is important to ensure that the input tensors are not overwritten before they are used
ggml_set_input(embd);
struct ggml_tensor * position = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, N);
ggml_set_name(position, "position");
ggml_set_input(position);
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx,
ggml_get_rows(ctx, model.wte, embd),
ggml_get_rows(ctx, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
cur,
model.layers[il].ln_1_g),
model.layers[il].ln_1_b);
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_attn_attn_b);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx,
ggml_cont_3d(ctx, Qcur, n_embd/n_head, n_head, N),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled =
ggml_scale(ctx,
KQ,
1.0f/sqrtf(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cont_3d(ctx,
ggml_permute(ctx,
ggml_reshape_3d(ctx,
ggml_view_1d(ctx, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
n_past + N, n_embd/n_head, n_head);
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cont_2d(ctx, KQV_merged, n_embd, N);
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_attn_proj_b);
}
// add the input
cur = ggml_add(ctx, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx,
ggml_mul(ctx,
cur,
model.layers[il].ln_2_g),
model.layers[il].ln_2_b);
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_mlp_fc_b);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx,
cur,
model.layers[il].c_mlp_proj_b);
}
// input for next layer
inpL = ggml_add(ctx, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx,
ggml_mul(ctx,
inpL,
model.ln_f_g),
model.ln_f_b);
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx, model.lm_head, inpL);
ggml_set_name(inpL, "logits");
// setting a tensor as the output will ensure that it is not overwritten by subsequent operations
ggml_set_output(inpL);
// logits -> probs
//inpL = ggml_soft_max(ctx0, inpL);
ggml_build_forward_expand(gf, inpL);
ggml_free(ctx);
return gf;
}
// evaluate the transformer
//
// - model: the model
// - allocr: ggml_gallocr to use to allocate the compute buffer
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
ggml_gallocr_t allocr,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_vocab = hparams.n_vocab;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, embd_inp.size());
// allocate the graph tensors
ggml_gallocr_alloc_graph(allocr, gf);
// set the graph inputs
struct ggml_tensor * embd = ggml_graph_get_tensor(gf, "embd");
ggml_backend_tensor_set(embd, embd_inp.data(), 0, N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_graph_get_tensor(gf, "position");
for (int i = 0; i < N; ++i) {
int32_t v = n_past + i;
ggml_backend_tensor_set(position, &v, i*sizeof(int32_t), sizeof(v));
}
// set backend options
if (ggml_backend_is_cpu(model.backend)) {
ggml_backend_cpu_set_n_threads(model.backend, n_threads);
}
// run the computation
ggml_backend_graph_compute(model.backend, gf);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
// get the graph outputs
struct ggml_tensor * logits = ggml_graph_get_tensor(gf, "logits");
//embd_w.resize(n_vocab*N);
//ggml_backend_tensor_get(logits, embd_w.data(), 0, sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
ggml_backend_tensor_get(logits, embd_w.data(), (n_vocab*(N-1))*sizeof(float), sizeof(float)*n_vocab);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab, params.n_ctx, params.n_gpu_layers)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
ggml_gallocr_t allocr = NULL;
// allocate the compute buffer
{
// create a graph allocator with the backend's default buffer type
allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
// create the worst case graph for memory usage estimation
int n_tokens = std::min(model.hparams.n_ctx, params.n_batch);
int n_past = model.hparams.n_ctx - n_tokens;
struct ggml_cgraph * gf = gpt2_graph(model, n_past, n_tokens);
// pre-allocate the compute buffer for the worst case (optional)
ggml_gallocr_reserve(allocr, gf);
size_t mem_size = ggml_gallocr_get_buffer_size(allocr, 0);
fprintf(stderr, "%s: compute buffer size: %.2f MB\n", __func__, mem_size/1024.0/1024.0);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, allocr, params.n_threads, n_past, embd, logits)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (!params.ignore_eos && embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
ggml_gallocr_free(allocr);
ggml_backend_buffer_free(model.buffer_w);
ggml_backend_buffer_free(model.buffer_kv);
ggml_backend_free(model.backend);
return 0;
}
ggml/examples/gpt-2/main-batched.cpp
+1210
View File
File diff suppressed because it is too large Load Diff
ggml/examples/gpt-2/main-ctx.cpp
+840
View File
@@ -0,0 +1,840 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gpt2_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
struct ggml_tensor * ln_2_g;
struct ggml_tensor * ln_2_b;
// attention
struct ggml_tensor * c_attn_attn_w;
struct ggml_tensor * c_attn_attn_b;
struct ggml_tensor * c_attn_proj_w;
struct ggml_tensor * c_attn_proj_b;
// mlp
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gpt2_model {
gpt2_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * wpe; // position embedding
struct ggml_tensor * lm_head; // language model head
std::vector<gpt2_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx_w;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s'\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx_w;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // wte
ctx_size += ggml_row_size(GGML_TYPE_F32, n_ctx*n_embd); // wpe
ctx_size += ggml_row_size(wtype, n_vocab*n_embd); // lm_head
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_2_b
ctx_size += n_layer*(ggml_row_size(wtype, 3*n_embd*n_embd)); // c_attn_attn_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 3*n_embd)); // c_attn_attn_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_attn_proj_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F32, n_embd); // memory_v
ctx_size += (6 + 12*n_layer)*512; // object overhead
printf("%s: ggml tensor size = %d bytes\n", __func__, (int) sizeof(ggml_tensor));
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx_w = ggml_init(params);
if (!model.ctx_w) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx);
model.lm_head = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
// map by name
model.tensors["model/ln_f/g"] = model.ln_f_g;
model.tensors["model/ln_f/b"] = model.ln_f_b;
model.tensors["model/wte"] = model.wte;
model.tensors["model/wpe"] = model.wpe;
model.tensors["model/lm_head"] = model.lm_head;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_2_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_attn_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 3*n_embd);
layer.c_attn_attn_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 3*n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g;
model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b;
model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g;
model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w;
model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w;
model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
size_t total_size = 0;
bool has_lm_head = false;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
// GPT-2 models share the WTE tensor as the LM head
if (name == "model/wte" && has_lm_head == false) {
memcpy(model.lm_head->data, tensor->data, ggml_nbytes(tensor));
}
if (name == "model/lm_head") {
has_lm_head = true;
}
total_size += ggml_nbytes(tensor);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size/1024.0/1024.0);
}
fin.close();
return true;
}
// evaluate the transformer
//
// - model: the model
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
bool gpt2_eval(
const gpt2_model & model,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w,
size_t & mem_per_token) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
const int n_vocab = hparams.n_vocab;
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate
buf_size = buf_size_new;
buf = realloc(buf, buf_size);
if (buf == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
return false;
}
}
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf,
/*.no_alloc =*/ false,
};
struct ggml_context * ctx0 = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
for (int i = 0; i < N; ++i) {
((int32_t *) position->data)[i] = n_past + i;
}
// wte + wpe
struct ggml_tensor * inpL =
ggml_add(ctx0,
ggml_get_rows(ctx0, model.wte, embd),
ggml_get_rows(ctx0, model.wpe, position));
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
// [ 768, N]
cur = ggml_norm(ctx0, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
// [ 768, N]
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
}
// attn
// [2304, 768] - model.layers[il].c_attn_attn_w
// [2304, 1] - model.layers[il].c_attn_attn_b
// [ 768, N] - cur (in)
// [2304, N] - cur (out)
//
// cur = attn_w*cur + attn_b
// [2304, N]
{
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_attn_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_attn_attn_b, cur),
cur);
}
// self-attention
{
struct ggml_tensor * Qcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 0*sizeof(float)*n_embd);
struct ggml_tensor * Kcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 1*sizeof(float)*n_embd);
struct ggml_tensor * Vcur = ggml_view_2d(ctx0, cur, n_embd, N, cur->nb[1], 2*sizeof(float)*n_embd);
// store key and value to memory
if (N >= 1) {
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_1d(ctx0, model.memory_v, N*n_embd, (ggml_element_size(model.memory_v)*n_embd)*(il*n_ctx + n_past));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
// [64, N, 12]
struct ggml_tensor * Q =
ggml_permute(ctx0,
ggml_cpy(ctx0,
Qcur,
ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd/n_head, n_head, N)),
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
// [64, n_past + N, 12]
struct ggml_tensor * K =
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// GG: flash attention
//struct ggml_tensor * V =
// ggml_cpy(ctx0,
// ggml_permute(ctx0,
// ggml_reshape_3d(ctx0,
// ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
// n_embd/n_head, n_head, n_past + N),
// 1, 2, 0, 3),
// ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_past + N, n_embd/n_head, n_head));
//struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, true);
// K * Q
// [n_past + N, N, 12]
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_scaled = ggml_scale_inplace(ctx0, KQ, 1.0f/sqrt(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
// [n_past + N, N, 12]
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
// [n_past + N, 64, 12]
struct ggml_tensor * V_trans =
ggml_cpy(ctx0,
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_v, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_v)*n_embd),
n_embd/n_head, n_head, n_past + N),
1, 2, 0, 3),
ggml_new_tensor_3d(ctx0, model.memory_v->type, n_past + N, n_embd/n_head, n_head));
// KQV = transpose(V) * KQ_soft_max
// [64, N, 12]
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V_trans, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
// [64, 12, N]
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
// [768, N]
cur = ggml_cpy(ctx0,
KQV_merged,
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
}
// projection
// [ 768, 768] - model.layers[il].c_attn_proj_w
// [ 768, 1] - model.layers[il].c_attn_proj_b
// [ 768, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
{
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_attn_proj_b, cur),
cur);
}
// add the input
cur = ggml_add(ctx0, cur, inpL);
struct ggml_tensor * inpFF = cur;
// feed-forward network
{
// norm
{
cur = ggml_norm(ctx0, inpFF, hparams.eps);
// cur = ln_2_g*cur + ln_2_b
// [ 768, N]
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_2_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_2_b, cur));
}
// fully connected
// [3072, 768] - model.layers[il].c_mlp_fc_w
// [3072, 1] - model.layers[il].c_mlp_fc_b
// [ 768, N] - cur (in)
// [3072, N] - cur (out)
//
// cur = fc_w*cur + fc_b
// [3072, N]
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_fc_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
// [3072, N]
cur = ggml_gelu(ctx0, cur);
// projection
// [ 768, 3072] - model.layers[il].c_mlp_proj_w
// [ 768, 1] - model.layers[il].c_mlp_proj_b
// [3072, N] - cur (in)
// [ 768, N] - cur (out)
//
// cur = proj_w*cur + proj_b
// [768, N]
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// input for next layer
inpL = ggml_add(ctx0, cur, inpFF);
}
// norm
{
// [ 768, N]
inpL = ggml_norm(ctx0, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
// [ 768, N]
inpL = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx0, model.ln_f_b, inpL));
}
// inpL = WTE * inpL
// [ 768, 50257] - model.lm_head
// [ 768, N] - inpL
inpL = ggml_mul_mat(ctx0, model.lm_head, inpL);
// logits -> probs
//inpL = ggml_soft_max_inplace(ctx0, inpL);
// run the computation
ggml_build_forward_expand(gf, inpL);
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-2.dot");
//}
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
// return result just for the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
if (mem_per_token == 0) {
mem_per_token = ggml_used_mem(ctx0)/N;
}
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
ggml_free(ctx0);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-2-117M/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gpt2_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: prompt: '%s'\n", __func__, params.prompt.c_str());
printf("%s: number of tokens in prompt = %zu, first 8 tokens: ", __func__, embd_inp.size());
for (int i = 0; i < std::min(8, (int) embd_inp.size()); i++) {
printf("%d ", embd_inp[i]);
}
printf("\n\n");
// submit the input prompt token-by-token
// this reduces the memory usage during inference, at the cost of a bit of speed at the beginning
std::vector<gpt_vocab::id> embd;
// determine the required inference memory per token:
size_t mem_per_token = 0;
gpt2_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gpt2_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) >= params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx_w);
return 0;
}
ggml/examples/gpt-2/main-sched.cpp
+1079
View File
File diff suppressed because it is too large Load Diff
ggml/examples/gpt-2/quantize.cpp
+184
View File
@@ -0,0 +1,184 @@
#include "ggml.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include <regex>
// default hparams (GPT-2 117M)
struct gpt2_hparams {
int32_t n_vocab = 50257;
int32_t n_ctx = 1024;
int32_t n_embd = 768;
int32_t n_head = 12;
int32_t n_layer = 12;
int32_t ftype = 1;
};
// quantize a model
bool gpt2_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
gpt_vocab vocab;
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
auto finp = std::ifstream(fname_inp, std::ios::binary);
if (!finp) {
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
return false;
}
auto fout = std::ofstream(fname_out, std::ios::binary);
if (!fout) {
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
return false;
}
// verify magic
{
uint32_t magic;
finp.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
return false;
}
fout.write((char *) &magic, sizeof(magic));
}
gpt2_hparams hparams;
// load hparams
{
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
}
// load vocab
{
int32_t n_vocab = 0;
finp.read ((char *) &n_vocab, sizeof(n_vocab));
fout.write((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
return false;
}
std::string word;
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
finp.read ((char *) &len, sizeof(len));
fout.write((char *) &len, sizeof(len));
word.resize(len);
finp.read ((char *) word.data(), len);
fout.write((char *) word.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// regexes of tensor names to be quantized
const std::vector<std::string> to_quant = {
"model/wte",
"model/lm_head",
"model/h.*/attn/c_attn/w",
"model/h.*/attn/c_proj/w",
"model/h.*/mlp/c_fc/w",
"model/h.*/mlp/c_proj/w",
};
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
return false;
}
finp.close();
fout.close();
return true;
}
// usage:
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
//
int main(int argc, char ** argv) {
if (argc != 4) {
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
ggml_print_ftypes(stderr);
return 1;
}
// needed to initialize f16 tables
{
struct ggml_init_params params = { 0, NULL, false };
struct ggml_context * ctx = ggml_init(params);
ggml_free(ctx);
}
const std::string fname_inp = argv[1];
const std::string fname_out = argv[2];
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
const int64_t t_main_start_us = ggml_time_us();
int64_t t_quantize_us = 0;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gpt2_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
return 1;
}
t_quantize_us = ggml_time_us() - t_start_us;
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n");
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
return 0;
}
ggml/examples/gpt-j/CMakeLists.txt
+13
View File
@@ -0,0 +1,13 @@
#
# gpt-j
set(TEST_TARGET gpt-j)
add_executable(${TEST_TARGET} main.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-j-quantize
set(TEST_TARGET gpt-j-quantize)
add_executable(${TEST_TARGET} quantize.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
ggml/examples/gpt-j/README.md
+239
View File
@@ -0,0 +1,239 @@
# gpt-j
Local GPT-J inference on your computer using C/C++
No video card required. You just need to have 16 GB of RAM.
## Motivation
The GPT-J 6B model is the open-source alternative to OpenAI's GPT-3. It's basically a neural network that allows you to
generate coherent, human-like text given a certain context (prompt).
The GPT-J model is quite big - the compact version of the model uses 16-bit floating point representation of the weights
and is still 12 GB big. This means that in order to run inference on your computer, you would need to have a video card
with at least 12 GB of video RAM. Alternatively, you can try to run the python implementations on the CPU, but that
would probably not be very efficient as they are primarily optimized for running on a GPU (or at least this is my guess -
I don't have much experience with python).
I wanted to try and run the model on my MacBook, so I decided to implement the model inference from scratch using my own
custom build tensor library. The tensor library (called [ggml](https://github.com/ggerganov/ggml), written in C) is in
early development stage, but it already allows me to run the GPT-J model.
On my 32GB MacBook M1 Pro, I achieve an inference speed of about `125 ms/token` or about ~6 words per second (1 word
typically consists of 1 or 2 tokens).
Here is a sample run with prompt `int main(int argc, char ** argv) {`:
```bash
$ time ./bin/gpt-j -p "int main(int argc, char ** argv) {"
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
gptj_model_load: n_vocab = 50400
gptj_model_load: n_ctx = 2048
gptj_model_load: n_embd = 4096
gptj_model_load: n_head = 16
gptj_model_load: n_layer = 28
gptj_model_load: n_rot = 64
gptj_model_load: f16 = 1
gptj_model_load: ggml ctx size = 13334.86 MB
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
gptj_model_load: ................................... done
gptj_model_load: model size = 11542.79 MB / num tensors = 285
main: number of tokens in prompt = 13
int main(int argc, char ** argv) {
(void)argc;
(void)argv;
{
struct sockaddr_in addr;
int addrlen;
char * ip = "192.168.1.4";
int i;
if ( (addrlen = sizeof(addr)) == -1 )
return -1;
for (i = 0; i < 10; ++i) {
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = inet_addr(ip);
main: mem per token = 16430420 bytes
main: load time = 6211.48 ms
main: sample time = 13.74 ms
main: predict time = 26420.34 ms / 124.62 ms per token
main: total time = 33035.37 ms
real 0m33.171s
user 3m32.269s
sys 0m3.686s
$
```
It took ~6.2 seconds to load the model to memory. After that, it took ~26.4 seconds to generate 200 tokens of what
looks like to be the beginning of a networking program in C. Pretty cool!
Here is another run, just for fun:
```bash
time ./bin/gpt-j -n 500 -t 8 -p "Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
"
gptj_model_load: loading model from 'models/gpt-j-6B/ggml-model.bin' - please wait ...
gptj_model_load: n_vocab = 50400
gptj_model_load: n_ctx = 2048
gptj_model_load: n_embd = 4096
gptj_model_load: n_head = 16
gptj_model_load: n_layer = 28
gptj_model_load: n_rot = 64
gptj_model_load: f16 = 1
gptj_model_load: ggml ctx size = 13334.86 MB
gptj_model_load: memory_size = 1792.00 MB, n_mem = 57344
gptj_model_load: ................................... done
gptj_model_load: model size = 11542.79 MB / num tensors = 285
main: number of tokens in prompt = 24
Ask HN: Inherited the worst code and tech team I have ever seen. How to fix it?
I've inherited a team with some very strange and un-documented practices, one of them is that they use an old custom
application with a very slow tech stack written in Python that the team doesn't want to touch but also doesn't want to
throw away as it has some "legacy" code in it.
The problem is, the tech stack is very very slow.
They have a single web server on a VM that is slow.
The server is a little bit busy (not very busy though) and they have a lot of processes (30+ that are constantly being
spawned by the application)
They have an application that is single threaded and was written in Python and the team don't want to touch this, and
the application is very slow.
My task as a new member of the team is to fix this.
I'm a senior dev on the team (3 years on the project) and have been told that I will take the lead on this task. I know
next to nothing about Python. So here is what I have so far.
What I have done is I've been trying to debug the processes with the "ps" command. This way I can see what is running
and where. From what I see, the application spawns 10 processes a minute and some of them are used for nothing.
I have also started to look for the code. The application source is not in GitHub or any other repository, it is only on
our internal GitLab.
What I've found so far:
The application uses a custom SQLAlchemy implementation to interact with the data. I've looked at the source, it looks
like an object cache or something like that. But from what I've seen, the cache gets full every 20 minutes and then gets
cleared with a special command.
Another strange thing is that the application creates a file for every entry in the database (even if the entry already
exists). I've looked at the file to see if it contains something, but it seems to be a JSON file with lots of records.
The other strange thing is that I can only find the database tables in the GitLab repository and not the code. So I
can't really understand how the application is supposed to interact with the database.
I also found a "log" directory, but the code is encrypted with AES. From what I've found, it is in
main: mem per token = 16430420 bytes
main: load time = 3900.10 ms
main: sample time = 32.58 ms
main: predict time = 68049.91 ms / 130.11 ms per token
main: total time = 73020.05 ms
real 1m13.156s
user 9m1.328s
sys. 0m7.103s
```
## Implementation details
The high level implementation of the model is contained in the [main.cpp](main.cpp) file. The core computations are
performed by the [ggml](https://github.com/ggerganov/ggml/blob/master/include/ggml.h) library.
#### Matrix multiplication
The most performance critical part of the implementation is of course the matrix multiplication routine. 99% of the time
is spent here, so it was important to optimize this as much as possible.
On Arm64, I utilize the 128-bit NEON intrinsics for 16-bit floating point operations:
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/src/ggml.c#L187-L243
These instructions allow each core to operate simultaneously on 64 16-bit floats. I'm no expert in SIMD, but after quite
some trials this was the most efficient code for dot product of a row and column that I could come up with. Combined
with the parallel computation on 8 CPU threads, I believe I'm close to the maximum performance that one could possibly
get on the M1 CPU. Still, I'm curious to know if there is a more efficient way to implement this.
#### Attempt to use the M1 GPU
One interesting property of the GPT-J transformer architecture is that it allows you to perform part of the inference in
parallel - i.e. the Feed-forward network can be computed in parallel to the Self-attention layer:
https://github.com/ggerganov/ggml/blob/fb558f78d905f85c54813602649ddd628ffe0f3a/examples/gpt-j/main.cpp#L507-L531
So I thought why not try and bring in the M1 GPU to compute half of the neural network in parallel to the CPU and
potentially gain some extra performance. Thanks to the M1's shared memory model, it was relatively easy to offload part
of the computation to the GPU using Apple's [Metal Performance
Shaders](https://developer.apple.com/documentation/metalperformanceshaders). The GPU shares the host memory, so there is
no need to copy the data back and forth as you would normally do with Cuda or OpenCL. The weight matrices are directly
available to be used by the GPU.
However, to my surprise, using MPS together with the CPU did not lead to any performance improvement at all. My
conclusion was that the 8-thread NEON CPU computation is already saturating the memory bandwidth of the M1 and since
the CPU and the GPU on the MacBook are sharing that bandwidth, it does not help to offload the computation to the GPU.
Another observation was that the MPS GPU matrix multiplication using 16-bit floats had the same performance as the
8-thread NEON CPU implementation. Again, I explain this with a saturated memory channel. But of course, my explanation
could be totally wrong and somehow the implementation wasn't utilizing the resources correctly.
In the end, I decided to not use MPS or the GPU all together.
### Zero memory allocations
Another property of my implementation is that it does not perform any memory allocations once the model is loaded into
memory. All required memory is allocated at the start of the program with a single `malloc` (technically 2 calls, but
that is not important).
## Usage
If you want to give this a try and you are on Linux or Mac OS, simply follow these instructions:
```bash
# Download the ggml-compatible GPT-J 6B model (requires 12GB disk space)
../examples/gpt-j/download-ggml-model.sh 6B
# Run the inference (requires 16GB of CPU RAM)
./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin -p "This is an example"
# Input prompt through pipe and run the inference.
echo "This is an example" > prompt.txt
cat prompt.txt | ./bin/gpt-j -m models/gpt-j-6B/ggml-model.bin
```
To run the `gpt-j` tool, you need the 12GB `ggml-model.bin` file which contains the GPT-J model in
[ggml](https://github.com/ggerganov/ggml) compatible format. In the instructions above, the binary file
is downloaded from my repository on Hugging Face using the [download-ggml-model.sh](download-ggml-model.sh) script.
You can also, download the file manually from this link:
https://huggingface.co/ggerganov/ggml/tree/main
---
Alternatively, if you don't want to download the 12GB ggml model file, you can perform the conversion yourself using
python.
First, you need to download the full GPT-J model from here: https://huggingface.co/EleutherAI/gpt-j-6B
Note that the full model is quite big - about 72 GB. After you download it, you need to convert it to ggml format using
the [convert-h5-to-ggml.py](convert-h5-to-ggml.py) script. This will generate the `ggml-model.bin` file, which you can
then use with the `gpt-j` program.
## GPT-2
I also implemented a tool for CPU inference using the smaller GPT-2 models. They have worse quality compared to GPT-J,
but are much faster to execute.
For example, the Small GPT-2 model is only 240 MB big and the inference speed on my MacBook is about 200 tokens/sec.
For more details, checkout the GPT-2 example here: [gpt-2](https://github.com/ggerganov/ggml/tree/master/examples/gpt-2)
ggml/examples/gpt-j/convert-h5-to-ggml.py
+173
View File
@@ -0,0 +1,173 @@
# Convert GPT-J-6B h5 transformer model to ggml format
#
# Load the model using GPTJForCausalLM.
# Iterate over all variables and write them to a binary file.
#
# For each variable, write the following:
# - Number of dimensions (int)
# - Name length (int)
# - Dimensions (int[n_dims])
# - Name (char[name_length])
# - Data (float[n_dims])
#
# By default, the bigger matrices are converted to 16-bit floats.
# This can be disabled by adding the "use-f32" CLI argument.
#
# At the start of the ggml file we write the model parameters
# and vocabulary.
#
import sys
import struct
import json
import torch
import numpy as np
from transformers import GPTJForCausalLM
# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py
def bytes_to_unicode():
"""
Returns list of utf-8 byte and a corresponding list of unicode strings.
The reversible bpe codes work on unicode strings.
This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
This is a signficant percentage of your normal, say, 32K bpe vocab.
To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
And avoids mapping to whitespace/control characters the bpe code barfs on.
"""
bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
cs = bs[:]
n = 0
for b in range(2**8):
if b not in bs:
bs.append(b)
cs.append(2**8+n)
n += 1
cs = [chr(n) for n in cs]
return dict(zip(bs, cs))
if len(sys.argv) < 3:
print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n")
print(" ftype == 0 -> float32")
print(" ftype == 1 -> float16")
sys.exit(1)
# output in the same directory as the model
dir_model = sys.argv[1]
fname_out = sys.argv[1] + "/ggml-model.bin"
with open(dir_model + "/vocab.json", "r", encoding="utf-8") as f:
encoder = json.load(f)
with open(dir_model + "/added_tokens.json", "r", encoding="utf-8") as f:
encoder_added = json.load(f)
with open(dir_model + "/config.json", "r", encoding="utf-8") as f:
hparams = json.load(f)
# possible data types
# ftype == 0 -> float32
# ftype == 1 -> float16
#
# map from ftype to string
ftype_str = ["f32", "f16"]
ftype = 1
if len(sys.argv) > 2:
ftype = int(sys.argv[2])
if ftype < 0 or ftype > 1:
print("Invalid ftype: " + str(ftype))
sys.exit(1)
fname_out = sys.argv[1] + "/ggml-model-" + ftype_str[ftype] + ".bin"
model = GPTJForCausalLM.from_pretrained(dir_model, low_cpu_mem_usage=True)
#print (model)
list_vars = model.state_dict()
#print (list_vars)
fout = open(fname_out, "wb")
fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex
fout.write(struct.pack("i", hparams["vocab_size"]))
fout.write(struct.pack("i", hparams["n_positions"]))
fout.write(struct.pack("i", hparams["n_embd"]))
fout.write(struct.pack("i", hparams["n_head"]))
fout.write(struct.pack("i", hparams["n_layer"]))
fout.write(struct.pack("i", hparams["rotary_dim"]))
fout.write(struct.pack("i", ftype))
byte_encoder = bytes_to_unicode()
byte_decoder = {v:k for k, v in byte_encoder.items()}
fout.write(struct.pack("i", len(encoder) + len(encoder_added)))
for key in encoder:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for key in encoder_added:
text = bytearray([byte_decoder[c] for c in key])
fout.write(struct.pack("i", len(text)))
fout.write(text)
for name in list_vars.keys():
data = list_vars[name].squeeze().numpy()
print("Processing variable: " + name + " with shape: ", data.shape)
# we don't need these
if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"):
print(" Skipping variable: " + name)
continue
n_dims = len(data.shape);
# ftype == 0 -> float32, ftype == 1 -> float16
ftype_cur = 0;
if ftype != 0:
if name[-7:] == ".weight" and n_dims == 2:
print(" Converting to float16")
data = data.astype(np.float16)
ftype_cur = 1
else:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
else:
if data.dtype != np.float32:
print(" Converting to float32")
data = data.astype(np.float32)
ftype_cur = 0
# for efficiency - transpose these matrices:
# (note - with latest ggml this is no longer more efficient, so disabling it)
# "transformer.h.*.mlp.fc_in.weight"
# "transformer.h.*.attn.out_proj.weight"
# "transformer.h.*.attn.q_proj.weight"
# "transformer.h.*.attn.k_proj.weight"
# "transformer.h.*.attn.v_proj.weight"
#if name.endswith(".mlp.fc_in.weight") or \
# name.endswith(".attn.out_proj.weight") or \
# name.endswith(".attn.q_proj.weight") or \
# name.endswith(".attn.k_proj.weight") or \
# name.endswith(".attn.v_proj.weight"):
# print(" Transposing")
# data = data.transpose()
# header
str = name.encode('utf-8')
fout.write(struct.pack("iii", n_dims, len(str), ftype_cur))
for i in range(n_dims):
fout.write(struct.pack("i", data.shape[n_dims - 1 - i]))
fout.write(str);
# data
data.tofile(fout)
fout.close()
print("Done. Output file: " + fname_out)
print("")
ggml/examples/gpt-j/download-ggml-model.sh
Executable
+69
View File
@@ -0,0 +1,69 @@
#!/bin/bash
# This script downloads GPT-J model files that have already been converted to ggml format.
# This way you don't have to convert them yourself.
#
# If you want to download the original GPT-J model files, use the "download-model.sh" script instead.
#src="https://ggml.ggerganov.com"
#pfx="ggml-model-gpt-j"
src="https://huggingface.co/ggerganov/ggml"
pfx="resolve/main/ggml-model-gpt-j"
ggml_path=$(dirname $(realpath $0))
# GPT-J models
models=( "6B" )
# list available models
function list_models {
printf "\n"
printf " Available models:"
for model in "${models[@]}"; do
printf " $model"
done
printf "\n\n"
}
if [ "$#" -ne 1 ]; then
printf "Usage: $0 <model>\n"
list_models
exit 1
fi
model=$1
if [[ ! " ${models[@]} " =~ " ${model} " ]]; then
printf "Invalid model: $model\n"
list_models
exit 1
fi
# download ggml model
printf "Downloading ggml model $model ...\n"
mkdir -p models/gpt-j-$model
if [ -x "$(command -v wget)" ]; then
wget --quiet --show-progress -O models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
elif [ -x "$(command -v curl)" ]; then
curl -L --output models/gpt-j-$model/ggml-model.bin $src/$pfx-$model.bin
else
printf "Either wget or curl is required to download models.\n"
exit 1
fi
if [ $? -ne 0 ]; then
printf "Failed to download ggml model $model \n"
printf "Please try again later or download the original GPT-J model files and convert them yourself.\n"
exit 1
fi
printf "Done! Model '$model' saved in 'models/gpt-j-$model/ggml-model.bin'\n"
printf "You can now use it like this:\n\n"
printf " $ ./bin/gpt-j -m models/gpt-j-$model/ggml-model.bin -p \"This is an example\"\n"
printf "\n"
ggml/examples/gpt-j/download-model.sh
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
printf "To obtain the GPT-J 6B model files, please visit: https://huggingface.co/EleutherAI/gpt-j-6B\n\n"
printf "The model is very big. For example, the reposirory above is 72GB in size.\n"
printf "If you are sure that you want to clone it, simply run the following command:\n\n"
printf " $ git clone https://huggingface.co/EleutherAI/gpt-j-6B models/gpt-j-6B\n\n"
printf "Alternatively, use the 'download-ggml-model.sh' script to download a 12GB ggml version of the model.\n"
printf "This version is enough to run inference using the ggml library.\n\n"
ggml/examples/gpt-j/main.cpp
+755
View File
@@ -0,0 +1,755 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
// default hparams (GPT-J 6B)
struct gptj_hparams {
int32_t n_vocab = 50400;
int32_t n_ctx = 2048;
int32_t n_embd = 4096;
int32_t n_head = 16;
int32_t n_layer = 28;
int32_t n_rot = 64;
int32_t ftype = 1;
float eps = 1e-5f;
};
struct gptj_layer {
// normalization
struct ggml_tensor * ln_1_g;
struct ggml_tensor * ln_1_b;
// attention
struct ggml_tensor * c_attn_q_proj_w;
struct ggml_tensor * c_attn_k_proj_w;
struct ggml_tensor * c_attn_v_proj_w;
struct ggml_tensor * c_attn_proj_w;
// ff
struct ggml_tensor * c_mlp_fc_w;
struct ggml_tensor * c_mlp_fc_b;
struct ggml_tensor * c_mlp_proj_w;
struct ggml_tensor * c_mlp_proj_b;
};
struct gptj_model {
gptj_hparams hparams;
// normalization
struct ggml_tensor * ln_f_g;
struct ggml_tensor * ln_f_b;
struct ggml_tensor * wte; // token embedding
struct ggml_tensor * lmh_g; // language model head
struct ggml_tensor * lmh_b; // language model bias
std::vector<gptj_layer> layers;
// key + value memory
struct ggml_tensor * memory_k;
struct ggml_tensor * memory_v;
//
struct ggml_context * ctx;
std::map<std::string, struct ggml_tensor *> tensors;
};
// load the model's weights from a file
bool gptj_model_load(const std::string & fname, gptj_model & model, gpt_vocab & vocab) {
printf("%s: loading model from '%s' - please wait ...\n", __func__, fname.c_str());
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "%s: failed to open '%s'\n", __func__, fname.c_str());
return false;
}
// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname.c_str());
return false;
}
}
// load hparams
{
auto & hparams = model.hparams;
fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char *) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
fin.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr = hparams.ftype / GGML_QNT_VERSION_FACTOR;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: n_rot = %d\n", __func__, hparams.n_rot);
printf("%s: ftype = %d\n", __func__, hparams.ftype);
printf("%s: qntvr = %d\n", __func__, qntvr);
hparams.ftype %= GGML_QNT_VERSION_FACTOR;
}
// load vocab
{
int32_t n_vocab = 0;
fin.read((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != model.hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname.c_str(), n_vocab, model.hparams.n_vocab);
return false;
}
std::string word;
std::vector<char> buf(128);
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
fin.read((char *) &len, sizeof(len));
buf.resize(len);
fin.read((char *) buf.data(), len);
word.assign(buf.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
ggml_type wtype = ggml_ftype_to_ggml_type((ggml_ftype) (model.hparams.ftype));
if (wtype == GGML_TYPE_COUNT) {
fprintf(stderr, "%s: invalid model file '%s' (bad ftype value %d)\n",
__func__, fname.c_str(), model.hparams.ftype);
return false;
}
auto & ctx = model.ctx;
size_t ctx_size = 0;
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_vocab = hparams.n_vocab;
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_embd); // ln_f_b
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // wte
ctx_size += ggml_row_size(wtype, n_embd*n_vocab); // lmh_g
ctx_size += ggml_row_size(GGML_TYPE_F32, n_vocab); // lmh_b
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_g
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // ln_1_b
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_q_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_k_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_v_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, n_embd*n_embd)); // c_attn_proj_w
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_fc_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, 4*n_embd)); // c_mlp_fc_b
ctx_size += n_layer*(ggml_row_size(wtype, 4*n_embd*n_embd)); // c_mlp_proj_w
ctx_size += n_layer*(ggml_row_size(GGML_TYPE_F32, n_embd)); // c_mlp_proj_b
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_k
ctx_size += n_ctx*n_layer*ggml_row_size(GGML_TYPE_F16, n_embd); // memory_v
ctx_size += (5 + 10*n_layer)*512; // object overhead
printf("%s: ggml ctx size = %6.2f MB\n", __func__, ctx_size/(1024.0*1024.0));
}
// create the ggml context
{
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
model.ctx = ggml_init(params);
if (!model.ctx) {
fprintf(stderr, "%s: ggml_init() failed\n", __func__);
return false;
}
}
// prepare memory for the weights
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_vocab = hparams.n_vocab;
model.layers.resize(n_layer);
model.wte = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.ln_f_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.ln_f_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.lmh_g = ggml_new_tensor_2d(ctx, wtype, n_embd, n_vocab);
model.lmh_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_vocab);
// map by name
model.tensors["transformer.wte.weight"] = model.wte;
model.tensors["transformer.ln_f.weight"] = model.ln_f_g;
model.tensors["transformer.ln_f.bias"] = model.ln_f_b;
model.tensors["lm_head.weight"] = model.lmh_g;
model.tensors["lm_head.bias"] = model.lmh_b;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
layer.ln_1_g = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.ln_1_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
layer.c_attn_q_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_k_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_v_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_attn_proj_w = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.c_mlp_fc_w = ggml_new_tensor_2d(ctx, wtype, n_embd, 4*n_embd);
layer.c_mlp_fc_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 4*n_embd);
layer.c_mlp_proj_w = ggml_new_tensor_2d(ctx, wtype, 4*n_embd, n_embd);
layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
// map by name
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.weight"] = layer.ln_1_g;
model.tensors["transformer.h." + std::to_string(i) + ".ln_1.bias"] = layer.ln_1_b;
model.tensors["transformer.h." + std::to_string(i) + ".attn.q_proj.weight"] = layer.c_attn_q_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.k_proj.weight"] = layer.c_attn_k_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.v_proj.weight"] = layer.c_attn_v_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".attn.out_proj.weight"] = layer.c_attn_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.weight"] = layer.c_mlp_fc_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_in.bias"] = layer.c_mlp_fc_b;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.weight"] = layer.c_mlp_proj_w;
model.tensors["transformer.h." + std::to_string(i) + ".mlp.fc_out.bias"] = layer.c_mlp_proj_b;
}
}
// key + value memory
{
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_mem = n_layer*n_ctx;
const int n_elements = n_embd*n_mem;
model.memory_k = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
model.memory_v = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, n_elements);
const size_t memory_size = ggml_nbytes(model.memory_k) + ggml_nbytes(model.memory_v);
printf("%s: memory_size = %8.2f MB, n_mem = %d\n", __func__, memory_size/1024.0/1024.0, n_mem);
}
// load weights
{
int n_tensors = 0;
size_t total_size = 0;
printf("%s: ", __func__);
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (fin.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
fin.read(&name[0], length);
if (model.tensors.find(name) == model.tensors.end()) {
fprintf(stderr, "%s: unknown tensor '%s' in model file\n", __func__, name.c_str());
return false;
}
auto tensor = model.tensors[name];
if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file\n", __func__, name.c_str());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, "%s: tensor '%s' has wrong shape in model file: got [%d, %d], expected [%d, %d]\n",
__func__, name.c_str(), (int) tensor->ne[0], (int) tensor->ne[1], ne[0], ne[1]);
return false;
}
// for debugging
if (0) {
printf("%24s - [%5d, %5d], type = %6s, %6.2f MB, %9zu bytes\n", name.c_str(), ne[0], ne[1], ggml_type_name(ggml_type(ttype)), ggml_nbytes(tensor)/1024.0/1024.0, ggml_nbytes(tensor));
}
const size_t bpe = ggml_type_size(ggml_type(ttype));
if ((nelements*bpe)/ggml_blck_size(tensor->type) != ggml_nbytes(tensor)) {
fprintf(stderr, "%s: tensor '%s' has wrong size in model file: got %zu, expected %zu\n",
__func__, name.c_str(), ggml_nbytes(tensor), nelements*bpe);
return false;
}
fin.read(reinterpret_cast<char *>(tensor->data), ggml_nbytes(tensor));
//printf("%42s - [%5d, %5d], type = %6s, %6.2f MB\n", name.c_str(), ne[0], ne[1], ttype == 0 ? "float" : "f16", ggml_nbytes(tensor)/1024.0/1024.0);
total_size += ggml_nbytes(tensor);
if (++n_tensors % 8 == 0) {
printf(".");
fflush(stdout);
}
}
printf(" done\n");
printf("%s: model size = %8.2f MB / num tensors = %d\n", __func__, total_size/1024.0/1024.0, n_tensors);
}
fin.close();
return true;
}
// evaluate the transformer
//
// - model: the model
// - n_threads: number of threads to use
// - n_past: the context size so far
// - embd_inp: the embeddings of the tokens in the context
// - embd_w: the predicted logits for the next token
//
// The GPT-J model requires about 16MB of memory per input token.
//
bool gptj_eval(
const gptj_model & model,
const int n_threads,
const int n_past,
const std::vector<gpt_vocab::id> & embd_inp,
std::vector<float> & embd_w,
size_t & mem_per_token) {
const int N = embd_inp.size();
const auto & hparams = model.hparams;
const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_ctx = hparams.n_ctx;
const int n_head = hparams.n_head;
const int n_vocab = hparams.n_vocab;
const int n_rot = hparams.n_rot;
static size_t buf_size = 256u*1024*1024;
static void * buf = malloc(buf_size);
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
// reallocate
buf_size = buf_size_new;
buf = realloc(buf, buf_size);
if (buf == nullptr) {
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
return false;
}
}
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf,
/*.no_alloc =*/ false,
};
struct ggml_context * ctx0 = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx0);
// KQ_pos - contains the positions
struct ggml_tensor * KQ_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
int * data = (int *) KQ_pos->data;
for (int i = 0; i < N; ++i) {
data[i] = n_past + i;
}
struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, N);
memcpy(embd->data, embd_inp.data(), N*ggml_element_size(embd));
// wte
struct ggml_tensor * inpL = ggml_get_rows(ctx0, model.wte, embd);
for (int il = 0; il < n_layer; ++il) {
struct ggml_tensor * cur;
// norm
{
cur = ggml_norm(ctx0, inpL, hparams.eps);
// cur = ln_1_g*cur + ln_1_b
cur = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.layers[il].ln_1_g, cur),
cur),
ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));
}
struct ggml_tensor * inpSA = cur;
// self-attention
{
struct ggml_tensor * Qcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_q_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
struct ggml_tensor * Kcur = ggml_rope_inplace(ctx0, ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_k_proj_w, cur), n_embd/n_head, n_head, N), KQ_pos, n_rot, 0);
// store key and value to memory
{
struct ggml_tensor * Vcur = ggml_transpose(ctx0, ggml_mul_mat(ctx0, model.layers[il].c_attn_v_proj_w, cur));
struct ggml_tensor * k = ggml_view_1d(ctx0, model.memory_k, N*n_embd, (ggml_element_size(model.memory_k)*n_embd)*(il*n_ctx + n_past));
struct ggml_tensor * v = ggml_view_2d(ctx0, model.memory_v, N, n_embd,
( n_ctx)*ggml_element_size(model.memory_v),
(il*n_ctx)*ggml_element_size(model.memory_v)*n_embd + n_past*ggml_element_size(model.memory_v));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Kcur, k));
ggml_build_forward_expand(gf, ggml_cpy(ctx0, Vcur, v));
}
// Q = Qcur.contiguous().view(n_embd/n_head, n_head, N).permute(0, 2, 1, 3)
struct ggml_tensor * Q =
ggml_permute(ctx0,
Qcur,
0, 2, 1, 3);
// K = Kmem.view(n_embd/n_head, n_head, n_past + N).permute(0, 2, 1, 3)
struct ggml_tensor * K =
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
ggml_view_1d(ctx0, model.memory_k, (n_past + N)*n_embd, il*n_ctx*ggml_element_size(model.memory_k)*n_embd),
n_embd/n_head, n_head, n_past + N),
0, 2, 1, 3);
// K * Q
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);
// KQ_scaled = KQ / sqrt(n_embd/n_head)
struct ggml_tensor * KQ_scaled =
ggml_scale_inplace(ctx0,
KQ,
1.0f/sqrt(float(n_embd)/n_head));
// KQ_masked = mask_past(KQ_scaled)
struct ggml_tensor * KQ_masked = ggml_diag_mask_inf_inplace(ctx0, KQ_scaled, n_past);
// KQ = soft_max(KQ_masked)
struct ggml_tensor * KQ_soft_max = ggml_soft_max_inplace(ctx0, KQ_masked);
// V_trans = Vmem.view(n_embd/n_head, n_head, n_past + N).permute(1, 2, 0, 3).contiguous()
struct ggml_tensor * V =
ggml_view_3d(ctx0, model.memory_v,
n_past + N, n_embd/n_head, n_head,
n_ctx*ggml_element_size(model.memory_v),
n_ctx*ggml_element_size(model.memory_v)*n_embd/n_head,
il*n_ctx*ggml_element_size(model.memory_v)*n_embd);
// KQV = transpose(V) * KQ_soft_max
struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
// KQV_merged = KQV.permute(0, 2, 1, 3)
struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);
// cur = KQV_merged.contiguous().view(n_embd, N)
cur = ggml_cpy(ctx0,
KQV_merged,
ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, N));
// projection (no bias)
cur = ggml_mul_mat(ctx0,
model.layers[il].c_attn_proj_w,
cur);
}
struct ggml_tensor * inpFF = cur;
// feed-forward network
// this is independent of the self-attention result, so it could be done in parallel to the self-attention
{
// note here we pass inpSA instead of cur
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_fc_w,
inpSA);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_fc_b, cur),
cur);
// GELU activation
cur = ggml_gelu(ctx0, cur);
// projection
// cur = proj_w*cur + proj_b
cur = ggml_mul_mat(ctx0,
model.layers[il].c_mlp_proj_w,
cur);
cur = ggml_add(ctx0,
ggml_repeat(ctx0, model.layers[il].c_mlp_proj_b, cur),
cur);
}
// self-attention + FF
cur = ggml_add(ctx0, cur, inpFF);
// input for next layer
inpL = ggml_add(ctx0, cur, inpL);
}
// norm
{
inpL = ggml_norm(ctx0, inpL, hparams.eps);
// inpL = ln_f_g*inpL + ln_f_b
inpL = ggml_add(ctx0,
ggml_mul(ctx0,
ggml_repeat(ctx0, model.ln_f_g, inpL),
inpL),
ggml_repeat(ctx0, model.ln_f_b, inpL));
}
// lm_head
{
inpL = ggml_mul_mat(ctx0, model.lmh_g, inpL);
inpL = ggml_add(ctx0,
ggml_repeat(ctx0, model.lmh_b, inpL),
inpL);
}
// logits -> probs
//inpL = ggml_soft_max_inplace(ctx0, inpL);
// run the computation
ggml_build_forward_expand(gf, inpL);
ggml_graph_compute_with_ctx(ctx0, gf, n_threads);
//if (n_past%100 == 0) {
// ggml_graph_print (&gf);
// ggml_graph_dump_dot(&gf, NULL, "gpt-j.dot");
//}
//embd_w.resize(n_vocab*N);
//memcpy(embd_w.data(), ggml_get_data(inpL), sizeof(float)*n_vocab*N);
// return result for just the last token
embd_w.resize(n_vocab);
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
if (mem_per_token == 0) {
mem_per_token = ggml_used_mem(ctx0)/N;
}
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
ggml_free(ctx0);
return true;
}
int main(int argc, char ** argv) {
ggml_time_init();
const int64_t t_main_start_us = ggml_time_us();
gpt_params params;
params.model = "models/gpt-j-6B/ggml-model.bin";
if (gpt_params_parse(argc, argv, params) == false) {
return 1;
}
if (params.seed < 0) {
params.seed = time(NULL);
}
printf("%s: seed = %d\n", __func__, params.seed);
std::mt19937 rng(params.seed);
if (params.prompt.empty()) {
params.prompt = gpt_random_prompt(rng);
}
int64_t t_load_us = 0;
gpt_vocab vocab;
gptj_model model;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gptj_model_load(params.model, model, vocab)) {
fprintf(stderr, "%s: failed to load model from '%s'\n", __func__, params.model.c_str());
return 1;
}
t_load_us = ggml_time_us() - t_start_us;
test_gpt_tokenizer(vocab, params.token_test);
}
int n_past = 0;
int64_t t_sample_us = 0;
int64_t t_predict_us = 0;
std::vector<float> logits;
// tokenize the prompt
std::vector<gpt_vocab::id> embd_inp = ::gpt_tokenize(vocab, params.prompt);
params.n_predict = std::min(params.n_predict, model.hparams.n_ctx - (int) embd_inp.size());
printf("%s: number of tokens in prompt = %zu\n", __func__, embd_inp.size());
printf("\n");
std::vector<gpt_vocab::id> embd;
// determine the required inference memory per token:
size_t mem_per_token = 0;
gptj_eval(model, params.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
for (size_t i = embd.size(); i < embd_inp.size() + params.n_predict; i++) {
// predict
if (embd.size() > 0) {
const int64_t t_start_us = ggml_time_us();
if (!gptj_eval(model, params.n_threads, n_past, embd, logits, mem_per_token)) {
printf("Failed to predict\n");
return 1;
}
t_predict_us += ggml_time_us() - t_start_us;
}
n_past += embd.size();
embd.clear();
if (i >= embd_inp.size()) {
// sample next token
const int top_k = params.top_k;
const float top_p = params.top_p;
const float temp = params.temp;
const int n_vocab = model.hparams.n_vocab;
gpt_vocab::id id = 0;
{
const int64_t t_start_sample_us = ggml_time_us();
id = gpt_sample_top_k_top_p(vocab, logits.data() + (logits.size() - n_vocab), top_k, top_p, temp, rng);
t_sample_us += ggml_time_us() - t_start_sample_us;
}
// add it to the context
embd.push_back(id);
} else {
// if here, it means we are still processing the input prompt
for (size_t k = i; k < embd_inp.size(); k++) {
embd.push_back(embd_inp[k]);
if (int32_t(embd.size()) > params.n_batch) {
break;
}
}
i += embd.size() - 1;
}
// display text
for (auto id : embd) {
printf("%s", vocab.id_to_token[id].c_str());
}
fflush(stdout);
// end of text token
if (embd.back() == 50256) {
break;
}
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n\n");
printf("%s: mem per token = %8zu bytes\n", __func__, mem_per_token);
printf("%s: load time = %8.2f ms\n", __func__, t_load_us/1000.0f);
printf("%s: sample time = %8.2f ms\n", __func__, t_sample_us/1000.0f);
printf("%s: predict time = %8.2f ms / %.2f ms per token\n", __func__, t_predict_us/1000.0f, t_predict_us/1000.0f/n_past);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
ggml_free(model.ctx);
return 0;
}
ggml/examples/gpt-j/quantize.cpp
+182
View File
@@ -0,0 +1,182 @@
#include "ggml.h"
#include "common.h"
#include "common-ggml.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <map>
#include <string>
#include <vector>
#include <regex>
// default hparams (GPT-J 6B)
struct gptj_hparams {
int32_t n_vocab = 50400;
int32_t n_ctx = 2048;
int32_t n_embd = 4096;
int32_t n_head = 16;
int32_t n_layer = 28;
int32_t n_rot = 64;
int32_t ftype = 1;
};
// quantize a model
bool gptj_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
gpt_vocab vocab;
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
auto finp = std::ifstream(fname_inp, std::ios::binary);
if (!finp) {
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
return false;
}
auto fout = std::ofstream(fname_out, std::ios::binary);
if (!fout) {
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
return false;
}
// verify magic
{
uint32_t magic;
finp.read((char *) &magic, sizeof(magic));
if (magic != GGML_FILE_MAGIC) {
fprintf(stderr, "%s: invalid model file '%s' (bad magic)\n", __func__, fname_inp.c_str());
return false;
}
fout.write((char *) &magic, sizeof(magic));
}
gptj_hparams hparams;
// load hparams
{
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
finp.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
finp.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
finp.read((char *) &hparams.n_head, sizeof(hparams.n_head));
finp.read((char *) &hparams.n_layer, sizeof(hparams.n_layer));
finp.read((char *) &hparams.n_rot, sizeof(hparams.n_rot));
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
printf("%s: n_vocab = %d\n", __func__, hparams.n_vocab);
printf("%s: n_ctx = %d\n", __func__, hparams.n_ctx);
printf("%s: n_embd = %d\n", __func__, hparams.n_embd);
printf("%s: n_head = %d\n", __func__, hparams.n_head);
printf("%s: n_layer = %d\n", __func__, hparams.n_layer);
printf("%s: ftype (src) = %d\n", __func__, hparams.ftype);
printf("%s: qntvr (src) = %d\n", __func__, qntvr_src);
printf("%s: ftype (dst) = %d\n", __func__, ftype_dst);
printf("%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
fout.write((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fout.write((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fout.write((char *) &hparams.n_head, sizeof(hparams.n_head));
fout.write((char *) &hparams.n_layer, sizeof(hparams.n_layer));
fout.write((char *) &hparams.n_rot, sizeof(hparams.n_rot));
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
}
// load vocab
{
int32_t n_vocab = 0;
finp.read ((char *) &n_vocab, sizeof(n_vocab));
fout.write((char *) &n_vocab, sizeof(n_vocab));
if (n_vocab != hparams.n_vocab) {
fprintf(stderr, "%s: invalid model file '%s' (bad vocab size %d != %d)\n",
__func__, fname_inp.c_str(), n_vocab, hparams.n_vocab);
return false;
}
std::string word;
for (int i = 0; i < n_vocab; i++) {
uint32_t len;
finp.read ((char *) &len, sizeof(len));
fout.write((char *) &len, sizeof(len));
word.resize(len);
finp.read ((char *) word.data(), len);
fout.write((char *) word.data(), len);
vocab.token_to_id[word] = i;
vocab.id_to_token[i] = word;
}
}
// regexes of tensor names to be quantized
const std::vector<std::string> to_quant = {
".*weight",
};
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, {})) {
fprintf(stderr, "%s: failed to quantize model '%s'\n", __func__, fname_inp.c_str());
return false;
}
finp.close();
fout.close();
return true;
}
// usage:
// ./gpt-2-quantize models/gpt-2-117M/ggml-model.bin models/gpt-2-117M/ggml-model-quant.bin type
//
int main(int argc, char ** argv) {
if (argc != 4) {
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
ggml_print_ftypes(stderr);
return 1;
}
// needed to initialize f16 tables
{
struct ggml_init_params params = { 0, NULL, false };
struct ggml_context * ctx = ggml_init(params);
ggml_free(ctx);
}
const std::string fname_inp = argv[1];
const std::string fname_out = argv[2];
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
const int64_t t_main_start_us = ggml_time_us();
int64_t t_quantize_us = 0;
// load the model
{
const int64_t t_start_us = ggml_time_us();
if (!gptj_model_quantize(fname_inp, fname_out, ggml_ftype(ftype))) {
fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
return 1;
}
t_quantize_us = ggml_time_us() - t_start_us;
}
// report timing
{
const int64_t t_main_end_us = ggml_time_us();
printf("\n");
printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0f);
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0f);
}
return 0;
}
ggml/examples/magika/CMakeLists.txt
+17
View File
@@ -0,0 +1,17 @@
#
# magika
set(TEST_TARGET magika)
add_executable(${TEST_TARGET} main.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# For GPU offloading
if (GGML_CUDA)
add_compile_definitions(GGML_USE_CUDA)
endif()
if (GGML_METAL)
add_compile_definitions(GGML_USE_METAL)
endif()
ggml/examples/magika/README.md
+23
View File
@@ -0,0 +1,23 @@
# Google Magika inference
Simple example that shows how to use GGML for inference with the [Google Magika](https://github.com/google/magika) file type detection model.
### Usage
- Obtain the Magika model in H5 format
- Pinned version: https://github.com/google/magika/blob/4460acb5d3f86807c3b53223229dee2afa50c025/assets_generation/models/standard_v1/model.h5
- Use `convert.py` to convert the model to gguf format:
```bash
$ python examples/magika/convert.py /path/to/model.h5
```
- Invoke the program with the model file and a list of files to identify:
```bash
$ build/bin/magika model.h5.gguf examples/sam/example.jpg examples/magika/convert.py README.md src/ggml.c /bin/gcc write.exe jfk.wav
examples/sam/example.jpg : jpeg (100.00%) pptx (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
examples/magika/convert.py : python (99.99%) javascript (0.00%) txt (0.00%) asm (0.00%) scala (0.00%)
README.md : markdown (100.00%) txt (0.00%) yaml (0.00%) ppt (0.00%) shell (0.00%)
src/ggml.c : c (99.95%) txt (0.04%) asm (0.01%) yaml (0.00%) html (0.00%)
/bin/gcc : elf (99.98%) odex (0.02%) pptx (0.00%) smali (0.00%) shell (0.00%)
write.exe : pebin (100.00%) ppt (0.00%) smali (0.00%) shell (0.00%) sevenzip (0.00%)
jfk.wav : wav (100.00%) ppt (0.00%) shell (0.00%) sevenzip (0.00%) scala (0.00%)
```
ggml/examples/magika/convert.py
+32
View File
@@ -0,0 +1,32 @@
import sys
from tensorflow import keras
import gguf
def convert(model_name):
model = keras.models.load_model(model_name, compile=False)
gguf_model_name = model_name + ".gguf"
gguf_writer = gguf.GGUFWriter(gguf_model_name, "magika")
for layer in model.layers:
# export layers with weights
if layer.weights:
for weight in layer.weights:
print(f" [{weight.name}] {weight.shape} {weight.dtype}")
weight_data = weight.numpy()
gguf_writer.add_tensor(weight.name, weight_data.T)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
print("Model converted and saved to '{}'".format(gguf_model_name))
if __name__ == '__main__':
if len(sys.argv) > 1:
model_file = sys.argv[1]
else:
model_file = "model.h5"
convert(model_file)
ggml/examples/magika/main.cpp
+374
View File
@@ -0,0 +1,374 @@
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpu.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include <algorithm>
#include <cmath>
#include <numeric>
#include <stdexcept>
#include <string>
#include <vector>
static const char * magika_labels[] = {
"ai", "apk", "appleplist", "asm", "asp",
"batch", "bmp", "bzip", "c", "cab",
"cat", "chm", "coff", "crx", "cs",
"css", "csv", "deb", "dex", "dmg",
"doc", "docx", "elf", "emf", "eml",
"epub", "flac", "gif", "go", "gzip",
"hlp", "html", "ico", "ini", "internetshortcut",
"iso", "jar", "java", "javabytecode", "javascript",
"jpeg", "json", "latex", "lisp", "lnk",
"m3u", "macho", "makefile", "markdown", "mht",
"mp3", "mp4", "mscompress", "msi", "mum",
"odex", "odp", "ods", "odt", "ogg",
"outlook", "pcap", "pdf", "pebin", "pem",
"perl", "php", "png", "postscript", "powershell",
"ppt", "pptx", "python", "pythonbytecode", "rar",
"rdf", "rpm", "rst", "rtf", "ruby",
"rust", "scala", "sevenzip", "shell", "smali",
"sql", "squashfs", "svg", "swf", "symlinktext",
"tar", "tga", "tiff", "torrent", "ttf",
"txt", "unknown", "vba", "wav", "webm",
"webp", "winregistry", "wmf", "xar", "xls",
"xlsb", "xlsx", "xml", "xpi", "xz",
"yaml", "zip", "zlibstream"
};
struct magika_hparams {
const int block_size = 4096;
const int beg_size = 512;
const int mid_size = 512;
const int end_size = 512;
const int min_file_size_for_dl = 16;
const int n_label = 113;
const float f_norm_eps = 0.001f;
const int padding_token = 256;
};
struct magika_model {
~magika_model() {
ggml_backend_buffer_free(buf_w);
ggml_backend_free(backend);
ggml_free(ctx_w);
}
magika_hparams hparams;
struct ggml_tensor * dense_w;
struct ggml_tensor * dense_b;
struct ggml_tensor * layer_norm_gamma;
struct ggml_tensor * layer_norm_beta;
struct ggml_tensor * dense_1_w;
struct ggml_tensor * dense_1_b;
struct ggml_tensor * dense_2_w;
struct ggml_tensor * dense_2_b;
struct ggml_tensor * layer_norm_1_gamma;
struct ggml_tensor * layer_norm_1_beta;
struct ggml_tensor * target_label_w;
struct ggml_tensor * target_label_b;
ggml_backend_t backend = ggml_backend_cpu_init();
ggml_backend_buffer_t buf_w = nullptr;
struct ggml_context * ctx_w = nullptr;
};
struct ggml_tensor * checked_get_tensor(struct ggml_context * ctx, const char * name) {
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
if (!tensor) {
fprintf(stderr, "%s: tensor '%s' not found\n", __func__, name);
throw std::runtime_error("ggml_get_tensor() failed");
}
return tensor;
}
bool magika_model_load(const std::string & fname, magika_model & model) {
auto & ctx = model.ctx_w;
struct gguf_init_params params = {
/*.no_alloc =*/ true,
/*.ctx =*/ &ctx,
};
struct gguf_context * ctx_gguf = gguf_init_from_file(fname.c_str(), params);
if (!ctx_gguf) {
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
return false;
}
model.buf_w = ggml_backend_alloc_ctx_tensors(ctx, model.backend);
if (!model.buf_w) {
fprintf(stderr, "%s: ggml_backend_alloc_ctx_tensors() failed\n", __func__);
gguf_free(ctx_gguf);
return false;
}
try {
model.dense_w = checked_get_tensor(ctx, "dense/kernel:0");
model.dense_b = checked_get_tensor(ctx, "dense/bias:0");
model.layer_norm_gamma = checked_get_tensor(ctx, "layer_normalization/gamma:0");
model.layer_norm_beta = checked_get_tensor(ctx, "layer_normalization/beta:0");
model.dense_1_w = checked_get_tensor(ctx, "dense_1/kernel:0");
model.dense_1_b = checked_get_tensor(ctx, "dense_1/bias:0");
model.dense_2_w = checked_get_tensor(ctx, "dense_2/kernel:0");
model.dense_2_b = checked_get_tensor(ctx, "dense_2/bias:0");
model.layer_norm_1_gamma = checked_get_tensor(ctx, "layer_normalization_1/gamma:0");
model.layer_norm_1_beta = checked_get_tensor(ctx, "layer_normalization_1/beta:0");
model.target_label_w = checked_get_tensor(ctx, "target_label/kernel:0");
model.target_label_b = checked_get_tensor(ctx, "target_label/bias:0");
} catch (const std::exception & e) {
fprintf(stderr, "%s: %s\n", __func__, e.what());
gguf_free(ctx_gguf);
return false;
}
FILE * f = fopen(fname.c_str(), "rb");
if (!f) {
fprintf(stderr, "%s: fopen() failed\n", __func__);
gguf_free(ctx_gguf);
return false;
}
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
for (int i = 0; i < n_tensors; i++) {
const char * name = gguf_get_tensor_name(ctx_gguf, i);
struct ggml_tensor * tensor = ggml_get_tensor(ctx, name);
size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
//printf("%-30s: [%3ld, %3ld, %3ld, %3ld] %s\n",
// name,
// tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->ne[3],
// ggml_type_name(tensor->type));
std::vector<uint8_t> buf(ggml_nbytes(tensor));
if (fseek(f, offs, SEEK_SET) != 0) {
fprintf(stderr, "%s: fseek() failed\n", __func__);
gguf_free(ctx_gguf);
fclose(f);
return false;
}
if (fread(buf.data(), 1, buf.size(), f) != buf.size()) {
fprintf(stderr, "%s: fread() failed\n", __func__);
gguf_free(ctx_gguf);
fclose(f);
return false;
}
ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size());
}
fclose(f);
gguf_free(ctx_gguf);
return true;
}
struct ggml_cgraph * magika_graph(
const magika_model & model,
const int n_files) {
const auto & hparams = model.hparams;
static size_t buf_size = ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead();
static std::vector<uint8_t> buf(buf_size);
struct ggml_init_params params = {
/*.mem_size =*/ buf_size,
/*.mem_buffer =*/ buf.data(),
/*.no_alloc =*/ true,
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_cgraph * gf = ggml_new_graph(ctx);
struct ggml_tensor * input = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 257, 1536, n_files); // one-hot
ggml_set_name(input, "input");
ggml_set_input(input);
struct ggml_tensor * cur;
// dense
cur = ggml_mul_mat(ctx, model.dense_w, input);
cur = ggml_add(ctx, cur, model.dense_b); // [128, 1536, n_files]
cur = ggml_gelu(ctx, cur);
// reshape
cur = ggml_reshape_3d(ctx, cur, 512, 384, n_files); // [384, 512, n_files]
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
// layer normalization
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
cur = ggml_mul(ctx, cur, model.layer_norm_gamma); // [384, 512, n_files]
cur = ggml_add(ctx, cur, model.layer_norm_beta); // [384, 512, n_files]
// dense_1
cur = ggml_cont(ctx, ggml_transpose(ctx, cur));
cur = ggml_mul_mat(ctx, model.dense_1_w, cur);
cur = ggml_add(ctx, cur, model.dense_1_b); // [256, 384, n_files]
cur = ggml_gelu(ctx, cur);
// dense_2
cur = ggml_mul_mat(ctx, model.dense_2_w, cur);
cur = ggml_add(ctx, cur, model.dense_2_b); // [256, 384, n_files]
cur = ggml_gelu(ctx, cur);
// global_max_pooling1d
cur = ggml_cont(ctx, ggml_transpose(ctx, cur)); // [384, 256, n_files]
cur = ggml_pool_1d(ctx, cur, GGML_OP_POOL_MAX, 384, 384, 0); // [1, 256, n_files]
cur = ggml_reshape_2d(ctx, cur, 256, n_files); // [256, n_files]
// layer normalization 1
cur = ggml_norm(ctx, cur, hparams.f_norm_eps);
cur = ggml_mul(ctx, cur, model.layer_norm_1_gamma); // [256, n_files]
cur = ggml_add(ctx, cur, model.layer_norm_1_beta); // [256, n_files]
// target_label
cur = ggml_mul_mat(ctx, model.target_label_w, cur);
cur = ggml_add(ctx, cur, model.target_label_b); // [n_label, n_files]
cur = ggml_soft_max(ctx, cur); // [n_label, n_files]
ggml_set_name(cur, "target_label_probs");
ggml_set_output(cur);
ggml_build_forward_expand(gf, cur);
return gf;
}
bool magika_eval(
struct magika_model & model,
const std::vector<std::string> & fnames) {
const auto & hparams = model.hparams;
static ggml_gallocr_t alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend));
struct ggml_cgraph * gf = magika_graph(model, fnames.size());
if (!ggml_gallocr_alloc_graph(alloc, gf)) {
fprintf(stderr, "%s: ggml_gallocr_alloc_graph() failed\n", __func__);
return false;
}
struct ggml_tensor * input = ggml_graph_get_tensor(gf, "input");
for (size_t i = 0; i < fnames.size(); i++) {
FILE * f = fopen(fnames[i].c_str(), "rb");
if (!f) {
fprintf(stderr, "%s: fopen() failed\n", __func__);
return false;
}
fseek(f, 0, SEEK_END);
long fsize = ftell(f);
// the buffer is padded with the padding_token if the file is smaller than the block size
std::vector<int> buf(1536, hparams.padding_token);
std::vector<uint8_t> read_buf(std::max(hparams.beg_size, std::max(hparams.mid_size, hparams.end_size)));
// read beg
fseek(f, 0, SEEK_SET);
int n_read = fread(read_buf.data(), 1, hparams.beg_size, f);
for (int j = 0; j < n_read; j++) {
// pad at the end
buf[j] = read_buf[j];
}
// read mid
long mid_offs = std::max(0L, (fsize - hparams.mid_size) / 2);
fseek(f, mid_offs, SEEK_SET);
n_read = fread(read_buf.data(), 1, hparams.mid_size, f);
for (int j = 0; j < n_read; j++) {
// pad at both ends
long mid_idx = hparams.beg_size + (hparams.mid_size / 2) - n_read / 2 + j;
buf[mid_idx] = read_buf[j];
}
// read end
long end_offs = std::max(0L, fsize - hparams.end_size);
fseek(f, end_offs, SEEK_SET);
n_read = fread(read_buf.data(), 1, hparams.end_size, f);
for (int j = 0; j < n_read; j++) {
// pad at the beginning
int end_idx = hparams.beg_size + hparams.mid_size + hparams.end_size - n_read + j;
buf[end_idx] = read_buf[j];
}
fclose(f);
const size_t inp_bytes = hparams.beg_size + hparams.mid_size + hparams.end_size;
// convert to one-hot
std::vector<float> one_hot(257*inp_bytes);
for (size_t j = 0; j < inp_bytes; j++) {
one_hot[257*j + buf[j]] = 1.0f;
}
ggml_backend_tensor_set(input, one_hot.data(), 257*inp_bytes*i*sizeof(float), 257*inp_bytes*sizeof(float));
}
if (ggml_backend_graph_compute(model.backend, gf) != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute() failed\n", __func__);
return false;
}
struct ggml_tensor * target_label_probs = ggml_graph_get_tensor(gf, "target_label_probs");
// print probabilities for the top labels of each file
for (size_t i = 0; i < fnames.size(); i++) {
std::vector<float> probs(hparams.n_label);
ggml_backend_tensor_get(target_label_probs, probs.data(), hparams.n_label*i*sizeof(float), hparams.n_label*sizeof(float));
// sort the probabilities
std::vector<int> idx(hparams.n_label);
std::iota(idx.begin(), idx.end(), 0);
std::sort(idx.begin(), idx.end(), [&probs](int i1, int i2) { return probs[i1] > probs[i2]; });
// print the top labels
const int top_n = 5;
printf("%-30s: ", fnames[i].c_str());
for (int j = 0; j < top_n; j++) {
printf("%s (%.2f%%) ", magika_labels[idx[j]], probs[idx[j]]*100);
}
printf("\n");
}
return true;
}
int main(int argc, const char ** argv) {
if (argc < 3) {
fprintf(stderr, "usage: %s <model> <file1> [<file2> ...]\n", argv[0]);
return 1;
}
const char * model_fname = argv[1];
std::vector<std::string> fnames;
for (int i = 2; i < argc; i++) {
fnames.push_back(argv[i]);
}
magika_model model;
if (!magika_model_load(model_fname, model)) {
fprintf(stderr, "magika_model_load() failed\n");
return 1;
}
magika_eval(model, fnames);
return 0;
}
ggml/examples/mnist/.gitignore
+3
View File
@@ -0,0 +1,3 @@
data/
*.gguf
*.ggml
ggml/examples/mnist/CMakeLists.txt
+58
View File
@@ -0,0 +1,58 @@
#
# mnist-common
set(TEST_TARGET mnist-common)
add_library(${TEST_TARGET} STATIC mnist-common.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common)
#
# mnist-eval
set(TEST_TARGET mnist-eval)
add_executable(${TEST_TARGET} mnist-eval.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
#
# mnist-train
set(TEST_TARGET mnist-train)
add_executable(${TEST_TARGET} mnist-train.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common mnist-common)
#
# mnist-wasm
if (EMSCRIPTEN)
set(TARGET mnist)
add_executable(${TARGET} mnist-common.cpp)
target_link_libraries(${TARGET} PRIVATE ggml ggml-cpu)
set_target_properties(${TARGET} PROPERTIES LINK_FLAGS " \
--bind \
-s FORCE_FILESYSTEM=1 \
-s USE_PTHREADS=1 \
-s PTHREAD_POOL_SIZE=10 \
-s ASSERTIONS=1 \
-s WASM=1 \
-s EXPORTED_RUNTIME_METHODS=\"['ccall', 'cwrap', 'setValue', 'getValue']\" \
-s EXPORTED_FUNCTIONS=\"['_wasm_eval','_wasm_random_digit','_malloc','_free']\" \
-s ALLOW_MEMORY_GROWTH=1 \
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/mnist-f32.gguf@/ \
--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/t10k-images-idx3-ubyte@/ \
")
# Copy output to web directory
add_custom_command(
TARGET ${TARGET} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.js
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.js
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.wasm
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.wasm
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_BINARY_DIR}/bin/mnist.worker.js
${CMAKE_CURRENT_SOURCE_DIR}/web/mnist.worker.js
)
endif()
ggml/examples/mnist/README.md
+206
View File
@@ -0,0 +1,206 @@
# MNIST Examples for GGML
This directory contains simple examples of how to use GGML for training and inference using the [MNIST dataset](https://yann.lecun.com/exdb/mnist/).
All commands listed in this README assume the working directory to be `examples/mnist`.
Please note that training in GGML is a work-in-progress and not production ready.
## Obtaining the data
A description of the dataset can be found on [Yann LeCun's website](https://yann.lecun.com/exdb/mnist/).
While it is also in principle possible to download the dataset from this website these downloads are frequently throttled and
it is recommended to use [HuggingFace](https://huggingface.co/datasets/ylecun/mnist) instead.
The dataset will be downloaded automatically when running `mnist-train-fc.py`.
## Fully connected network
For our first example we will train a fully connected network.
To train a fully connected model in PyTorch and save it as a GGUF file, run:
```bash
$ python3 mnist-train-fc.py mnist-fc-f32.gguf
...
Test loss: 0.066377+-0.010468, Test accuracy: 97.94+-0.14%
Model tensors saved to mnist-fc-f32.gguf:
fc1.weight (500, 784)
fc1.bias (500,)
fc2.weight (10, 500)
fc2.bias (10,)
```
The training script includes an evaluation of the model on the test set.
To evaluate the model on the CPU using GGML, run:
```bash
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
__________________________________####__________________
______________________________########__________________
__________________________##########____________________
______________________##############____________________
____________________######________####__________________
__________________________________####__________________
__________________________________####__________________
________________________________####____________________
______________________________####______________________
________________________##########______________________
______________________########__####____________________
________________________##__________##__________________
____________________________________##__________________
__________________________________##____________________
__________________________________##____________________
________________________________##______________________
____________________________####________________________
__________##____________######__________________________
__________##############________________________________
________________####____________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
mnist_model_init_from_file: loading model weights from 'mnist-fc-f32.gguf'
mnist_model_init_from_file: model arch is mnist-fc
mnist_model_init_from_file: successfully loaded weights from mnist-fc-f32.gguf
main: loaded model in 109.44 ms
mnist_model_eval: model evaluation on 10000 images took 76.92 ms, 7.69 us/image
main: predicted digit is 3
main: test_loss=0.066379+-0.009101
main: test_acc=97.94+-0.14%
```
In addition to the evaluation on the test set the GGML evaluation also prints a random image from the test set as well as the model prediction for said image.
To train a fully connected model on the CPU using GGML run:
``` bash
$ ../../build/bin/mnist-train mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
```
It can then be evaluated with the same binary as above.
## Convolutional network
To train a convolutional network using TensorFlow run:
```bash
$ python3 mnist-train-cnn.py mnist-cnn-f32.gguf
...
Test loss: 0.047947
Test accuracy: 98.46%
GGUF model saved to 'mnist-cnn-f32.gguf'
```
The saved model can be evaluated on the CPU using the `mnist-eval` binary:
```bash
$ ../../build/bin/mnist-eval mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
______________________________________##________________
______________________________________##________________
______________________________________##________________
____________________________________##__________________
__________________________________####__________________
__________________________________##____________________
________________________________##______________________
______________________________##________________________
____________________________####________________________
____________________________##__________________________
__________________________##____________________________
________________________##______________________________
______________________##________________________________
____________________####________________________________
____________________##__________________________________
__________________##____________________________________
________________##______________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
________________________________________________________
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no
ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no
ggml_cuda_init: found 1 CUDA devices:
Device 0: NVIDIA GeForce RTX 3090, compute capability 8.6, VMM: yes
mnist_model: using CUDA0 (NVIDIA GeForce RTX 3090) as primary backend
mnist_model: unsupported operations will be executed on the following fallback backends (in order of priority):
mnist_model: - CPU (AMD Ryzen 9 5950X 16-Core Processor)
mnist_model_init_from_file: loading model weights from 'mnist-cnn-f32.gguf'
mnist_model_init_from_file: model arch is mnist-cnn
mnist_model_init_from_file: successfully loaded weights from mnist-cnn-f32.gguf
main: loaded model in 91.99 ms
mnist_model_eval: model evaluation on 10000 images took 267.61 ms, 26.76 us/image
main: predicted digit is 1
main: test_loss=0.047955+-0.007029
main: test_acc=98.46+-0.12%
```
Like with the fully connected network the convolutional network can also be trained using GGML:
``` bash
$ ../../build/bin/mnist-train mnist-cnn mnist-cnn-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte
```
As always, the evaluation is done using `mnist-eval` and like with the fully connected network the GGML graph is exported to `mnist-cnn-f32.ggml`.
## Hardware Acceleration
Both the training and evaluation code is agnostic in terms of hardware as long as the corresponding GGML backend has implemented the necessary operations.
A specific backend can be selected by appending the above commands with a backend name.
The compute graphs then schedule the operations to preferentially use the specified backend.
Note that if a backend does not implement some of the necessary operations a CPU fallback is used instead which may result in bad performance.
## Web demo
The evaluation code can be compiled to WebAssembly using [Emscripten](https://emscripten.org/) (may need to re-login to update `$PATH` after installation).
First, copy the GGUF file of either of the trained models to `examples/mnist` and name it `mnist-f32.gguf`.
Copy the test set to `examples/mnist` and name it `t10k-images-idx3-ubyte`.
Symlinking these files will *not* work!
Compile the code like so:
```bash
$ cd ../../
$ mkdir -p build-em
$ emcmake cmake .. -DGGML_BUILD_EXAMPLES=ON \
-DCMAKE_C_FLAGS="-pthread -matomics -mbulk-memory" \
-DCMAKE_CXX_FLAGS="-pthread -matomics -mbulk-memory"
$ make mnist
```
The compilation output is copied into `examples/mnist/web`.
To run it, you need an HTTP server.
For example:
``` bash
$ python3 examples/mnist/server.py
Serving directory '/home/danbev/work/ai/ggml/examples/mnist/web' at http://localhost:8000
Application context root: http://localhost:8000/
```
The web demo can then be accessed via the link printed on the console.
Simply draw a digit on the canvas and the model will try to predict what it's supposed to be.
Alternatively, click the "Random" button to retrieve a random digit from the test set.
Be aware that like all neural networks the one we trained is susceptible to distributional shift:
if the numbers you draw look different than the ones in the training set
(e.g. because they're not centered) the model will perform comparatively worse.
An online demo can be accessed [here](https://mnist.ggerganov.com).
ggml/examples/mnist/mnist-common.cpp
+496
View File
@@ -0,0 +1,496 @@
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-opt.h"
#include "mnist-common.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <fstream>
#include <random>
#include <string>
#include <utility>
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset) {
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open images file %s\n", fname.c_str());
return false;
}
fin.seekg(16);
uint8_t image[MNIST_NINPUT];
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
float * buf = ggml_get_data_f32(images);
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
for (int64_t iex = 0; iex < images->ne[1]; ++iex) {
fin.read((char *) image, sizeof(image));
for (int64_t i = 0; i < MNIST_NINPUT; ++i) {
buf[iex*MNIST_NINPUT + i] = image[i] / 255.0f; // Normalize to [0, 1]
}
}
return true;
}
void mnist_image_print(FILE * stream, ggml_opt_dataset_t dataset, const int iex) {
struct ggml_tensor * images = ggml_opt_dataset_data(dataset);
GGML_ASSERT(images->ne[0] == MNIST_NINPUT);
GGML_ASSERT(iex < images->ne[1]);
const float * image = ggml_get_data_f32(images) + iex*MNIST_NINPUT;
for (int64_t row = 0; row < MNIST_HW; row++) {
for (int64_t col = 0; col < MNIST_HW; col++) {
const int rgb = roundf(255.0f * image[row*MNIST_HW + col]);
#ifdef _WIN32
fprintf(stream, "%s", rgb >= 220 ? "##" : "__"); // Represented via text.
#else
fprintf(stream, "\033[48;2;%d;%d;%dm \033[0m", rgb, rgb, rgb); // Represented via colored blocks.
#endif // _WIN32
}
fprintf(stream, "\n");
}
}
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset) {
auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open labels file %s\n", fname.c_str());
return 0;
}
fin.seekg(8);
uint8_t label;
struct ggml_tensor * labels = ggml_opt_dataset_labels(dataset);
float * buf = ggml_get_data_f32(labels);
GGML_ASSERT(labels->ne[0] == MNIST_NCLASSES);
for (int64_t iex = 0; iex < labels->ne[1]; ++iex) {
fin.read((char *) &label, sizeof(label));
for (int64_t i = 0; i < MNIST_NCLASSES; ++i) {
buf[iex*MNIST_NCLASSES + i] = i == label ? 1.0f : 0.0f;
}
}
return true;
}
// Temporary util function for loading data from GGUF to a backend != CPU until GGML itself provides this functionality:
bool load_from_gguf(const char * fname, struct ggml_context * ctx_ggml, struct gguf_context * ctx_gguf) {
FILE * f = ggml_fopen(fname, "rb");
if (!f) {
return false;
}
const size_t buf_size = 4*1024*1024;
void * buf = malloc(buf_size);
const int n_tensors = gguf_get_n_tensors(ctx_gguf);
for (int i = 0; i < n_tensors; i++) {
const char * name = gguf_get_tensor_name(ctx_gguf, i);
struct ggml_tensor * tensor = ggml_get_tensor(ctx_ggml, name);
if (!tensor) {
continue;
}
const size_t offs = gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, i);
if (fseek(f, offs, SEEK_SET) != 0) {
fclose(f);
free(buf);
return false;
}
const size_t nbytes = ggml_nbytes(tensor);
for (size_t pos = 0; pos < nbytes; pos += buf_size) {
const size_t nbytes_cpy = buf_size < nbytes - pos ? buf_size : nbytes - pos;
if (fread(buf, 1, nbytes_cpy, f) != nbytes_cpy) {
fclose(f);
free(buf);
return false;
}
ggml_backend_tensor_set(tensor, buf, pos, nbytes_cpy);
}
}
fclose(f);
free(buf);
return true;
}
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
mnist_model model(backend, nbatch_logical, nbatch_physical);
fprintf(stderr, "%s: loading model weights from '%s'\n", __func__, fname.c_str());
struct gguf_context * ctx;
{
struct gguf_init_params params = {
/*.no_alloc =*/ true,
/*.ctx =*/ &model.ctx_gguf,
};
ctx = gguf_init_from_file(fname.c_str(), params);
if (!ctx) {
fprintf(stderr, "%s: gguf_init_from_file() failed\n", __func__);
exit(1);
}
}
model.arch = gguf_get_val_str(ctx, gguf_find_key(ctx, "general.architecture"));
fprintf(stderr, "%s: model arch is %s\n", __func__, model.arch.c_str());
if (model.arch == "mnist-fc") {
model.fc1_weight = ggml_get_tensor(model.ctx_gguf, "fc1.weight");
GGML_ASSERT(model.fc1_weight->ne[0] == MNIST_NINPUT);
GGML_ASSERT(model.fc1_weight->ne[1] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc1_weight->ne[2] == 1);
GGML_ASSERT(model.fc1_weight->ne[3] == 1);
model.fc1_bias = ggml_get_tensor(model.ctx_gguf, "fc1.bias");
GGML_ASSERT(model.fc1_bias->ne[0] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc1_bias->ne[1] == 1);
GGML_ASSERT(model.fc1_bias->ne[2] == 1);
GGML_ASSERT(model.fc1_bias->ne[3] == 1);
model.fc2_weight = ggml_get_tensor(model.ctx_gguf, "fc2.weight");
GGML_ASSERT(model.fc2_weight->ne[0] == MNIST_NHIDDEN);
GGML_ASSERT(model.fc2_weight->ne[1] == MNIST_NCLASSES);
GGML_ASSERT(model.fc2_weight->ne[2] == 1);
GGML_ASSERT(model.fc2_weight->ne[3] == 1);
model.fc2_bias = ggml_get_tensor(model.ctx_gguf, "fc2.bias");
GGML_ASSERT(model.fc2_bias->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.fc2_bias->ne[1] == 1);
GGML_ASSERT(model.fc2_bias->ne[2] == 1);
GGML_ASSERT(model.fc2_bias->ne[3] == 1);
} else if (model.arch == "mnist-cnn") {
model.conv1_kernel = ggml_get_tensor(model.ctx_gguf, "conv1.kernel");
GGML_ASSERT(model.conv1_kernel->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv1_kernel->ne[0] == 3);
GGML_ASSERT(model.conv1_kernel->ne[1] == 3);
GGML_ASSERT(model.conv1_kernel->ne[2] == 1);
GGML_ASSERT(model.conv1_kernel->ne[3] == MNIST_CNN_NCB);
model.conv1_bias = ggml_get_tensor(model.ctx_gguf, "conv1.bias");
GGML_ASSERT(model.conv1_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv1_bias->ne[0] == 1);
GGML_ASSERT(model.conv1_bias->ne[1] == 1);
GGML_ASSERT(model.conv1_bias->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(model.conv1_bias->ne[3] == 1);
model.conv2_kernel = ggml_get_tensor(model.ctx_gguf, "conv2.kernel");
GGML_ASSERT(model.conv2_kernel->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv2_kernel->ne[0] == 3);
GGML_ASSERT(model.conv2_kernel->ne[1] == 3);
GGML_ASSERT(model.conv2_kernel->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(model.conv2_kernel->ne[3] == MNIST_CNN_NCB*2);
model.conv2_bias = ggml_get_tensor(model.ctx_gguf, "conv2.bias");
GGML_ASSERT(model.conv2_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.conv2_bias->ne[0] == 1);
GGML_ASSERT(model.conv2_bias->ne[1] == 1);
GGML_ASSERT(model.conv2_bias->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(model.conv2_bias->ne[3] == 1);
model.dense_weight = ggml_get_tensor(model.ctx_gguf, "dense.weight");
GGML_ASSERT(model.dense_weight->type == GGML_TYPE_F32);
GGML_ASSERT(model.dense_weight->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
GGML_ASSERT(model.dense_weight->ne[1] == MNIST_NCLASSES);
GGML_ASSERT(model.dense_weight->ne[2] == 1);
GGML_ASSERT(model.dense_weight->ne[3] == 1);
model.dense_bias = ggml_get_tensor(model.ctx_gguf, "dense.bias");
GGML_ASSERT(model.dense_bias->type == GGML_TYPE_F32);
GGML_ASSERT(model.dense_bias->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.dense_bias->ne[1] == 1);
GGML_ASSERT(model.dense_bias->ne[2] == 1);
GGML_ASSERT(model.dense_bias->ne[3] == 1);
} else {
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
}
model.buf_gguf = ggml_backend_alloc_ctx_tensors(model.ctx_gguf, model.backends[0]);
if(!load_from_gguf(fname.c_str(), model.ctx_gguf, ctx)) {
fprintf(stderr, "%s: loading weights from %s failed\n", __func__, fname.c_str());
exit(1);
}
// The space in ctx_gguf exactly fits the model weights,
// the images (which also need to be statically allocated) need to be put in a different context.
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, nbatch_physical);
ggml_set_name(model.images, "images");
ggml_set_input(model.images);
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
fprintf(stderr, "%s: successfully loaded weights from %s\n", __func__, fname.c_str());
return model;
}
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical) {
mnist_model model(backend, nbatch_logical, nbatch_physical);
model.arch = arch;
std::random_device rd{};
std::mt19937 gen{rd()};
std::normal_distribution<float> nd{0.0f, 1e-2f};
std::vector<ggml_tensor *> init_tensors;
if (model.arch == "mnist-fc") {
fprintf(stderr, "%s: initializing random weights for a fully connected model\n", __func__);
model.fc1_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NHIDDEN);
model.fc1_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN);
model.fc2_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NHIDDEN, MNIST_NCLASSES);
model.fc2_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
ggml_set_name(model.fc1_weight, "fc1.weight");
ggml_set_name(model.fc1_bias, "fc1.bias");
ggml_set_name(model.fc2_weight, "fc2.weight");
ggml_set_name(model.fc2_bias, "fc2.bias");
init_tensors.push_back(model.fc1_weight);
init_tensors.push_back(model.fc1_bias);
init_tensors.push_back(model.fc2_weight);
init_tensors.push_back(model.fc2_bias);
} else if (model.arch == "mnist-cnn") {
model.conv1_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, 1, MNIST_CNN_NCB);
model.conv1_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB);
model.conv2_kernel = ggml_new_tensor_4d(model.ctx_static, GGML_TYPE_F32, 3, 3, MNIST_CNN_NCB, MNIST_CNN_NCB*2);
model.conv2_bias = ggml_new_tensor_3d(model.ctx_static, GGML_TYPE_F32, 1, 1, MNIST_CNN_NCB*2);
model.dense_weight = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), MNIST_NCLASSES);
model.dense_bias = ggml_new_tensor_1d(model.ctx_static, GGML_TYPE_F32, MNIST_NCLASSES);
ggml_set_name(model.conv1_kernel, "conv1.kernel");
ggml_set_name(model.conv1_bias, "conv1.bias");
ggml_set_name(model.conv2_kernel, "conv2.kernel");
ggml_set_name(model.conv2_bias, "conv2.bias");
ggml_set_name(model.dense_weight, "dense.weight");
ggml_set_name(model.dense_bias, "dense.bias");
init_tensors.push_back(model.conv1_kernel);
init_tensors.push_back(model.conv1_bias);
init_tensors.push_back(model.conv2_kernel);
init_tensors.push_back(model.conv2_bias);
init_tensors.push_back(model.dense_weight);
init_tensors.push_back(model.dense_bias);
} else {
fprintf(stderr, "%s: unknown model arch: %s\n", __func__, model.arch.c_str());
}
model.images = ggml_new_tensor_2d(model.ctx_static, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NBATCH_PHYSICAL);
ggml_set_name(model.images, "images");
ggml_set_input(model.images);
model.buf_static = ggml_backend_alloc_ctx_tensors(model.ctx_static, model.backends[0]);
for (ggml_tensor * t : init_tensors) {
GGML_ASSERT(t->type == GGML_TYPE_F32);
const int64_t ne = ggml_nelements(t);
std::vector<float> tmp(ne);
for (int64_t i = 0; i < ne; ++i) {
tmp[i] = nd(gen);
}
ggml_backend_tensor_set(t, tmp.data(), 0, ggml_nbytes(t));
}
return model;
}
void mnist_model_build(mnist_model & model) {
if (model.arch == "mnist-fc") {
ggml_set_param(model.fc1_weight);
ggml_set_param(model.fc1_bias);
ggml_set_param(model.fc2_weight);
ggml_set_param(model.fc2_bias);
ggml_tensor * fc1 = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_mul_mat(model.ctx_compute, model.fc1_weight, model.images),
model.fc1_bias));
model.logits = ggml_add(model.ctx_compute,
ggml_mul_mat(model.ctx_compute, model.fc2_weight, fc1),
model.fc2_bias);
} else if (model.arch == "mnist-cnn") {
ggml_set_param(model.conv1_kernel);
ggml_set_param(model.conv1_bias);
ggml_set_param(model.conv2_kernel);
ggml_set_param(model.conv2_bias);
ggml_set_param(model.dense_weight);
ggml_set_param(model.dense_bias);
struct ggml_tensor * images_2D = ggml_reshape_4d(model.ctx_compute, model.images, MNIST_HW, MNIST_HW, 1, model.images->ne[1]);
struct ggml_tensor * conv1_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_conv_2d(model.ctx_compute, model.conv1_kernel, images_2D, 1, 1, 1, 1, 1, 1),
model.conv1_bias));
GGML_ASSERT(conv1_out->ne[0] == MNIST_HW);
GGML_ASSERT(conv1_out->ne[1] == MNIST_HW);
GGML_ASSERT(conv1_out->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(conv1_out->ne[3] == model.nbatch_physical);
struct ggml_tensor * conv2_in = ggml_pool_2d(model.ctx_compute, conv1_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
GGML_ASSERT(conv2_in->ne[0] == MNIST_HW/2);
GGML_ASSERT(conv2_in->ne[1] == MNIST_HW/2);
GGML_ASSERT(conv2_in->ne[2] == MNIST_CNN_NCB);
GGML_ASSERT(conv2_in->ne[3] == model.nbatch_physical);
struct ggml_tensor * conv2_out = ggml_relu(model.ctx_compute, ggml_add(model.ctx_compute,
ggml_conv_2d(model.ctx_compute, model.conv2_kernel, conv2_in, 1, 1, 1, 1, 1, 1),
model.conv2_bias));
GGML_ASSERT(conv2_out->ne[0] == MNIST_HW/2);
GGML_ASSERT(conv2_out->ne[1] == MNIST_HW/2);
GGML_ASSERT(conv2_out->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(conv2_out->ne[3] == model.nbatch_physical);
struct ggml_tensor * dense_in = ggml_pool_2d(model.ctx_compute, conv2_out, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
GGML_ASSERT(dense_in->ne[0] == MNIST_HW/4);
GGML_ASSERT(dense_in->ne[1] == MNIST_HW/4);
GGML_ASSERT(dense_in->ne[2] == MNIST_CNN_NCB*2);
GGML_ASSERT(dense_in->ne[3] == model.nbatch_physical);
dense_in = ggml_reshape_2d(model.ctx_compute,
ggml_cont(model.ctx_compute, ggml_permute(model.ctx_compute, dense_in, 1, 2, 0, 3)),
(MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2), model.nbatch_physical);
GGML_ASSERT(dense_in->ne[0] == (MNIST_HW/4)*(MNIST_HW/4)*(MNIST_CNN_NCB*2));
GGML_ASSERT(dense_in->ne[1] == model.nbatch_physical);
GGML_ASSERT(dense_in->ne[2] == 1);
GGML_ASSERT(dense_in->ne[3] == 1);
model.logits = ggml_add(model.ctx_compute, ggml_mul_mat(model.ctx_compute, model.dense_weight, dense_in), model.dense_bias);
} else {
GGML_ASSERT(false);
}
ggml_set_name(model.logits, "logits");
ggml_set_output(model.logits);
GGML_ASSERT(model.logits->type == GGML_TYPE_F32);
GGML_ASSERT(model.logits->ne[0] == MNIST_NCLASSES);
GGML_ASSERT(model.logits->ne[1] == model.nbatch_physical);
GGML_ASSERT(model.logits->ne[2] == 1);
GGML_ASSERT(model.logits->ne[3] == 1);
}
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset) {
ggml_opt_result_t result = ggml_opt_result_init();
ggml_opt_params params = ggml_opt_default_params(model.backend_sched, GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
params.ctx_compute = model.ctx_compute;
params.inputs = model.images;
params.outputs = model.logits;
params.build_type = GGML_OPT_BUILD_TYPE_FORWARD;
ggml_opt_context_t opt_ctx = ggml_opt_init(params);
{
const int64_t t_start_us = ggml_time_us();
ggml_opt_epoch(opt_ctx, dataset, nullptr, result, /*idata_split =*/ 0, nullptr, nullptr);
const int64_t t_total_us = ggml_time_us() - t_start_us;
const double t_total_ms = 1e-3*t_total_us;
const int nex = ggml_opt_dataset_data(dataset)->ne[1];
fprintf(stderr, "%s: model evaluation on %d images took %.2lf ms, %.2lf us/image\n",
__func__, nex, t_total_ms, (double) t_total_us/nex);
}
ggml_opt_free(opt_ctx);
return result;
}
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split) {
ggml_opt_fit(model.backend_sched, model.ctx_compute, model.images, model.logits, dataset,
GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, GGML_OPT_OPTIMIZER_TYPE_ADAMW, ggml_opt_get_default_optimizer_params, nepoch, model.nbatch_logical, val_split, false);
}
void mnist_model_save(mnist_model & model, const std::string & fname) {
printf("%s: saving model to '%s'\n", __func__, fname.c_str());
struct ggml_context * ggml_ctx;
{
struct ggml_init_params params = {
/*.mem_size =*/ 100 * 1024*1024,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ false,
};
ggml_ctx = ggml_init(params);
}
gguf_context * gguf_ctx = gguf_init_empty();
gguf_set_val_str(gguf_ctx, "general.architecture", model.arch.c_str());
std::vector<struct ggml_tensor *> weights;
if (model.arch == "mnist-fc") {
weights = {model.fc1_weight, model.fc1_bias, model.fc2_weight, model.fc2_bias};
} else if (model.arch == "mnist-cnn") {
weights = {model.conv1_kernel, model.conv1_bias, model.conv2_kernel, model.conv2_bias, model.dense_weight, model.dense_bias};
} else {
GGML_ASSERT(false);
}
for (struct ggml_tensor * t : weights) {
struct ggml_tensor * copy = ggml_dup_tensor(ggml_ctx, t);
ggml_set_name(copy, t->name);
ggml_backend_tensor_get(t, copy->data, 0, ggml_nbytes(t));
gguf_add_tensor(gguf_ctx, copy);
}
gguf_write_to_file(gguf_ctx, fname.c_str(), false);
ggml_free(ggml_ctx);
gguf_free(gguf_ctx);
}
#ifdef __cplusplus
extern "C" {
#endif
int wasm_eval(uint8_t * digitPtr) {
std::vector<float> digit(digitPtr, digitPtr + MNIST_NINPUT);
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, 1, 1);
struct ggml_tensor * data = ggml_opt_dataset_data(dataset);
float * buf = ggml_get_data_f32(data);
for (int i = 0; i < MNIST_NINPUT; ++i) {
buf[i] = digitPtr[i] / 255.0f;
}
ggml_set_zero(ggml_opt_dataset_labels(dataset)); // The labels are not needed.
mnist_model model = mnist_model_init_from_file("mnist-f32.gguf", "CPU", /*nbatch_logical =*/ 1, /*nbatch_physical =*/ 1);
mnist_model_build(model);
ggml_opt_result_t result = mnist_model_eval(model, dataset);
int32_t pred;
ggml_opt_result_pred(result, &pred);
return pred;
}
int wasm_random_digit(char * digitPtr) {
auto fin = std::ifstream("t10k-images-idx3-ubyte", std::ios::binary);
if (!fin) {
fprintf(stderr, "failed to open digits file\n");
return 0;
}
srand(time(NULL));
// Seek to a random digit: 16-byte header + 28*28 * (random 0 - 10000)
fin.seekg(16 + MNIST_NINPUT * (rand() % MNIST_NTEST));
fin.read(digitPtr, MNIST_NINPUT);
return 1;
}
#ifdef __cplusplus
}
#endif
ggml/examples/mnist/mnist-common.h
+166
View File
@@ -0,0 +1,166 @@
#include <algorithm>
#include <cstdint>
#include <random>
#include <string>
#include <thread>
#include <vector>
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpu.h"
#include "ggml-opt.h"
#define MNIST_NTRAIN 60000
#define MNIST_NTEST 10000
// Gradient accumulation can be achieved by setting the logical batch size to a multiple of the physical one.
// The logical batch size determines how many datapoints are used for a gradient update.
// The physical batch size determines how many datapoints are processed in parallel, larger values utilize compute better but need more memory.
#define MNIST_NBATCH_LOGICAL 1000
#define MNIST_NBATCH_PHYSICAL 500
static_assert(MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL == 0, "MNIST_NBATCH_LOGICAL % MNIST_NBATCH_PHYSICAL != 0");
static_assert(MNIST_NTRAIN % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
static_assert(MNIST_NTEST % MNIST_NBATCH_LOGICAL == 0, "MNIST_NTRAIN % MNIST_NBATCH_LOGICAL != 0");
#define MNIST_HW 28
#define MNIST_NINPUT (MNIST_HW*MNIST_HW)
#define MNIST_NCLASSES 10
#define MNIST_NHIDDEN 500
// NCB = number of channels base
#define MNIST_CNN_NCB 8
struct mnist_model {
std::string arch;
ggml_backend_sched_t backend_sched;
std::vector<ggml_backend_t> backends;
const int nbatch_logical;
const int nbatch_physical;
struct ggml_tensor * images = nullptr;
struct ggml_tensor * logits = nullptr;
struct ggml_tensor * fc1_weight = nullptr;
struct ggml_tensor * fc1_bias = nullptr;
struct ggml_tensor * fc2_weight = nullptr;
struct ggml_tensor * fc2_bias = nullptr;
struct ggml_tensor * conv1_kernel = nullptr;
struct ggml_tensor * conv1_bias = nullptr;
struct ggml_tensor * conv2_kernel = nullptr;
struct ggml_tensor * conv2_bias = nullptr;
struct ggml_tensor * dense_weight = nullptr;
struct ggml_tensor * dense_bias = nullptr;
struct ggml_context * ctx_gguf = nullptr;
struct ggml_context * ctx_static = nullptr;
struct ggml_context * ctx_compute = nullptr;
ggml_backend_buffer_t buf_gguf = nullptr;
ggml_backend_buffer_t buf_static = nullptr;
mnist_model(const std::string & backend_name, const int nbatch_logical, const int nbatch_physical)
: nbatch_logical(nbatch_logical), nbatch_physical(nbatch_physical) {
std::vector<ggml_backend_dev_t> devices;
const int ncores_logical = std::thread::hardware_concurrency();
const int nthreads = std::min(ncores_logical, (ncores_logical + 4) / 2);
// Add primary backend:
if (!backend_name.empty()) {
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name.c_str());
if (dev == nullptr) {
fprintf(stderr, "%s: ERROR: backend %s not found, available:\n", __func__, backend_name.c_str());
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev_i = ggml_backend_dev_get(i);
fprintf(stderr, " - %s (%s)\n", ggml_backend_dev_name(dev_i), ggml_backend_dev_description(dev_i));
}
exit(1);
}
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
GGML_ASSERT(backend);
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, nthreads);
}
backends.push_back(backend);
devices.push_back(dev);
}
// Add all available backends as fallback.
// A "backend" is a stream on a physical device so there is no problem with adding multiple backends for the same device.
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
GGML_ASSERT(backend);
if (ggml_backend_is_cpu(backend)) {
ggml_backend_cpu_set_n_threads(backend, nthreads);
}
backends.push_back(backend);
devices.push_back(dev);
}
// The order of the backends passed to ggml_backend_sched_new determines which backend is given priority.
backend_sched = ggml_backend_sched_new(backends.data(), nullptr, backends.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);
fprintf(stderr, "%s: using %s (%s) as primary backend\n",
__func__, ggml_backend_name(backends[0]), ggml_backend_dev_description(devices[0]));
if (backends.size() >= 2) {
fprintf(stderr, "%s: unsupported operations will be executed on the following fallback backends (in order of priority):\n", __func__);
for (size_t i = 1; i < backends.size(); ++i) {
fprintf(stderr, "%s: - %s (%s)\n", __func__, ggml_backend_name(backends[i]), ggml_backend_dev_description(devices[i]));
}
}
{
const size_t size_meta = 1024*ggml_tensor_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ size_meta,
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ctx_static = ggml_init(params);
}
{
// The compute context needs a total of 3 compute graphs: forward pass + backwards pass (with/without optimizer step).
const size_t size_meta = GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ size_meta,
/*.mem_buffer =*/ nullptr,
/*.no_alloc =*/ true,
};
ctx_compute = ggml_init(params);
}
}
~mnist_model() {
ggml_free(ctx_gguf);
ggml_free(ctx_static);
ggml_free(ctx_compute);
ggml_backend_buffer_free(buf_gguf);
ggml_backend_buffer_free(buf_static);
ggml_backend_sched_free(backend_sched);
for (ggml_backend_t backend : backends) {
ggml_backend_free(backend);
}
}
};
bool mnist_image_load(const std::string & fname, ggml_opt_dataset_t dataset);
void mnist_image_print(FILE * f, ggml_opt_dataset_t dataset, const int iex);
bool mnist_label_load(const std::string & fname, ggml_opt_dataset_t dataset);
mnist_model mnist_model_init_from_file(const std::string & fname, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
mnist_model mnist_model_init_random(const std::string & arch, const std::string & backend, const int nbatch_logical, const int nbatch_physical);
void mnist_model_build(mnist_model & model);
ggml_opt_result_t mnist_model_eval(mnist_model & model, ggml_opt_dataset_t dataset);
void mnist_model_train(mnist_model & model, ggml_opt_dataset_t dataset, const int nepoch, const float val_split);
void mnist_model_save(mnist_model & model, const std::string & fname);
ggml/examples/mnist/mnist-eval.cpp
+67
View File
@@ -0,0 +1,67 @@
#include "ggml.h"
#include "ggml-opt.h"
#include "mnist-common.h"
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <ctime>
#include <string>
#include <thread>
#include <vector>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
int main(int argc, char ** argv) {
srand(time(NULL));
ggml_time_init();
if (argc != 4 && argc != 5) {
fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
exit(1);
}
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);
if (!mnist_image_load(argv[2], dataset)) {
return 1;
}
if (!mnist_label_load(argv[3], dataset)) {
return 1;
}
const int iex = rand() % MNIST_NTEST;
mnist_image_print(stdout, dataset, iex);
const std::string backend = argc >= 5 ? argv[4] : "";
const int64_t t_start_us = ggml_time_us();
mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
mnist_model_build(model);
const int64_t t_load_us = ggml_time_us() - t_start_us;
fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0);
ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);
std::vector<int32_t> pred(MNIST_NTEST);
ggml_opt_result_pred(result_eval, pred.data());
fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]);
double loss;
double loss_unc;
ggml_opt_result_loss(result_eval, &loss, &loss_unc);
fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc);
double accuracy;
double accuracy_unc;
ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);
fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc);
ggml_opt_result_free(result_eval);
return 0;
}
ggml/examples/mnist/mnist-train-cnn.py
Executable
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
import sys
from time import time
import gguf
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
def train(model_path):
# Model / data parameters
num_classes = 10
input_shape = (28, 28, 1)
# Load the data and split it between train and test sets
(x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
# Scale images to the [0, 1] range
x_train = x_train.astype("float32") / 255
x_test = x_test.astype("float32") / 255
x_train = np.expand_dims(x_train, -1)
x_test = np.expand_dims(x_test, -1)
print("x_train shape:", x_train.shape)
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = keras.Sequential(
[
keras.Input(shape=input_shape, dtype=tf.float32),
layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
layers.MaxPooling2D(pool_size=(2, 2)),
layers.Flatten(),
layers.Dense(num_classes, activation="softmax", dtype=tf.float32),
]
)
model.summary()
batch_size = 1000
epochs = 30
model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
t_start = time()
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
print(f"Training took {time()-t_start:.2f}s")
score = model.evaluate(x_test, y_test, verbose=0)
print(f"Test loss: {score[0]:.6f}")
print(f"Test accuracy: {100*score[1]:.2f}%")
gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn")
conv1_kernel = model.layers[0].weights[0].numpy()
conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])
gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3))
conv1_bias = model.layers[0].weights[1].numpy()
gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1))
conv2_kernel = model.layers[2].weights[0].numpy()
conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])
gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3))
conv2_bias = model.layers[2].weights[1].numpy()
gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1))
dense_weight = model.layers[-1].weights[0].numpy()
dense_weight = dense_weight.transpose()
gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16))
dense_bias = model.layers[-1].weights[1].numpy()
gguf_writer.add_tensor("dense.bias", dense_bias)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
print(f"GGUF model saved to '{model_path}'")
if __name__ == '__main__':
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <model_path>")
sys.exit(1)
train(sys.argv[1])
ggml/examples/mnist/mnist-train-fc.py
+131
View File
@@ -0,0 +1,131 @@
import gguf
import numpy as np
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
import sys
from time import time
input_size = 784 # img_size = (28,28) ---> 28*28=784 in total
hidden_size = 500 # number of nodes at hidden layer
num_classes = 10 # number of output classes discrete range [0,9]
num_epochs = 30 # number of times which the entire dataset is passed throughout the model
batch_size = 1000 # the size of input data used for one iteration
lr = 1e-3 # size of step
class Net(nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
super(Net, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.relu = nn.ReLU()
self.fc2 = nn.Linear(hidden_size, num_classes)
def forward(self, x):
out = self.fc1(x)
out = self.relu(out)
out = self.fc2(out)
return out
def train(model_path):
train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)
test_data = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())
assert len(train_data) == 60000
assert len(test_data) == 10000
kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)
train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True, **kwargs_train_test)
test_gen = torch.utils.data.DataLoader(dataset=test_data, shuffle=False, **kwargs_train_test)
net = Net(input_size, hidden_size, num_classes)
if torch.cuda.is_available():
net.cuda()
loss_function = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
t_start = time()
for epoch in range(num_epochs):
loss_history = []
ncorrect = 0
for i, (images, labels) in enumerate(train_gen):
images = Variable(images.view(-1, 28*28))
labels = Variable(labels)
if torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
optimizer.zero_grad()
outputs = net(images)
loss = loss_function(outputs, labels)
loss_history.append(loss.cpu().data)
_, predictions = torch.max(outputs, 1)
ncorrect += (predictions == labels).sum()
loss.backward()
optimizer.step()
if (i + 1)*batch_size % 10000 == 0:
loss_mean = np.mean(loss_history)
accuracy = ncorrect / ((i + 1) * batch_size)
print(
f"Epoch [{epoch+1:02d}/{num_epochs}], "
f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], "
f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%")
print()
print(f"Training took {time()-t_start:.2f}s")
loss_history = []
ncorrect = 0
for i, (images, labels) in enumerate(test_gen):
images = Variable(images.view(-1, 28*28))
labels = Variable(labels)
if torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
outputs = net(images)
loss = loss_function(outputs, labels)
loss_history.append(loss.cpu().data)
_, predictions = torch.max(outputs, 1)
ncorrect += (predictions == labels).sum().cpu().numpy()
loss_mean = np.mean(loss_history)
loss_uncertainty = np.std(loss_history) / np.sqrt(len(loss_history) - 1)
accuracy_mean = ncorrect / (len(test_gen) * batch_size)
accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))
print()
print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%")
gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc")
print()
print(f"Model tensors saved to {model_path}:")
for tensor_name in net.state_dict().keys():
data = net.state_dict()[tensor_name].squeeze().cpu().numpy()
print(tensor_name, "\t", data.shape)
gguf_writer.add_tensor(tensor_name, data)
gguf_writer.write_header_to_file()
gguf_writer.write_kv_data_to_file()
gguf_writer.write_tensors_to_file()
gguf_writer.close()
if __name__ == '__main__':
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} <model_path>")
sys.exit(1)
train(sys.argv[1])
ggml/examples/mnist/mnist-train.cpp
+39
View File
@@ -0,0 +1,39 @@
#include "ggml-opt.h"
#include "mnist-common.h"
#include <cmath>
#include <cstdio>
#include <cstring>
#include <ctime>
#include <string>
#if defined(_MSC_VER)
#pragma warning(disable: 4244 4267) // possible loss of data
#endif
int main(int argc, char ** argv) {
if (argc != 5 && argc != 6) {
fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
exit(0);
}
// The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.
// With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).
// A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.
ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);
if (!mnist_image_load(argv[3], dataset)) {
return 1;
}
if (!mnist_label_load(argv[4], dataset)) {
return 1;
}
mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
mnist_model_build(model);
mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);
mnist_model_save(model, argv[2]);
}
ggml/examples/mnist/server.py
+36
View File
@@ -0,0 +1,36 @@
import http.server
import socketserver
import os
import sys
DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))
PORT = 8000
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=DIRECTORY, **kwargs)
def end_headers(self):
# Add required headers for SharedArrayBuffer
self.send_header("Cross-Origin-Opener-Policy", "same-origin")
self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
self.send_header("Access-Control-Allow-Origin", "*")
super().end_headers()
# Enable address reuse
class CustomServer(socketserver.TCPServer):
allow_reuse_address = True
try:
with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
print(f"Application context root: http://localhost:{PORT}/")
try:
httpd.serve_forever()
except KeyboardInterrupt:
print("\nServer stopped.")
# Force complete exit
sys.exit(0)
except OSError as e:
print(f"Error: {e}")
sys.exit(1)
ggml/examples/perf-metal/CMakeLists.txt
+7
View File
@@ -0,0 +1,7 @@
#
# perf-metal
set(TEST_TARGET perf-metal)
add_executable(${TEST_TARGET} perf-metal.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml)
ggml/examples/perf-metal/perf-metal.cpp
+152
View File
@@ -0,0 +1,152 @@
// basic tool to experiment with the Metal backend
//
// 1. Get GPU trace of a dummy graph:
//
// rm -rf /tmp/perf-metal.gputrace
// make -j perf-metal && METAL_CAPTURE_ENABLED=1 ./bin/perf-metal
// open /tmp/perf-metal.gputrace
//
// https://github.com/ggerganov/llama.cpp/issues/9507
//
#include "ggml.h"
#include "ggml-alloc.h"
#include "ggml-backend.h"
#include "ggml-metal.h"
#include <cstdio>
#include <vector>
#include <thread>
int main(int argc, char ** argv) {
int n_op = 1024;
int n_iter = 128;
if (argc > 1) {
n_op = std::atoi(argv[1]);
}
if (argc > 2) {
n_iter = std::atoi(argv[2]);
}
printf("%s: n_op = %d, n_iter = %d\n", __func__, n_op, n_iter);
const int ne00 = 8;
const int ne01 = 8;
const int ne11 = 8;
std::vector<float> data0(ne00*ne01, 1.0f);
std::vector<float> data1(ne00*ne01, 1.0f/ne00);
ggml_backend_t backend = ggml_backend_metal_init();
if (!backend) {
fprintf(stderr, "%s: ggml_backend_metal_init() failed\n", __func__);
return 1;
}
const size_t ctx_size = 2 * ggml_tensor_overhead();
struct ggml_init_params params = {
/*.mem_size =*/ ctx_size,
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
struct ggml_context * ctx = ggml_init(params);
struct ggml_tensor * t0 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne01);
struct ggml_tensor * t1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, ne00, ne11);
ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);
ggml_backend_tensor_set(t0, data0.data(), 0, ggml_nbytes(t0));
ggml_backend_tensor_set(t1, data1.data(), 0, ggml_nbytes(t1));
struct ggml_cgraph * gf = NULL;
struct ggml_context * ctx_cgraph = NULL;
// create a dummy compute graph:
//
// x = mul_mat(t0, t1)
// x = x * 1.0f
// x = mul_mat(x, t1)
// x = x * 1.0f
// ... repeat n_op times ...
//
{
struct ggml_init_params params0 = {
/*.mem_size =*/ 4*n_op*ggml_tensor_overhead() + ggml_graph_overhead(),
/*.mem_buffer =*/ NULL,
/*.no_alloc =*/ true,
};
ctx_cgraph = ggml_init(params0);
gf = ggml_new_graph_custom(ctx_cgraph, 4*n_op, false);
struct ggml_tensor * cur = ggml_mul_mat(ctx_cgraph, t0, t1);
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
for (int i = 0; i < n_op - 1; i++) {
cur = ggml_mul_mat(ctx_cgraph, cur, t1);
cur = ggml_scale(ctx_cgraph, cur, 1.0f);
}
cur = ggml_scale(ctx_cgraph, cur, 42.0f);
ggml_build_forward_expand(gf, cur);
}
printf("%s: graph nodes = %d\n", __func__, ggml_graph_n_nodes(gf));
ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));
ggml_gallocr_alloc_graph(allocr, gf);
{
// warm-up
ggml_backend_graph_compute(backend, gf);
const int64_t t_start = ggml_time_us();
for (int iter = 0; iter < n_iter; iter++) {
ggml_backend_graph_compute(backend, gf);
}
const int64_t t_end = ggml_time_us();
// actual trace
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
//std::this_thread::sleep_for(std::chrono::milliseconds(1000)); // NOTE: these intervals do not appear in the XCode trace!
ggml_backend_metal_capture_next_compute(backend);
ggml_backend_graph_compute(backend, gf);
printf("%s: time = %f ms\n", __func__, (t_end - t_start) / 1000.0 / n_iter);
}
{
struct ggml_tensor * res = ggml_graph_node(gf, -1);
std::vector<float> data(res->ne[0] * res->ne[1], 0.0f);
ggml_backend_tensor_get(res, data.data(), 0, ggml_nbytes(res));
for (int i1 = 0; i1 < res->ne[1]; i1++) {
for (int i0 = 0; i0 < res->ne[0]; i0++) {
printf("%f ", data[i1*res->ne[0] + i0]);
}
printf("\n");
}
}
ggml_free(ctx_cgraph);
ggml_gallocr_free(allocr);
ggml_free(ctx);
ggml_backend_buffer_free(buffer);
ggml_backend_free(backend);
return 0;
}
ggml/examples/prompts/dolly-v2.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 12092,3645,2
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
The book costs $19.99 => 510,1984,4815,370,746,15,1525
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
They are renovating their house. => 3726,403,30074,839,616,2419,15
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
The traffic is congest => 510,7137,310,25801
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
ggml/examples/prompts/gpt-2-chinese.txt
+1
View File
@@ -0,0 +1 @@
请问洗手间在哪里？ => 6435,7309,3819,2797,7313,1762,1525,7027,8043
ggml/examples/prompts/gpt-2.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 15496,2159,0
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
They are renovating their house. => 2990,389,24317,803,511,2156,13
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
The traffic is congest => 464,4979,318,22791
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
ggml/examples/prompts/gpt-j.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 15496,2159,0
I can't believe it's already Friday!" => 40,460,470,1975,340,338,1541,3217,2474
The URL for the website is https://www.example.com." => 464,10289,329,262,3052,318,3740,1378,2503,13,20688,13,785,526
"She said, 'I love to travel.'" => 1,3347,531,11,705,40,1842,284,3067,11496
'The temperature is 25.5°C.' => 6,464,5951,318,1679,13,20,7200,34,2637
"Let's meet at 2:30 p.m. in the park." => 1,5756,338,1826,379,362,25,1270,279,13,76,13,287,262,3952,526
The book costs $19.99 => 464,1492,3484,720,1129,13,2079
"John's favorite color is blue." => 1,7554,338,4004,3124,318,4171,526
Th@nk y0u f0r y0ur h3lp! => 817,31,77,74,331,15,84,277,15,81,331,15,333,289,18,34431,0
C@n I g3t a c0ffee, pl3@se? => 34,31,77,314,308,18,83,257,269,15,5853,11,458,18,31,325,30
W0w! Th@t's @m@zing! => 54,15,86,0,536,31,83,338,2488,76,31,9510,0
H0w 4re y0u t0d@y? => 39,15,86,604,260,331,15,84,256,15,67,31,88,30
I l0ve t0 tr@vel @r0und the w0rld. => 40,300,15,303,256,15,491,31,626,2488,81,15,917,262,266,15,81,335,13
Wh@t's y0ur f@v0rite m0vie? => 1199,31,83,338,331,15,333,277,31,85,15,6525,285,15,85,494,30
The cat is sleeping on the mat. => 464,3797,318,11029,319,262,2603,13
I need to buy some groceries for dinner. => 40,761,284,2822,617,38464,329,8073,13
The sun is shining brightly in the sky. => 464,4252,318,22751,35254,287,262,6766,13
She is reading a book in the park. => 3347,318,3555,257,1492,287,262,3952,13
We went for a walk on the beach yesterday. => 1135,1816,329,257,2513,319,262,10481,7415,13
He plays the guitar like a pro. => 1544,5341,262,10047,588,257,386,13
They are going to the movies tonight. => 2990,389,1016,284,262,6918,9975,13
The flowers are blooming in the garden. => 464,12734,389,24924,3383,287,262,11376,13
I enjoy listening to classical music. => 40,2883,8680,284,15993,2647,13
We need to buy groceries for the week. => 1135,761,284,2822,38464,329,262,1285,13
The dog is chasing its tail in circles. => 464,3290,318,20023,663,7894,287,13332,13
She is wearing a beautiful red dress. => 3347,318,5762,257,4950,2266,6576,13
He is a talented actor in Hollywood. => 1544,318,257,12356,8674,287,8502,13
The children are playing in the playground. => 464,1751,389,2712,287,262,24817,13
I'm going to visit my grandparents this weekend. => 40,1101,1016,284,3187,616,28571,428,5041,13
The coffee tastes bitter without sugar. => 464,6891,18221,12922,1231,7543,13
They are planning a surprise party for her. => 2990,389,5410,257,5975,2151,329,607,13
She sings like an angel on stage. => 3347,33041,588,281,18304,319,3800,13
We should take a vacation to relax. => 1135,815,1011,257,14600,284,8960,13
He is studying medicine at the university. => 1544,318,11065,9007,379,262,6403,13
The rain is pouring heavily outside. => 464,6290,318,23147,7272,2354,13
I enjoy watching romantic movies. => 40,2883,4964,14348,6918,13
They are celebrating their anniversary today. => 2990,389,17499,511,11162,1909,13
She dances gracefully to the music. => 3347,38207,11542,2759,284,262,2647,13
He is an excellent basketball player. => 1544,318,281,6275,9669,2137,13
The baby is sleeping soundly in the crib. => 464,5156,318,11029,2128,306,287,262,48083,13
I need to finish my homework before dinner. => 40,761,284,5461,616,26131,878,8073,13
They are organizing a charity event next month. => 2990,389,16924,257,11016,1785,1306,1227,13
She is cooking a delicious meal for us. => 3347,318,10801,257,12625,9799,329,514,13
We should go hiking in the mountains. => 1135,815,467,24522,287,262,12269,13
The car broke down on the way to work. => 464,1097,6265,866,319,262,835,284,670,13
He loves playing video games in his free time. => 1544,10408,2712,2008,1830,287,465,1479,640,13
The birds are chirping in the trees. => 464,10087,389,442,343,13886,287,262,7150,13
I want to learn how to play the piano. => 40,765,284,2193,703,284,711,262,19132,13
They are building a new shopping mall in the city. => 2990,389,2615,257,649,9735,17374,287,262,1748,13
She is writing a novel in her spare time. => 3347,318,3597,257,5337,287,607,13952,640,13
We are going to the zoo this Saturday. => 1135,389,1016,284,262,26626,428,3909,13
The cake looks delicious with chocolate frosting. => 464,12187,3073,12625,351,11311,21682,278,13
He is a talented painter who sells his artwork. => 1544,318,257,12356,34537,508,16015,465,16257,13
The students are studying for their exams. => 464,2444,389,11065,329,511,26420,13
I enjoy swimming in the ocean. => 40,2883,14899,287,262,9151,13
They are renovating their house. => 2990,389,24317,803,511,2156,13
She is practicing yoga to stay healthy. => 3347,318,18207,20351,284,2652,5448,13
We should plant flowers in the garden. => 1135,815,4618,12734,287,262,11376,13
The traffic is heavy during rush hour. => 464,4979,318,4334,1141,10484,1711,13
He is a skilled chef who creates amazing dishes. => 1544,318,257,14297,21221,508,8075,4998,16759,13
The baby is crawling on the floor. => 464,5156,318,34499,319,262,4314,13
I need to buy a new pair of shoes. => 40,761,284,2822,257,649,5166,286,10012,13
They are going on a road trip across the country. => 2990,389,1016,319,257,2975,5296,1973,262,1499,13
She is playing the piano beautifully. => 3347,318,2712,262,19132,21104,13
We are going to a concert tomorrow night. => 1135,389,1016,284,257,10010,9439,1755,13
The cake tastes delicious with vanilla frosting. => 464,12187,18221,12625,351,16858,21682,278,13
He is a dedicated teacher who inspires his students. => 1544,318,257,7256,4701,508,38934,465,2444,13
The students are participating in a science fair. => 464,2444,389,11983,287,257,3783,3148,13
I enjoy hiking in the mountains. => 40,2883,24522,287,262,12269,13
They are organizing a beach cleanup next weekend. => 2990,389,16924,257,10481,27425,1306,5041,13
She is taking photographs of nature. => 3347,318,2263,12566,286,3450,13
We should try a new restaurant in town. => 1135,815,1949,257,649,7072,287,3240,13
The traffic is moving slowly on the highway. => 464,4979,318,3867,6364,319,262,12763,13
He is a talented singer with a beautiful voice. => 1544,318,257,12356,14015,351,257,4950,3809,13
The baby is laughing and giggling. => 464,5156,318,14376,290,30442,1359,13
I need to do laundry and wash my clothes. => 40,761,284,466,25724,290,13502,616,8242,13
They are planning a trip to Europe. => 2990,389,5410,257,5296,284,2031,13
She is learning how to play the guitar. => 3347,318,4673,703,284,711,262,10047,13
We are going to a museum this Sunday. => 1135,389,1016,284,257,13257,428,3502,13
The coffee smells amazing in the morning. => 464,6891,25760,4998,287,262,3329,13
He is a hardworking farmer who grows crops. => 1544,318,257,1327,16090,18739,508,13676,14450,13
The students are presenting their research projects. => 464,2444,389,17728,511,2267,4493,13
I enjoy playing soccer with my friends. => 40,2883,2712,11783,351,616,2460,13
They are volunteering at a local shelter. => 2990,389,41434,379,257,1957,11772,13
She is practicing martial arts for self-defense. => 3347,318,18207,15618,10848,329,2116,12,19774,13
We should try a new recipe for dinner. => 1135,815,1949,257,649,8364,329,8073,13
The traffic is congest => 464,4979,318,22791
The sun is shining brightly today. => 464,4252,318,22751,35254,1909,13
I enjoy reading books in my free time. => 40,2883,3555,3835,287,616,1479,640,13
She plays the piano beautifully. => 3347,5341,262,19132,21104,13
The cat chased the mouse around the room. => 464,3797,26172,262,10211,1088,262,2119,13
I love eating pizza with extra cheese. => 40,1842,6600,14256,351,3131,9891,13
He always wears a hat wherever he goes. => 1544,1464,17326,257,6877,14530,339,2925,13
The flowers in the garden are blooming. => 464,12734,287,262,11376,389,24924,3383,13
She danced gracefully on the stage. => 3347,39480,11542,2759,319,262,3800,13
The dog barked loudly in the park. => 464,3290,21405,276,23112,287,262,3952,13
We went swimming in the ocean yesterday. => 1135,1816,14899,287,262,9151,7415,13
He speaks fluent French and Spanish. => 1544,9209,43472,4141,290,7897,13
The train arrived at the station on time. => 464,4512,5284,379,262,4429,319,640,13
She cooked a delicious meal for her family. => 3347,15847,257,12625,9799,329,607,1641,13
ggml/examples/prompts/gpt-neox-japanese.txt
+1
View File
@@ -0,0 +1 @@
明日の天気はどうですか。 => 263,7353,268,18461,271,1722,18405,265
ggml/examples/prompts/gpt-neox.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 12092,3645,2
I can't believe it's already Friday!" => 42,476,626,2868,352,434,2168,6794,1476
The URL for the website is https://www.example.com." => 510,10611,323,253,4422,310,5987,1358,2700,15,11667,15,681,449
"She said, 'I love to travel.'" => 3,2993,753,13,686,42,2389,281,4288,18574
'The temperature is 25.5°C.' => 8,510,3276,310,2030,15,22,3272,36,2464
"Let's meet at 2:30 p.m. in the park." => 3,1466,434,2525,387,374,27,1229,268,15,78,15,275,253,5603,449
The book costs $19.99 => 510,1984,4815,370,746,15,1525
"John's favorite color is blue." => 3,8732,434,7583,3295,310,4797,449
Th@nk y0u f0r y0ur h3lp! => 1044,33,30664,340,17,86,269,17,83,340,17,321,288,20,24343,2
C@n I g3t a c0ffee, pl3@se? => 36,33,79,309,305,20,85,247,260,17,71,6851,13,499,20,33,339,32
W0w! Th@t's @m@zing! => 56,17,88,2,596,33,85,434,1214,78,33,8537,2
H0w 4re y0u t0d@y? => 41,17,88,577,250,340,17,86,246,17,69,33,90,32
I l0ve t0 tr@vel @r0und the w0rld. => 42,298,17,306,246,17,492,33,652,1214,83,17,1504,253,259,17,83,392,15
Wh@t's y0ur f@v0rite m0vie? => 3152,33,85,434,340,17,321,269,33,87,17,3852,278,17,25858,32
The cat is sleeping on the mat. => 510,5798,310,14343,327,253,1111,15
I need to buy some groceries for dinner. => 42,878,281,4489,690,45160,447,323,8955,15
The sun is shining brightly in the sky. => 510,5101,310,28115,43925,275,253,8467,15
She is reading a book in the park. => 2993,310,4361,247,1984,275,253,5603,15
We went for a walk on the beach yesterday. => 1231,2427,323,247,2940,327,253,11600,11066,15
He plays the guitar like a pro. => 1328,7120,253,12609,751,247,354,15
They are going to the movies tonight. => 3726,403,1469,281,253,11321,11608,15
The flowers are blooming in the garden. => 510,12405,403,30601,272,275,253,10329,15
I enjoy listening to classical music. => 42,4264,11298,281,8946,3440,15
We need to buy groceries for the week. => 1231,878,281,4489,45160,447,323,253,2129,15
The dog is chasing its tail in circles. => 510,4370,310,31702,697,8105,275,14240,15
She is wearing a beautiful red dress. => 2993,310,9398,247,5389,2502,7619,15
He is a talented actor in Hollywood. => 1328,310,247,21220,12353,275,14759,15
The children are playing in the playground. => 510,2151,403,4882,275,253,41008,15
I'm going to visit my grandparents this weekend. => 42,1353,1469,281,4143,619,37186,436,8849,15
The coffee tastes bitter without sugar. => 510,8574,27491,17123,1293,8618,15
They are planning a surprise party for her. => 3726,403,7219,247,9326,3128,323,617,15
She sings like an angel on stage. => 2993,44718,751,271,23087,327,3924,15
We should take a vacation to relax. => 1231,943,1379,247,18125,281,7921,15
He is studying medicine at the university. => 1328,310,12392,9921,387,253,9835,15
The rain is pouring heavily outside. => 510,9313,310,31226,11306,3345,15
I enjoy watching romantic movies. => 42,4264,7487,18109,11321,15
They are celebrating their anniversary today. => 3726,403,28765,616,19054,3063,15
She dances gracefully to the music. => 2993,47078,14426,2920,281,253,3440,15
He is an excellent basketball player. => 1328,310,271,7126,14648,4760,15
The baby is sleeping soundly in the crib. => 510,6858,310,14343,3590,314,275,253,260,725,15
I need to finish my homework before dinner. => 42,878,281,8416,619,32110,1078,8955,15
They are organizing a charity event next month. => 3726,403,26169,247,19489,2362,1735,1770,15
She is cooking a delicious meal for us. => 2993,310,12398,247,17319,11484,323,441,15
We should go hiking in the mountains. => 1231,943,564,33061,275,253,14700,15
The car broke down on the way to work. => 510,1113,9377,1066,327,253,1039,281,789,15
He loves playing video games in his free time. => 1328,14528,4882,3492,3958,275,521,1959,673,15
The birds are chirping in the trees. => 510,11260,403,36494,14650,275,253,7139,15
I want to learn how to play the piano. => 42,971,281,3037,849,281,1132,253,18542,15
They are building a new shopping mall in the city. => 3726,403,3652,247,747,12701,28974,275,253,2846,15
She is writing a novel in her spare time. => 2993,310,4028,247,4460,275,617,18345,673,15
We are going to the zoo this Saturday. => 1231,403,1469,281,253,41089,436,7814,15
The cake looks delicious with chocolate frosting. => 510,15221,4453,17319,342,14354,34724,272,15
He is a talented painter who sells his artwork. => 1328,310,247,21220,27343,665,27924,521,28227,15
The students are studying for their exams. => 510,3484,403,12392,323,616,34666,15
I enjoy swimming in the ocean. => 42,4264,17120,275,253,12927,15
They are renovating their house. => 3726,403,30074,839,616,2419,15
She is practicing yoga to stay healthy. => 2993,310,25815,25551,281,3297,5875,15
We should plant flowers in the garden. => 1231,943,4444,12405,275,253,10329,15
The traffic is heavy during rush hour. => 510,7137,310,5536,1309,16949,4964,15
He is a skilled chef who creates amazing dishes. => 1328,310,247,18024,26540,665,10513,8644,17114,15
The baby is crawling on the floor. => 510,6858,310,44922,327,253,5254,15
I need to buy a new pair of shoes. => 42,878,281,4489,247,747,4667,273,12682,15
They are going on a road trip across the country. => 3726,403,1469,327,247,3971,7408,2439,253,2586,15
She is playing the piano beautifully. => 2993,310,4882,253,18542,27839,15
We are going to a concert tomorrow night. => 1231,403,1469,281,247,12699,10873,2360,15
The cake tastes delicious with vanilla frosting. => 510,15221,27491,17319,342,26724,34724,272,15
He is a dedicated teacher who inspires his students. => 1328,310,247,9940,9732,665,6381,2731,521,3484,15
The students are participating in a science fair. => 510,3484,403,15299,275,247,5859,4344,15
I enjoy hiking in the mountains. => 42,4264,33061,275,253,14700,15
They are organizing a beach cleanup next weekend. => 3726,403,26169,247,11600,34709,1735,8849,15
She is taking photographs of nature. => 2993,310,3192,15928,273,3753,15
We should try a new restaurant in town. => 1231,943,1611,247,747,10301,275,3874,15
The traffic is moving slowly on the highway. => 510,7137,310,4886,7808,327,253,17657,15
He is a talented singer with a beautiful voice. => 1328,310,247,21220,16057,342,247,5389,4318,15
The baby is laughing and giggling. => 510,6858,310,17053,285,41542,1981,15
I need to do laundry and wash my clothes. => 42,878,281,513,29023,285,14841,619,10015,15
They are planning a trip to Europe. => 3726,403,7219,247,7408,281,3060,15
She is learning how to play the guitar. => 2993,310,4715,849,281,1132,253,12609,15
We are going to a museum this Sunday. => 1231,403,1469,281,247,16064,436,6926,15
The coffee smells amazing in the morning. => 510,8574,34247,8644,275,253,4131,15
He is a hardworking farmer who grows crops. => 1328,310,247,1892,21107,24718,665,17202,19492,15
The students are presenting their research projects. => 510,3484,403,15250,616,2561,6493,15
I enjoy playing soccer with my friends. => 42,4264,4882,20391,342,619,3858,15
They are volunteering at a local shelter. => 3726,403,10057,2158,387,247,1980,17824,15
She is practicing martial arts for self-defense. => 2993,310,25815,29731,14635,323,1881,14,29337,15
We should try a new recipe for dinner. => 1231,943,1611,247,747,13612,323,8955,15
The traffic is congest => 510,7137,310,25801
The sun is shining brightly today. => 510,5101,310,28115,43925,3063,15
I enjoy reading books in my free time. => 42,4264,4361,5098,275,619,1959,673,15
She plays the piano beautifully. => 2993,7120,253,18542,27839,15
The cat chased the mouse around the room. => 510,5798,40754,253,6521,1475,253,2316,15
I love eating pizza with extra cheese. => 42,2389,9123,22534,342,4465,12173,15
He always wears a hat wherever he goes. => 1328,1900,31394,247,7856,20312,344,4566,15
The flowers in the garden are blooming. => 510,12405,275,253,10329,403,30601,272,15
She danced gracefully on the stage. => 2993,39860,14426,2920,327,253,3924,15
The dog barked loudly in the park. => 510,4370,21939,264,31311,275,253,5603,15
We went swimming in the ocean yesterday. => 1231,2427,17120,275,253,12927,11066,15
He speaks fluent French and Spanish. => 1328,16544,2938,290,5112,285,9883,15
The train arrived at the station on time. => 510,6194,7244,387,253,4660,327,673,15
She cooked a delicious meal for her family. => 2993,18621,247,17319,11484,323,617,2021,15
ggml/examples/prompts/polyglot-ko.txt
+3
View File
@@ -0,0 +1,3 @@
이것은 테스트 이다. => 12271,296,6474,28037,17
걱정할 필요 없다. => 18311,482,1062,550,267,17
버그는 언젠가 고쳐진다. => 6904,272,8575,10381,1765,17
ggml/examples/prompts/replit.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 6466,147,2317,350
I can't believe it's already Friday!" => 286,512,172,185,13392,393,172,155,3239,147,29249,8537
The URL for the website is https://www.example.com." => 505,5635,250,170,11745,235,147,303,262,552,148,811,148,241,148,161
"She said, 'I love to travel.'" => 161,10386,4089,150,206,286,8440,194,147,12363,148,172,161
'The temperature is 25.5°C.' => 172,505,147,9502,235,147,20022,8516,228,148,172
"Let's meet at 2:30 p.m. in the park." => 161,8997,172,155,17120,536,147,162,5245,147,207,148,204,148,219,170,147,17664,148,161
The book costs $19.99 => 505,147,2277,17494,236,166,11824
"John's favorite color is blue." => 161,7475,172,155,147,11105,147,349,235,17046,148,161
Th@nk y0u f0r y0ur h3lp! => 6309,240,9019,147,237,159,247,147,202,159,223,147,237,159,2458,147,226,171,3899,350
C@n I g3t a c0ffee, pl3@se? => 228,240,211,398,147,267,171,185,216,147,196,159,13360,163,150,147,1287,171,240,155,163,272
W0w! Th@t's @m@zing! => 450,159,274,350,147,6309,240,185,172,155,268,204,240,301,248,350
H0w 4re y0u t0d@y? => 304,159,274,320,440,147,237,159,247,147,185,159,182,240,237,272
I l0ve t0 tr@vel @r0und the w0rld. => 286,997,159,1290,147,185,159,147,490,240,3893,268,223,159,3981,170,147,274,159,223,2833,148
Wh@t's y0ur f@v0rite m0vie? => 450,226,240,185,172,155,147,237,159,2458,147,202,240,252,159,5961,163,147,204,159,24373,272
The cat is sleeping on the mat. => 505,147,1604,235,147,3987,248,347,170,147,1297,148
I need to buy some groceries for dinner. => 286,1645,194,147,8068,1499,147,10022,1037,10023,250,147,182,2749,148
The sun is shining brightly in the sky. => 505,147,5852,235,147,7304,2967,147,215,649,391,219,170,147,7310,148
She is reading a book in the park. => 10386,235,9838,216,147,2277,219,170,147,17664,148
We went for a walk on the beach yesterday. => 3250,10825,250,216,147,8156,347,170,294,5371,147,28830,148
He plays the guitar like a pro. => 5301,7084,155,170,147,4604,2214,1425,216,3474,148
They are going to the movies tonight. => 18815,429,6552,194,170,147,15877,194,7907,148
The flowers are blooming in the garden. => 505,147,22953,155,429,147,10411,2799,248,219,170,147,22140,148
I enjoy listening to classical music. => 286,23162,15876,248,194,239,4251,147,7395,148
We need to buy groceries for the week. => 3250,1645,194,147,8068,147,10022,1037,10023,250,170,9238,148
The dog is chasing its tail in circles. => 505,147,6540,235,147,196,916,248,1602,147,5129,219,147,4095,155,148
She is wearing a beautiful red dress. => 10386,235,147,16427,248,216,147,23447,147,1160,147,14592,148
He is a talented actor in Hollywood. => 5301,235,216,147,29750,246,147,5112,219,147,16924,391,10477,148
The children are playing in the playground. => 505,7934,429,7084,248,219,170,7084,12055,148
I'm going to visit my grandparents this weekend. => 286,172,204,6552,194,9939,1247,147,11806,12019,291,9238,314,148
The coffee tastes bitter without sugar. => 505,147,21526,147,20931,155,5145,1430,1988,147,28759,148
They are planning a surprise party for her. => 18815,429,147,23661,216,147,29240,147,7344,250,1869,148
She sings like an angel on stage. => 10386,147,155,6502,1425,426,147,26028,347,12685,148
We should take a vacation to relax. => 3250,936,4654,216,147,15388,946,194,1998,2744,148
He is studying medicine at the university. => 5301,235,7959,248,147,20742,1668,536,170,147,8025,148
The rain is pouring heavily outside. => 505,147,6885,235,5306,248,1189,5451,391,8096,148
I enjoy watching romantic movies. => 286,23162,147,3355,248,147,26080,4140,147,15877,148
They are celebrating their anniversary today. => 18815,429,147,30000,5841,1669,147,24734,5464,1770,13386,148
She dances gracefully to the music. => 10386,147,182,1626,155,147,267,8771,8001,194,170,147,7395,148
He is an excellent basketball player. => 5301,235,426,147,12300,675,185,147,26646,5132,6294,148
The baby is sleeping soundly in the crib. => 505,147,23597,235,147,3987,248,12642,391,219,170,147,7696,215,148
I need to finish my homework before dinner. => 286,1645,194,147,6717,1247,147,1071,2722,2643,147,182,2749,148
They are organizing a charity event next month. => 18815,429,147,16442,248,216,1054,1511,1663,2399,12821,148
She is cooking a delicious meal for us. => 10386,235,147,20453,248,216,3936,23455,147,26658,250,147,539,148
We should go hiking in the mountains. => 3250,936,4242,147,2254,5357,219,170,147,204,18028,155,148
The car broke down on the way to work. => 505,7553,147,510,10036,4288,347,170,3699,194,1916,148
He loves playing video games in his free time. => 5301,8440,155,7084,248,8722,147,11281,219,1439,4002,801,148
The birds are chirping in the trees. => 505,147,13043,155,429,147,3904,223,4639,219,170,5311,155,148
I want to learn how to play the piano. => 286,1857,194,14167,2496,194,7084,170,147,207,23635,148
They are building a new shopping mall in the city. => 18815,429,11038,216,277,147,22184,147,204,609,219,170,147,2416,148
She is writing a novel in her spare time. => 10386,235,3242,216,147,25814,219,1869,6772,2382,801,148
We are going to the zoo this Saturday. => 3250,429,6552,194,170,147,25101,291,147,31426,148
The cake looks delicious with chocolate frosting. => 505,147,24422,16303,3936,23455,312,147,5619,533,2239,147,202,3973,3431,148
He is a talented painter who sells his artwork. => 5301,235,216,147,29750,246,147,9226,279,2888,13004,155,1439,12234,2722,148
The students are studying for their exams. => 505,15707,429,7959,248,250,1669,147,12398,155,148
I enjoy swimming in the ocean. => 286,23162,147,4729,8528,248,219,170,147,26193,148
They are renovating their house. => 18815,429,991,10724,3643,1669,13788,148
She is practicing yoga to stay healthy. => 10386,235,147,18453,248,147,5063,1186,194,15344,147,28550,148
We should plant flowers in the garden. => 3250,936,147,9212,147,22953,155,219,170,147,22140,148
The traffic is heavy during rush hour. => 505,147,11097,235,147,22232,4340,147,22319,147,5686,148
He is a skilled chef who creates amazing dishes. => 5301,235,216,147,8891,246,9784,202,2888,13720,147,28880,147,23852,383,148
The baby is crawling on the floor. => 505,147,23597,235,147,22120,248,347,170,147,5895,148
I need to buy a new pair of shoes. => 286,1645,194,147,8068,216,277,12632,210,147,155,21953,155,148
They are going on a road trip across the country. => 18815,429,6552,347,216,147,6362,147,11395,9762,170,11305,148
She is playing the piano beautifully. => 10386,235,7084,248,170,147,207,23635,147,23447,391,148
We are going to a concert tomorrow night. => 3250,429,6552,194,216,1710,4391,29524,12716,148
The cake tastes delicious with vanilla frosting. => 505,147,24422,147,20931,155,3936,23455,312,5535,7476,147,202,3973,3431,148
He is a dedicated teacher who inspires his students. => 5301,235,216,326,8298,3460,147,9675,2888,147,28801,155,1439,15707,148
The students are participating in a science fair. => 505,15707,429,147,30961,3643,219,216,147,10587,147,7636,148
I enjoy hiking in the mountains. => 286,23162,147,2254,5357,219,170,147,204,18028,155,148
They are organizing a beach cleanup next weekend. => 18815,429,147,16442,248,216,294,5371,147,10401,2399,9238,314,148
She is taking photographs of nature. => 10386,235,147,12345,147,4709,1547,155,210,147,211,8603,148
We should try a new restaurant in town. => 3250,936,147,746,216,277,147,11007,219,147,10200,148
The traffic is moving slowly on the highway. => 505,147,11097,235,147,8601,147,9880,391,347,170,5976,3330,148
He is a talented singer with a beautiful voice. => 5301,235,216,147,29750,246,147,155,248,279,312,216,147,23447,147,9316,148
The baby is laughing and giggling. => 505,147,23597,235,147,23066,248,221,147,2341,3631,2869,148
I need to do laundry and wash my clothes. => 286,1645,194,543,960,3981,2154,221,147,27589,1247,147,22141,383,148
They are planning a trip to Europe. => 18815,429,147,23661,216,147,11395,194,13131,148
She is learning how to play the guitar. => 10386,235,11754,2496,194,7084,170,147,4604,2214,148
We are going to a museum this Sunday. => 3250,429,6552,194,216,147,204,433,1177,291,147,29111,148
The coffee smells amazing in the morning. => 505,147,21526,31454,155,147,28880,219,170,20701,148
He is a hardworking farmer who grows crops. => 5301,235,216,8524,14992,147,16679,279,2888,147,6044,155,147,8650,155,148
The students are presenting their research projects. => 505,15707,429,5130,248,1669,13217,14235,148
I enjoy playing soccer with my friends. => 286,23162,7084,248,147,9351,5318,312,1247,147,5347,155,148
They are volunteering at a local shelter. => 18815,429,147,5238,7478,163,12798,536,216,2491,2905,1359,279,148
She is practicing martial arts for self-defense. => 10386,235,147,18453,248,147,3261,185,4381,12234,155,250,623,153,29896,148
We should try a new recipe for dinner. => 3250,936,147,746,216,277,147,9851,250,147,182,2749,148
The traffic is congest => 505,147,11097,235,1710,14169
The sun is shining brightly today. => 505,147,5852,235,147,7304,2967,147,215,649,391,13386,148
I enjoy reading books in my free time. => 286,23162,9838,147,9670,219,1247,4002,801,148
She plays the piano beautifully. => 10386,7084,155,170,147,207,23635,147,23447,391,148
The cat chased the mouse around the room. => 505,147,1604,147,196,916,246,170,12551,6890,170,9654,148
I love eating pizza with extra cheese. => 286,8440,147,163,3643,147,207,8403,312,8230,9784,383,163,148
He always wears a hat wherever he goes. => 5301,5418,147,16427,155,216,147,4879,2171,2433,1189,16177,148
The flowers in the garden are blooming. => 505,147,22953,155,219,170,147,22140,429,147,10411,2799,248,148
She danced gracefully on the stage. => 10386,13378,12408,147,267,8771,8001,347,170,12685,148
The dog barked loudly in the park. => 505,147,6540,147,973,293,246,147,30182,391,219,170,147,17664,148
We went swimming in the ocean yesterday. => 3250,10825,147,4729,8528,248,219,170,147,26193,147,28830,148
He speaks fluent French and Spanish. => 5301,147,13285,155,147,21677,147,254,17590,221,147,31519,148
The train arrived at the station on time. => 505,147,872,147,20712,182,536,170,147,7184,347,801,148
She cooked a delicious meal for her family. => 10386,147,20453,246,216,3936,23455,147,26658,250,1869,147,2002,148
ggml/examples/prompts/starcoder.txt
+100
View File
@@ -0,0 +1,100 @@
Hello World! => 8279,10896,19
I can't believe it's already Friday!" => 59,883,1330,13710,561,1182,3425,506,25674,11555
The URL for the website is https://www.example.com." => 1318,3834,436,322,9575,438,1678,555,1499,32,2763,32,508,3107
"She said, 'I love to travel.'" => 20,25387,9884,30,330,59,14290,372,25283,29329
'The temperature is 25.5°C.' => 25,1318,13587,438,225,36,39,32,39,23767,53,4564
"Let's meet at 2:30 p.m. in the park." => 20,9809,1182,18450,821,225,36,44,37,34,298,32,95,32,328,322,880,93,3107
The book costs $19.99 => 1318,7618,25950,398,35,43,32,43,43
"John's favorite color is blue." => 20,19693,1182,27448,1963,438,10087,3107
Th@nk y0u f0r y0ur h3lp! => 1027,50,19877,533,34,103,296,34,100,533,34,305,420,37,1915,19
C@n I g3t a c0ffee, pl3@se? => 53,50,96,439,485,37,102,312,281,34,21298,30,1278,37,50,277,49
W0w! Th@t's @m@zing! => 73,34,105,19,947,50,102,1182,477,95,50,26768,19
H0w 4re y0u t0d@y? => 58,34,105,225,38,268,533,34,103,273,34,86,50,107,49
I l0ve t0 tr@vel @r0und the w0rld. => 59,456,34,587,273,34,554,50,1203,477,100,34,642,322,341,34,100,1381,32
Wh@t's y0ur f@v0rite m0vie? => 2444,50,102,1182,533,34,305,296,50,104,34,1049,345,34,104,1075,49
The cat is sleeping on the mat. => 1318,10501,438,9368,299,544,322,2491,32
I need to buy some groceries for dinner. => 59,1849,372,16968,1629,20234,85,6958,436,343,3369,32
The sun is shining brightly in the sky. => 1318,15323,438,787,19068,38231,631,328,322,26718,32
She is reading a book in the park. => 25387,438,9175,312,7618,328,322,880,93,32
We went for a walk on the beach yesterday. => 3122,14236,436,312,13503,544,322,526,867,39485,32
He plays the guitar like a pro. => 1331,41271,322,3932,19931,2124,312,534,32
They are going to the movies tonight. => 31805,884,6783,372,322,27889,26076,694,32
The flowers are blooming in the garden. => 1318,7290,483,884,323,18466,299,328,322,485,22461,32
I enjoy listening to classical music. => 59,31567,20498,372,443,1578,17522,32
We need to buy groceries for the week. => 3122,1849,372,16968,20234,85,6958,436,322,8209,32
The dog is chasing its tail in circles. => 1318,27435,438,663,9949,2819,13203,328,46428,32
She is wearing a beautiful red dress. => 25387,438,996,6992,312,36493,3346,343,714,32
He is a talented actor in Hollywood. => 1331,438,312,273,9556,318,16038,328,48228,631,21118,32
The children are playing in the playground. => 1318,5713,884,19788,328,322,4654,1749,32
I'm going to visit my grandparents this weekend. => 59,3464,6783,372,7725,1672,33162,19277,458,40618,32
The coffee tastes bitter without sugar. => 1318,36917,273,633,307,3493,391,2876,309,18628,32
They are planning a surprise party for her. => 31805,884,26116,312,6178,9251,15270,436,7791,32
She sings like an angel on stage. => 25387,309,2052,2124,600,600,17691,544,10019,32
We should take a vacation to relax. => 3122,1395,4818,312,29164,367,372,41972,32
He is studying medicine at the university. => 1331,438,14866,299,32388,482,821,322,707,9190,32
The rain is pouring heavily outside. => 1318,36987,438,9202,299,46003,2801,11127,32
I enjoy watching romantic movies. => 59,31567,37652,26045,7268,27889,32
They are celebrating their anniversary today. => 31805,884,48278,839,1741,3623,23921,5810,672,11610,32
She dances gracefully to the music. => 25387,343,3151,31376,4938,372,322,17522,32
He is an excellent basketball player. => 1331,438,600,39203,48400,11653,4362,32
The baby is sleeping soundly in the crib. => 1318,323,17156,438,9368,299,9934,631,328,322,281,7972,32
I need to finish my homework before dinner. => 59,1849,372,11361,1672,6765,1007,2670,343,3369,32
They are organizing a charity event next month. => 31805,884,10558,6183,312,1351,543,1692,2354,6811,32
She is cooking a delicious meal for us. => 25387,438,23682,299,312,409,406,2406,597,279,436,1770,32
We should go hiking in the mountains. => 3122,1395,1983,420,1546,299,328,322,10874,1907,32
The car broke down on the way to work. => 1318,6346,43289,2835,544,322,3352,372,1389,32
He loves playing video games in his free time. => 1331,598,4954,19788,6027,19705,328,6697,3741,1133,32
The birds are chirping in the trees. => 1318,8424,3210,884,663,476,7075,328,322,23453,32
I want to learn how to play the piano. => 59,2637,372,7350,2624,372,4654,322,298,25757,32
They are building a new shopping mall in the city. => 31805,884,9038,312,537,40692,345,464,328,322,11297,32
She is writing a novel in her spare time. => 25387,438,4127,312,32913,328,7791,1869,586,1133,32
We are going to the zoo this Saturday. => 3122,884,6783,372,322,1288,604,458,358,30288,32
The cake looks delicious with chocolate frosting. => 1318,281,1062,7780,409,406,2406,623,10408,27589,296,20932,299,32
He is a talented painter who sells his artwork. => 1331,438,312,273,9556,318,42300,6560,10800,101,6697,5549,1007,32
The students are studying for their exams. => 1318,16512,884,14866,299,436,3623,538,1462,32
I enjoy swimming in the ocean. => 59,31567,2535,449,6714,328,322,337,18857,32
They are renovating their house. => 31805,884,316,15007,1741,3623,17075,32
She is practicing yoga to stay healthy. => 25387,438,11808,11636,533,40067,372,20005,44538,32
We should plant flowers in the garden. => 3122,1395,26795,7290,483,328,322,485,22461,32
The traffic is heavy during rush hour. => 1318,16391,438,32389,5929,540,1372,12021,32
He is a skilled chef who creates amazing dishes. => 1331,438,312,3001,12088,44051,6560,9585,36986,1214,4279,32
The baby is crawling on the floor. => 1318,323,17156,438,281,1294,2920,544,322,17648,32
I need to buy a new pair of shoes. => 59,1849,372,16968,312,537,6092,432,787,37764,32
They are going on a road trip across the country. => 31805,884,6783,544,312,24122,19337,10160,322,10769,32
She is playing the piano beautifully. => 25387,438,19788,322,298,25757,526,4846,325,514,107,32
We are going to a concert tomorrow night. => 3122,884,6783,372,312,457,6989,31841,19212,32
The cake tastes delicious with vanilla frosting. => 1318,281,1062,273,633,307,409,406,2406,623,44653,296,20932,299,32
He is a dedicated teacher who inspires his students. => 1331,438,312,23112,30877,6560,26194,8017,6697,16512,32
The students are participating in a science fair. => 1318,16512,884,24623,1741,328,312,27536,19375,32
I enjoy hiking in the mountains. => 59,31567,420,1546,299,328,322,10874,1907,32
They are organizing a beach cleanup next weekend. => 31805,884,10558,6183,312,526,867,13144,2354,40618,32
She is taking photographs of nature. => 25387,438,15137,15110,23626,432,24406,32
We should try a new restaurant in town. => 3122,1395,1596,312,537,43719,328,38212,32
The traffic is moving slowly on the highway. => 1318,16391,438,14089,12899,631,544,322,3857,3073,32
He is a talented singer with a beautiful voice. => 1331,438,312,273,9556,318,309,10118,623,312,36493,20309,32
The baby is laughing and giggling. => 1318,323,17156,438,2317,2943,299,461,485,365,36088,32
I need to do laundry and wash my clothes. => 59,1849,372,745,2317,642,994,461,341,917,1672,7375,46948,32
They are planning a trip to Europe. => 31805,884,26116,312,19337,372,27268,32
She is learning how to play the guitar. => 25387,438,9608,2624,372,4654,322,3932,19931,32
We are going to a museum this Sunday. => 3122,884,6783,372,312,345,539,378,458,358,28036,32
The coffee smells amazing in the morning. => 1318,36917,309,42153,101,36986,328,322,33768,32
He is a hardworking farmer who grows crops. => 1331,438,312,6784,13578,9019,2302,6560,485,2138,25170,1069,32
The students are presenting their research projects. => 1318,16512,884,5024,299,3623,13234,8528,32
I enjoy playing soccer with my friends. => 59,31567,19788,22682,10035,623,1672,22523,32
They are volunteering at a local shelter. => 31805,884,3920,45585,8637,821,312,2196,309,2542,391,32
She is practicing martial arts for self-defense. => 25387,438,11808,11636,345,502,564,5549,101,436,630,31,43694,32
We should try a new recipe for dinner. => 3122,1395,1596,312,537,15233,436,343,3369,32
The traffic is congest => 1318,16391,438,457,2776
The sun is shining brightly today. => 1318,15323,438,787,19068,38231,631,11610,32
I enjoy reading books in my free time. => 59,31567,9175,21739,328,1672,3741,1133,32
She plays the piano beautifully. => 25387,41271,322,298,25757,526,4846,325,514,107,32
The cat chased the mouse around the room. => 1318,10501,663,16109,322,8459,6835,322,8355,32
I love eating pizza with extra cheese. => 59,14290,484,1741,47630,623,6717,8277,30315,32
He always wears a hat wherever he goes. => 1331,5182,996,4177,312,25793,2154,424,938,13107,32
The flowers in the garden are blooming. => 1318,7290,483,328,322,485,22461,884,323,18466,299,32
She danced gracefully on the stage. => 25387,343,6087,31376,4938,544,322,10019,32
The dog barked loudly in the park. => 1318,27435,323,1087,318,598,836,631,328,322,880,93,32
We went swimming in the ocean yesterday. => 3122,14236,2535,449,6714,328,322,337,18857,39485,32
He speaks fluent French and Spanish. => 1331,24498,101,38055,43652,461,14911,1708,32
The train arrived at the station on time. => 1318,5683,2099,32114,821,322,18662,544,1133,32
She cooked a delicious meal for her family. => 25387,23682,318,312,409,406,2406,597,279,436,7791,13872,32

Some files were not shown because too many files have changed in this diff Show More