civ/hot-step-cpp-ROCm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial release

Browse Source
This commit is contained in:
civ
2026-08-16 18:24:52 +07:00
commit 876886a39a
13244 changed files with 2353959 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+88
View File
@@ -0,0 +1,88 @@
# ============================================================================
# HOT-Step 9000 CPP — Docker Build Context Exclusions
# Keep the build context small and fast by excluding everything not needed
# for the Docker build.
# ============================================================================
# ── Git / Version Control ───────────────────────────────────────────
.git/
.gitmodules
# ── Engine build artifacts (rebuilt inside Docker) ──────────────────
engine/build/
engine/deps/
engine/.cache/
engine/__pycache__/
# ── TensorRT-LLM (not used in Phase 1) ─────────────────────────────
engine/trtllm-libs/
engine/trtllm-include/
# ── Node.js dependencies (reinstalled inside Docker) ────────────────
server/node_modules/
server/dist/
ui/node_modules/
# ── Runtime data (bind-mounted at runtime, not baked in) ────────────
server/data/
models/
adapters/
checkpoints/
data/
logs/
# ── Build / Release artifacts ───────────────────────────────────────
build/
release/
release-staging/
runtime/
benchmark-results/
# ── Agent / IDE / Dev tooling ───────────────────────────────────────
.agents/
.agent/
.claude/
.gemini/
.vscode/
.idea/
scratch/
docs/plans/
# ── Windows-only binaries and objects ───────────────────────────────
*.exe
*.dll
*.lib
*.obj
*.pdb
*.ilk
*.exp
# ── Model weights (mounted at runtime) ─────────────────────────────
*.gguf
*.safetensors
*.pt
*.pth
*.onnx
# ── Audio files (generated at runtime) ──────────────────────────────
*.wav
*.mp3
*.flac
*.ogg
*.aac
*.wma
*.m4a
# ── EXCEPT noise samples (baked into image) ─────────────────────────
!noise_samples/
!noise_samples/*.wav
# ── Misc ────────────────────────────────────────────────────────────
__pycache__/
*.pyc
*.tar.gz
nul
Thumbs.db
.DS_Store
.env
*.patch
.env.docker
+41
View File
@@ -0,0 +1,41 @@
# HOT-Step CPP — Docker Environment Configuration
# This file configures the app for running inside a Docker container.
# Paths reference the container filesystem layout.
# ── Engine ──────────────────────────────────────────────────────────
# Binary location (Ninja single-config layout — no Release/ subdirectory)
ACESTEPCPP_EXE=/app/engine/ace-server
# Model + adapter directories (bind-mounted from host)
ACESTEPCPP_MODELS=/app/models
ACESTEPCPP_ADAPTERS=/app/adapters
# ── Path Mapping (Windows → Docker) ─────────────────────────────────
# Translates Windows-native paths in album presets to container mounts.
# JSON object: { "<windows_prefix>": "<container_mount>" }
# Keeps presets working on both Windows and Docker without modification.
DOCKER_PATH_MAP={"D:\\Ace-Step-Latest\\All LoKR Files\\sidestep\\xl-base-turbo-05":"/app/adapters","D:\\Ace-Step-Latest\\Datasets-LoRA-LoKR":"/app/datasets"}
# Bind to all interfaces (required for Docker port mapping to work)
ACESTEPCPP_HOST=0.0.0.0
ACESTEPCPP_PORT=8085
# CUDA graph optimization (concurrent streams for Q/K/V attention branches)
# Free 7-15% speedup on DiT denoising — 50 identical forward passes = perfect for graphs
GGML_CUDA_GRAPH_OPT=1
# ── Server ──────────────────────────────────────────────────────────
SERVER_PORT=3001
SERVER_HOST=0.0.0.0
# Data directory (bind-mounted from host — SQLite DB + generated audio)
DATA_DIR=/app/server/data
# ── Vite (not used in Docker — UI is pre-built) ────────────────────
# VITE_PORT=3000
# VITE_HOST=0.0.0.0
# ── Lyric Studio (optional — add API keys as needed) ────────────────
# GENIUS_ACCESS_TOKEN=
# GEMINI_API_KEY=
# DEFAULT_LLM_PROVIDER=gemini
.env.example
+69
View File
@@ -0,0 +1,69 @@
# HOT-Step CPP Configuration
# This file is OPTIONAL — smart defaults work without it.
# Copy to .env and edit only if you need to override defaults.
# ace-server executable
# Default: ./engine/build/Release/ace-server.exe (built from source)
# ACESTEPCPP_EXE=D:\custom\path\to\ace-server.exe
# Model directory (GGUF files)
# Default: ./models/
# ACESTEPCPP_MODELS=D:\custom\path\to\models
# Adapter directory (LoRA safetensors)
# Default: ./adapters/
# ACESTEPCPP_ADAPTERS=D:\custom\path\to\adapters
# ace-server network settings (rarely need changing)
ACESTEPCPP_PORT=8085
ACESTEPCPP_HOST=127.0.0.1
# Node.js server
SERVER_PORT=3001
DATA_DIR=./server/data
# Dev mode (Vite)
VITE_PORT=3000
VITE_HOST=0.0.0.0
# -----------------------------------------------------------------------------
# Lyric Studio (Lireek) Integration
# -----------------------------------------------------------------------------
# Genius API
GENIUS_ACCESS_TOKEN=
# LLM Providers (at least one required for Lyric Studio)
DEFAULT_LLM_PROVIDER=gemini
GEMINI_API_KEY=
OPENAI_API_KEY=
ANTHROPIC_API_KEY=
# Local LLM Providers
OLLAMA_BASE_URL=http://localhost:11434
OLLAMA_MODEL=llama3
LMSTUDIO_BASE_URL=http://localhost:1234/v1
LMSTUDIO_MODEL=
UNSLOTH_BASE_URL=http://127.0.0.1:8888
UNSLOTH_USERNAME=
UNSLOTH_PASSWORD=
UNSLOTH_MODEL=
# llama.cpp server (llama-server / llama-cli --server)
LLAMACPP_BASE_URL=http://127.0.0.1:8080/v1
LLAMACPP_MODEL=
# Generic OpenAI-compatible endpoint (oMLX, vLLM, LocalAI, etc.)
# Only appears in the UI when BASE_URL is set.
OPENAI_COMPAT_BASE_URL=
OPENAI_COMPAT_API_KEY=
OPENAI_COMPAT_MODEL=
OPENAI_COMPAT_NAME=OpenAI Compatible
# Models to use
GEMINI_MODEL=gemini-2.5-flash
OPENAI_MODEL=gpt-4o-mini
ANTHROPIC_MODEL=claude-3-5-haiku-20241022
# Storage
LYRICS_EXPORT_DIR=./data/lyrics
.gitignore
+147
View File
@@ -0,0 +1,147 @@
# Dependencies
node_modules/
# Build output
ui/dist/
server/dist/
build/
# Data (local, not committed)
server/data/
server/server/data/
# Root runtime data (SQLite DB, generated audio/latents) and model weights —
# anchored to root: server/src/data/ and ui/src/data/ are tracked source
/data/
/models/
# Local machine config
/.mcp.json
/SafeSettings.json
*.db
*.db-shm
*.db-wal
*.db-journal
# Session logs (generated per-run)
logs/
# Environment (contains local paths)
.env
# Restart marker (transient, created by /api/restart)
.restart-requested
# Engine build artifacts
engine/acestep.cpp/
engine/build/
engine/deps/
engine/.cache/
# Binaries & compiled objects
*.exe
!Essentia/*.exe
*.dll
*.lib
*.obj
*.o
*.a
*.so
*.dylib
*.pdb
*.ilk
*.exp
# Audio files
*.wav
!noise_samples/*.wav
*.mp3
*.flac
*.ogg
*.aac
*.wma
*.m4a
*.bf16
# Model weights
*.gguf
*.safetensors
*.pt
*.pth
*.onnx
# ONNX external data files (dit-stream has ~500 weight files with no extension)
models/onnx/dit-stream/
# Downloaded tools (whisper.cpp, etc.)
tools/*
!tools/onnx-export/
!tools/mcp-lyricstudio/
tools/mcp-lyricstudio/node_modules/
# Test output logs
engine/tests/*.log
# Python
__pycache__/
*.pyc
*.pyo
# OS
Thumbs.db
.DS_Store
# IDE
.vscode/
.idea/
# Agents & dev tooling (local only)
.agents/
.agent/
# .claude/ is local-only EXCEPT the committed skill library
.claude/*
!.claude/skills/
.gemini/
# Dev-only scripts (local only, not needed by end users)
dev.bat
dev-rebuild.bat
server/restart-loop.cmd
# Local-only content
benchmark-results/
docs/plans/
scratch/
# Co-dev drop folder (VST plugins etc. — install locally, never commit)
toinstall/
# Co-dev outbound folder (modified plugin copies to send back)
tosend/
# Release build artifacts (but keep scripts and configs)
release/staging/
release/out/
release/.node-cache/
release/node_modules/
release/package-lock.json
release-staging/
runtime/
*.tar.gz
# Windows NUL device phantom
nul
# i18n dev-only utility scripts
ui/add_strings.mjs
ui/temp_strings.json
# TensorRT-LLM (local dev only)
engine/trtllm-libs/
engine/trtllm-include/
# TRT experiment files (local dev only)
test_trtllm_*.cpp
test_trtllm_*.h
create_test_onnx.py
test_smoke.*
rebuild_engine.*
Important.md
# Docker (container-specific overrides, not committed)
docker-compose.override.yml
.gitmodules
+6
View File
@@ -0,0 +1,6 @@
[submodule "engine/vendor/vst3sdk"]
path = engine/vendor/vst3sdk
url = https://github.com/steinbergmedia/vst3sdk.git
[submodule "engine/ggml"]
path = engine/ggml
url = https://github.com/ggml-org/ggml.git
CLAUDE.md
+99
View File
@@ -0,0 +1,99 @@
# CLAUDE.md — HOT-Step CPP
Orientation map for agents. Keep this short and navigational — point at the deep docs, don't duplicate them.
## What this is
A desktop app for **local AI music generation** — a heavily-extended superset of [acestep.cpp](https://github.com/ServeurpersoCom/acestep.cpp) (a C++/GGML port of ACE-Step 1.5). Caption + lyrics in → stereo 48 kHz audio out, fully local. Ships as portable releases (Windows CUDA/Vulkan/CPU, Linux, macOS Metal). GitHub: `scragnog/HOT-Step-CPP`.
## Architecture (3 tiers)
| Tier | Stack | Location | Role |
|------|-------|----------|------|
| **Engine** | C++17 / CUDA / GGML | [engine/](engine/) | Inference binaries: `ace-lm`, `ace-synth`, `ace-server`, `ace-understand`, `neural-codec`, `mp3-codec`, `quantize`. Pipeline: LM → DiT → VAE |
| **Server** | Node / TypeScript / Express / better-sqlite3 | [server/src/](server/src/) | Orchestrates the engine, manages songs/jobs/SQLite, serves UI. Per-feature [routes/](server/src/routes/) + [services/](server/src/services/) |
| **UI** | React 19 / Vite / Zustand / Tailwind | [ui/src/](ui/src/) | Browser frontend. Component folder per "studio" |
```
LAUNCH.bat → Node server (Express :3001)
├── serves React frontend (prebuilt ui/dist/)
├── /api/* → SQLite
└── spawns child: ace-server.exe (C++ engine) on :8085
```
| Service | Port |
|---------|------|
| Node server | 3001 (prod) |
| Vite dev server | 3000 (dev, HMR) |
| ace-server (C++ engine) | 8085 (default, `config.ts`) |
## Environment
- **Windows 11 + PowerShell.** This repo's primary dev environment is Windows. The Claude Code harness also gives you a Bash (POSIX) tool — each takes its own syntax. In PowerShell use `;` not `&&`.
- **Node 18–22 LTS only.** Node 24+ breaks dependencies (`engines` field enforces `<24`).
## Build & run rules (IMPORTANT — learned the hard way)
- **C++ engine changes → `dev-rebuild.bat`, NEVER `engine/build.cmd` directly.** The Node server auto-respawns ace-server on crash; killing it without clean shutdown causes an infinite respawn + file-lock loop. `dev-rebuild.bat` handles clean shutdown + rebuild — it does **not** relaunch; start the app again yourself with `dev.bat`/`LAUNCH.bat`.
- Recompile **immediately** after editing any `engine/src/` or `engine/tools/` file — don't wait to be asked.
- **NEVER `cmake --build . --clean-first`** unless the GGML/CUDA layer itself changed — CUDA kernel recompilation is **20+ min**. For stale `.obj` issues, delete only `engine/build/acestep-core.dir/` and `engine/build/Release/acestep-core.lib`.
- **Don't `npm run build` during dev.** Type-check with `npx tsc --noEmit`. Only build before user testing.
- **`dev.bat`** = dev mode (Vite :3000 HMR + Node :3001, tsx watch auto-restart). **`LAUNCH.bat`** = prod. Use `dev.bat` for development.
## Git rules
- **All work on `master`. No feature branches, ever.**
- **Never `git add -A`** (re-adds gitignored dirs: `.agents/`, `checkpoints/`, `node_modules/`, etc.). **Never `git add -f`** on gitignored paths. Stage explicit paths.
- **Push requires explicit user approval — always ask first.**
- Commit to local git **often** (data has been lost before to uncommitted files).
- **Releases:** push a `vX.Y.Z` tag → the `Release` workflow builds all platforms and drafts a GitHub Release. **Any pushed `v*` tag triggers a build** — use a `-CI-Test` suffix for throwaway compile checks, and don't push local feature tags matching `v*`. Full process + gotchas: [docs/RELEASING.md](docs/RELEASING.md).
- Use `gh` CLI for GitHub ops (authenticated as `scragnog`).
## Upstream sync (fork hooks that break silently)
The C++ engine is a patched fork of acestep.cpp. Three upstream files carry HOT-Step `#include` hooks that break if overwritten during a sync:
| Upstream file | Hook | If lost |
|---|---|---|
| `pipeline-synth-ops.cpp` | `hot-step-sampler.h` (replaces `dit-sampler.h`) | **SILENT** — compiles, but all solvers/guidance/schedulers go dead |
| `model-store.h` | `hot-step-params.h` | compile error |
| `dit.h` | `adapter-merge.h` + `adapter-runtime.h` | compile error |
After any sync: run `engine/verify-hooks.ps1`. Full process: `docs/plans/upstream-sync-workflow.md` *(local, gitignored)*.
## UI / browser verification
- **Don't use the built-in browser agent to visually verify UI** — too slow/unreliable here. **Ask the user to check**; they provide screenshots/feedback. Browser agent is fine for non-visual tasks (hitting API endpoints).
## Debugging — logs
App writes per-session logs to `logs/` at repo root:
```
logs/YYYY-MM-DD_HH-MM-SS/ ← one folder per session (name-sorted = time-sorted)
├── ace_engine.log ← C++ engine output
├── node_console.log ← Node server output
└── generations/gen_<uuid>_<task>.log
```
Start with the newest session folder. Generation failures → matching `gen_*.log` first, then cross-ref `ace_engine.log` + `node_console.log`. Startup/crash → `node_console.log` + `ace_engine.log`.
## Plugin system
Solvers (17), schedulers (9), guidance modes, and postprocess are **hot-loadable Lua plugins** in [engine/plugins/](engine/plugins/) — drop a `.lua` in the right subdir, appears in the UI next launch, no C++ rebuild. Each plugin can declare its own UI params. Native C++ bridge via `apg()`; advanced plugins use `post_step()` for extra forward passes. **Adding a solver/scheduler/guidance = write a `.lua` plugin** (the old approach of editing `dit-sampler.h` is obsolete — the engine now routes through `hot-step-sampler.h`). Authoring guide: [docs/PLUGINS.md](docs/PLUGINS.md).
## Read-Y-for-X index
| For… | Read |
|------|------|
| **Any maintenance task — start here** (per-domain procedures, gotchas, distilled institutional knowledge) | [.claude/skills/README.md](.claude/skills/README.md) — 13 fact-checked skills |
| Full feature catalogue (100+) | [FEATURES.md](FEATURES.md) |
| Engine internals, CLI, request JSON, generation modes | [engine/docs/ARCHITECTURE.md](engine/docs/ARCHITECTURE.md) |
| **Training system** (dataset→preprocess→LM/DiT training→audition; ace-train, FSQ, ggml training gotchas) | [docs/TRAINING.md](docs/TRAINING.md) |
| Writing a Lua plugin | [docs/PLUGINS.md](docs/PLUGINS.md) |
| Build / install / releases | [README.md](README.md) |
| Cutting & publishing a release (agent runbook) | [docs/RELEASING.md](docs/RELEASING.md) |
| Internal design/investigation docs (perf, adapters, upstream sync, feature designs) | `docs/plans/` *(gitignored, local-only)* |
| In-app assistant behaviour/KB | [server/src/data/assistant-knowledge.md](server/src/data/assistant-knowledge.md) |
> **Doc convention:** committed contributor-facing docs = `README.md`, `FEATURES.md`, `docs/PLUGINS.md`, `engine/docs/ARCHITECTURE.md`. Internal planning/investigation docs live in `docs/plans/`, which is **gitignored** (local only). This file (`CLAUDE.md`) is committed.
Dockerfile
+130
View File
@@ -0,0 +1,130 @@
# ============================================================================
# HOT-Step 9000 CPP — Docker Build
# Multi-stage: Engine (CUDA) → UI (Vite) → Server deps → Runtime
#
# Usage:
# docker compose build # Dev build (Blackwell only)
# docker compose build --build-arg CUDA_ARCHS="75;80;86;89;90;120a" # Distribution
# ============================================================================
# ── Stage 1: Engine Builder ─────────────────────────────────────────
# CUDA devel image: has nvcc, CUDA headers, cuDNN for building
FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 AS engine-builder
# Default to Blackwell (sm_120a) for fast dev builds.
# Override with --build-arg for multi-arch distribution.
ARG CUDA_ARCHS="120a"
RUN apt-get update && apt-get install -y --no-install-recommends \
cmake ninja-build build-essential git curl ca-certificates \
&& rm -rf /var/lib/apt/lists/*
# TensorRT SDK for DiT/LM acceleration (native TRT API)
RUN apt-get update && apt-get install -y --no-install-recommends \
libnvinfer-dev \
libnvinfer-plugin-dev \
libnvonnxparsers-dev \
&& rm -rf /var/lib/apt/lists/*
# Download ONNX Runtime GPU SDK (Linux x64) for SuperSep stem separation
ARG ORT_VERSION=1.25.1
RUN curl -L "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-gpu-${ORT_VERSION}.tgz" \
| tar xz -C /opt \
&& mv "/opt/onnxruntime-linux-x64-gpu-${ORT_VERSION}" /opt/onnxruntime
WORKDIR /build
COPY engine/ .
# Build the C++ engine with CUDA + ONNX Runtime
RUN cmake -B build -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DGGML_CUDA=ON \
-DGGML_CUDA_GRAPHS=ON \
-DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
-DGGML_NATIVE=OFF \
-DGGML_CPU_ALL_VARIANTS=ON \
-DGGML_BACKEND_DL=ON \
-DORT_ROOT=/opt/onnxruntime \
&& cmake --build build --config Release -j"$(nproc)"
# Stage all binaries + shared libs into /staging for clean COPY
RUN mkdir -p /staging/engine \
&& for bin in ace-server mastering mp3-codec vst-host; do \
[ -f "build/${bin}" ] && cp "build/${bin}" /staging/engine/; \
done \
&& find build/ -maxdepth 1 -name '*.so' -exec cp {} /staging/engine/ \; \
&& find build/ -maxdepth 1 -name '*.so.*' -exec cp {} /staging/engine/ \; \
&& cp /opt/onnxruntime/lib/libonnxruntime*.so* /staging/engine/ 2>/dev/null || true
# ── Stage 2: UI Builder ─────────────────────────────────────────────
FROM node:22-slim AS ui-builder
WORKDIR /build/ui
COPY ui/package*.json ./
RUN npm install
COPY ui/ .
RUN npx vite build
# ── Stage 3: Server Dependencies ────────────────────────────────────
# Full node image (not slim) — better-sqlite3 needs Python + g++ for native build
FROM node:22 AS server-deps
WORKDIR /build/server
COPY server/package*.json ./
# Install production deps. tsx is in both devDependencies and optionalDependencies,
# but npm --omit=dev deduplicates and skips it. Install explicitly.
RUN npm install --omit=dev && npm install tsx
# ── Stage 4: Runtime ────────────────────────────────────────────────
FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04
# Install Node.js 22 (LTS) + runtime libraries the engine needs
# TensorRT 11 runtime libraries for native DiT/LM acceleration (dit-trt.h)
# Note: ORT TRT EP needs TRT 10 (libnvinfer.so.10) but segfaults due to
# version mismatch with CUDA 12.8. ORT falls back to CUDA EP gracefully
# which is fine for the small text/cond encoders. Native TRT 11 handles DiT.
RUN apt-get update && apt-get install -y --no-install-recommends \
libnvinfer11 \
libnvinfer-plugin11 \
libnvonnxparsers11 \
&& rm -rf /var/lib/apt/lists/*
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl ca-certificates libgomp1 \
&& curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
&& apt-get install -y --no-install-recommends nodejs \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Engine binaries + shared libraries (GGML backends, ORT, cuDNN)
COPY --from=engine-builder /staging/engine/ /app/engine/
# NOTE: Lua plugins are bind-mounted at runtime via docker-compose.yml
# Server source code + production dependencies
COPY server/ /app/server/
COPY --from=server-deps /build/server/node_modules/ /app/server/node_modules/
# UI static files (production build)
COPY --from=ui-builder /build/ui/dist/ /app/ui/dist/
# Noise samples (small WAV files for noise profiling, baked into image)
COPY noise_samples/ /app/noise_samples/
# Docker-specific environment config
COPY .env.docker /app/.env
# Entrypoint script
COPY docker/entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh
# GGML backends + ORT need to find their .so files
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/app/engine:${LD_LIBRARY_PATH}
ENV NODE_ENV=production
EXPOSE 3001 8085
ENTRYPOINT ["/app/entrypoint.sh"]
Essentia/essentia_streaming_extractor_music.exe
BIN
View File
Binary file not shown.
FEATURES.md
+400
View File
@@ -0,0 +1,400 @@
# HOT-Step CPP — Features
Everything HOT-Step CPP adds on top of the base [acestep.cpp](https://github.com/ServeurpersoCom/acestep.cpp) engine.
---
## C++ Inference Engine
Built on acestep.cpp (GGML/CUDA), with extensive modifications to the sampling, scheduling, and guidance systems:
### Lua Plugin Architecture
All solvers, schedulers, guidance modes, and postprocess plugins are implemented as hot-loadable Lua plugins. Drop a `.lua` file into the appropriate `engine/plugins/` subdirectory and it appears in the UI at next launch — no C++ rebuild required. Each plugin can declare its own user-facing parameters (sliders, toggles, dropdowns) via a schema table, which the UI renders dynamically. Solvers can declare `owns_loop=true` to take full control of the denoising loop for adaptive solvers like DOPRI5.
The engine provides a native C++ bridge for performance-critical operations (APG momentum smoothing, perpendicular projection, norm thresholding) that Lua plugins can call via the `apg()` function. Advanced plugins can also declare a `post_step()` hook that receives model evaluation callbacks for techniques requiring extra forward passes at arbitrary latent positions. See the **[Plugin Authoring Guide](PLUGINS.md)** for the full API reference.
#### Solvers (17)
ODE/SDE solvers for the flow matching sampling loop:
| Plugin | Description |
|--------|-------------|
| **Euler** | 1st-order Euler method (1 NFE) |
| **Heun** | 2nd-order Heun's method (2 NFE) |
| **RK4** | Classic 4th-order Runge-Kutta (4 NFE) |
| **RK5** | 5th-order Runge-Kutta (6 NFE) |
| **GL2s** | Gauss-Legendre 2-stage implicit Runge-Kutta (2 NFE) |
| **RF-Solver** | 2nd-order rectified flow solver (2 NFE) |
| **DPM++ 2M** | DPM-Solver++ multistep 2nd-order (1 NFE) |
| **DPM++ 2M Adaptive** | Adaptive step-size variant of DPM++ 2M |
| **DPM++ 3M** | DPM-Solver++ multistep 3rd-order (1 NFE) |
| **UniPC** | Unified predictor-corrector (1 NFE) |
| **UniPC-P** | UniPC with p-corrector (1 NFE) |
| **JKASS Quality** | Multi-evaluation adaptive solver (4 NFE) |
| **JKASS Fast** | Single-evaluation JKASS variant (1 NFE) |
| **AFLOPS / AFLOPS-2** | Adaptive flow ODE solver with error estimation |
| **DOPRI5 / DOP853** | Dormand-Prince adaptive solvers (5th/8th order) |
| **SDE** | Stochastic differential equation solver with Philox RNG |
| **STORK-2 / STORK-4** | Stochastic Taylor Runge-Kutta solvers |
#### Schedulers (9)
Noise schedule curves for the denoising trajectory:
| Plugin | Description |
|--------|-------------|
| **Linear** | Uniform timestep spacing |
| **Cosine** | Cosine-annealed schedule |
| **Power** | Polynomial schedule with configurable exponent |
| **SGM Uniform** | Score-based generative model uniform schedule |
| **DDIM Uniform** | DDIM-style uniform schedule |
| **Linear-Quadratic** | Linear start transitioning to quadratic |
| **Beta (5,7)** | Beta distribution schedule |
| **Bong Tangent** | Tangent-based custom schedule |
| **Beta Math** | Generalised beta distribution with configurable α/β |
#### Guidance Modes (7)
Classifier-free guidance strategies, all routed through the native APG bridge:
| Plugin | Description |
|--------|-------------|
| **APG** | Analytical Perpendicular Guidance — momentum smoothing, perpendicular projection, norm thresholding |
| **Dynamic CFG** | Adaptive guidance scale that varies across timesteps — high early for structure, low late for detail |
| **CFG++** | Manifold-constrained guidance for few-step models |
| **Rescaled CFG** | Standard-deviation-based rescaling to prevent oversaturation |
| **CFG-Zero⋆** | Zero-init guidance — zeroes early ODE steps where CFG predictions are counterproductive (Fan et al. 2025) |
| **SMC-CFG** | Sliding Mode Control guidance — control-theoretic correction for stability at high scales (Han et al. 2025) |
| **CFG-MP** | Manifold Projection — iterative post-step projection using extra model evaluations to reduce the prediction gap (Su et al. 2025). Uses the `post_step()` hook for model callbacks |
#### Postprocess Plugins
Lua postprocess plugins that replace or augment the built-in VAE tiled decoder. Each plugin can declare its own UI parameters.
| Plugin | Description |
|--------|-------------|
| **MD Audio Tiled Core** | Advanced tiled VAE decode with OLA crossfading, dual-pass merge, and integrated DSP chain. By [MDMAchine](https://github.com/MDMAchine). |
### Other Engine Features
| Feature | Description |
|---------|-------------|
| **Composite 2-Stage Scheduler** | Blend two scheduler curves across the denoising trajectory for fine-grained noise control. |
| **Auto-Shift** | Adaptive noise shift scaling that adjusts based on track duration and step count. |
| **DCW Sampling** | Differential Correction in Wavelet domain — an alternative sampling technique calibrated for the GGML engine. |
| **Sideband Parameter Channel** | Extension layer for passing HOT-Step-specific parameters without modifying upstream function signatures, keeping the acestep.cpp sync path clean. |
| **Latent Post-Processing** | Latent shift, latent rescale, and custom timestep scheduling — expose the latent space for experimentation. |
| **LM Seed Locking** | Ties the LM seed to the DiT seed — locking the seed locks both, randomising randomises both. |
| **Upstream Sync Infrastructure** | Marker-based system for tracking acestep.cpp divergence and cleanly merging upstream changes. |
| **Safetensors Model Support** | Dual-format model loading — HuggingFace safetensors directories work alongside GGUF files for DiT, LM, Text Encoder, VAE, and Cond Encoder. Auto-detected by path (directory = safetensors, `.gguf` = GGUF). BF16 safetensors produce bit-perfect output vs BF16 GGUF. Format-agnostic `WeightSource` abstraction enables adapters to work with both base model formats. |
| **Cover Noise Method** | Configurable noise injection method for cover generation with rescale implementation. |
---
## LoRA / Adapter System
| Feature | Description |
|---------|-------------|
| **Per-Group Adapter Scales** | Independent scale control for self_attn, cross_attn, mlp, and cond_embed weight groups. |
| **K-Quant Adapter Support** | Custom CUDA copy kernels for GPU-accelerated merge of Q4_K_M, Q5_K_M, and Q6_K quantised adapters. |
| **Threaded CPU Dequant** | Multi-threaded CPU fallback with AVX512 fast-path for K-quant adapter merge when GPU copy isn't available. |
| **Runtime LoRA Mode** | Apply LoRA deltas in the forward pass graph at inference time, instead of permanently merging weights. Switchable per-generation. |
| **Adapter Browser** | File browser modal with scan endpoints, trigger word injection, and support for absolute paths outside the registry. |
| **Adapter Scale Override Presets** | Predefined scale profiles (e.g. "vocals up", "instruments up") selectable from the sidebar. |
| **Merge Model Detection** | Correctly identifies SFT-turbo blend models and skips inappropriate guidance clamping that would degrade output. |
| **Safetensors Base Model Support** | Adapter merge and runtime LoRA work with both GGUF and safetensors base models via the `WeightSource` abstraction — no format-specific code paths. |
---
## Audio Processing Pipeline
| Feature | Description |
|---------|-------------|
| **Lossless WAV Pipeline** | Engine outputs WAV16; 32-bit float WAV used throughout the processing chain to preserve dynamic range. |
| **Matchering Mastering Engine** | Integrated loudness, EQ, and dynamics matching to a user-supplied reference track. |
| **Mastered / Unmastered Toggle** | Instant A/B comparison via dual WaveSurfer instances with synced playback position. |
| **Spectral Denoiser** | Wiener-filter spectral subtraction for post-generation artifact removal (evolved from initial spectral gating approach). |
| **Profile-Based Denoiser** | Learns a noise profile from a reference sample for targeted, surgical artifact removal. |
| **Spectral Lifter** | Post-processing pipeline with tunable parameters for spectral shaping. Ported to native C++ (originally a Python subprocess). |
| **VST3 Host** | Scans, loads, and runs VST3 plugins for offline audio processing — 40+ plugins detected from standard install paths. |
| **VST3 Chain in Pipeline** | Wire a VST3 processing chain directly into the generation output — mastering, EQ, compression, etc. from your existing plugin collection. |
| **Real-Time Monitor (WASAPI)** | Low-latency audio preview with seek and transport controls for auditioning output before committing. |
| **Duration Buffer + Auto-Trim** | Generates slightly longer than requested, then detects natural song endings and trims cleanly — no more abrupt cuts. |
| **Configurable Fade-Out** | Slider-controlled fade duration; automatically skipped when auto-trim detects a clean ending. |
| **Download with Format Conversion** | Export as WAV, MP3, or FLAC with configurable defaults in Settings. |
| **48kHz Native Processing** | Mastering pipeline operates at the native 48kHz sample rate — no lossy resample round-trip. |
| **PP-VAE Neural Audio Polish** | Post-processing VAE that runs generated audio through an encode→decode round-trip to smooth spectral artifacts and improve tonal coherence. Optional wet/dry blend slider. F32 recommended for best quality. |
| **ScragVAE Decoder** | Fine-tuned VAE decoder with +38% high-frequency energy and +29dB dynamic range improvement. Drop-in replacement for the standard decoder — selectable at runtime from the Models dropdown. |
| **AI Cover Art** | Automatic 1024×1024 album cover art generation using FLUX.2-klein-4B via stable-diffusion.cpp. Downloads model + sd-cli binary on first use (~5.2 GB). Toggleable auto-generation after audio creation, plus on-demand "Generate Cover Art" from the song context menu. Prompts built from song subject or lyrics keywords. |
| **Vocal Naturalizer** ⚠️ | **Experimental.** 5-stage DSP humanization pipeline for AI-generated vocals. Applies vibrato injection, formant randomization, metallic reduction, quantization masking, and transition smoothing directly to the full mix using frequency-band-targeted filters. Runs between Spectral Lifter and VST Chain, automatically skipped on instrumentals. All parameters exposed as sliders in a dedicated accordion. **Note:** This feature is under active development and may subtly degrade audio quality or interfere with downstream VST/mastering processing. A/B test with it disabled to verify results. Ported from [ComfyUI_MusicTools](https://github.com/jeankassio/ComfyUI_MusicTools) (MIT License). |
| **Audio Quality Evaluator** | Automatic post-generation quality scoring via spectral analysis. Three weighted metrics: **Metallic Sound** (40%, spectral rolloff at 85th percentile), **Word Cuts** (40%, spectral flux discontinuities via z-score analysis), and **Noise/Hiss** (20%, zero-crossing rate). Produces a 0–100% score per track. Selectable target — evaluate unmastered (raw), mastered (post-processed), or both for direct comparison. Scores stored in the song database and displayed as colour-coded badges (green ≥80%, amber 50–79%, red <50%) in Library cards with per-metric hover tooltips. Pure TypeScript implementation using a custom radix-2 Cooley-Tukey FFT — no external DSP dependencies. Ported from [JK-AceStep-Nodes](https://github.com/jeankassio/JK-AceStep-Nodes) (MIT License). |
---
## UI / UX
### Create Modes
Two creation modes for different workflows, both sharing the same engine pipeline:
#### Auto-Gen
AI-driven song creation — minimal input, maximum automation:
| Feature | Description |
|---------|-------------|
| **Genre-First Workflow** | Select from a curated, searchable genre taxonomy to define the song's style. Random genre selection available. |
| **Three Lyric Modes** | Instrumental, AI-generated lyrics (with optional subject), or fully automated with random subject selection. |
| **LLM Lyric Generation** | External LLM writes lyrics, style caption, and title — supports Gemini, LM Studio, OpenAI-compatible providers. |
| **Preview Mode** | Toggle to review and edit AI-generated lyrics before committing to audio generation. |
| **Random Subject** | Let the LLM pick the topic — generates a subject, then lyrics for that subject, then a matching title. |
| **Random Genre** | One-click random genre selection from the full taxonomy. |
| **Serial Queue** | Jobs run one at a time through an internal queue — queue multiple while one generates. |
| **Live Progress** | Real-time stage updates (generating lyrics → resolving metadata → submitting → generating audio) with elapsed time. |
| **Structured LLM Metadata** | AI-generated metadata (BPM, duration, key, time signature) via structured LLM prompts with editable system prompt. Caption rewrite operates independently of LM skip. |
#### Custom-Gen
Full manual control for power users:
| Feature | Description |
|---------|-------------|
| **Complete Parameter Control** | Set style caption, lyrics, title, artist, BPM, duration, key signature, and time signature. |
| **Instrumental Toggle** | Switch between vocal and instrumental modes. |
| **Queue-Based Generation** | Queue multiple generations with configurable parallel job limits. |
| **Direct Engine Access** | All global engine settings (solvers, schedulers, guidance, adapters) apply directly. |
| **Artist-Title Metadata** | Fields for artist name, subject description, and key signature — embedded in generation metadata. |
### General UI
| Feature | Description |
|---------|-------------|
| **Full React + Tailwind UI** | Purpose-built dark-themed interface, ported and extended from the Python-based HOT-Step 9000. |
| **WaveSurfer.js Waveform Player** | Bars-mode waveform visualisation with hover plugin; animated collapse/expand on pause. |
| **Spectrum Analyzer** | audioMotion-analyzer integration with mirrored bar mode and configurable density. |
| **Global Parameter Top Bar** | All engine settings extracted into a persistent, colour-coded top bar with collapsible section dropdowns. |
| **VRAM Indicator** | Real-time GPU memory usage display in the top bar. |
| **Terminal Panel** | Verbose generation progress streamed via SSE with batched UI updates for performance. |
| **Generation Queue** | Queue additional generations while one is running; completed/cancelled jobs auto-dismiss after 3 seconds. |
| **JSON Preset Export / Import** | Save and load complete generation parameter sets as JSON files. |
| **Global Playlist Sidebar** | Persistent, resizable playlist replacing per-page floating players. |
| **Inline Song Rename** | Pencil icon on any track for quick title editing. |
| **Bulk Select & Delete** | Multi-select tracks in the library for batch deletion. |
| **Human-Readable Model Labels** | Friendly names for GGUF model files with enriched badge summaries showing quantisation and size. |
| **Toggle Switches** | All boolean controls use styled toggle switches instead of plain checkboxes. |
| **Persistent UI State** | Accordion states, sidebar collapse, scroll positions, and panel sizes all persist across navigation. |
| **Per-Track Download Buttons** | Download individual tracks directly from the playlist sidebar. |
| **A/B Comparison** | Dual-track playback for comparing two generations side by side. Global A/B mini-bar above the player for cross-view comparison with seed-locked comparison support. |
| **Library View Modes** | Three view modes — Grid (card overlay with cover art), List, and Table. Table mode has resizable columns with drag handles and localStorage persistence. |
| **Send to Playlist Toggle** | Toggle in the generation queue to auto-send completed tracks to the playlist sidebar. |
| **Player Stop Button** | Dedicated Stop button to decouple playbar collapse from pause behaviour. |
| **Model Descriptions** | Rich model descriptions shown in the Models tab dropdowns — each model displays its characteristics, recommended use case, and format badge (GGUF/ST). |
| **Format Badges** | Custom model dropdowns with visual GGUF/ST format badges to distinguish between quantised GGUF files and native safetensors models. |
| **Dynamic Plugin Parameters** | Solver, scheduler, guidance, and postprocess plugins can declare custom UI parameters (sliders, toggles, dropdowns) that render dynamically — no hardcoded UI needed. |
---
## Lyric Studio
A complete AI-powered lyrics and music generation workspace, powered by the Lireek backend:
| Feature | Description |
|---------|-------------|
| **Lireek Backend** | Full server-side lyric engine with SQLite database for artists, albums, profiles, and generations. |
| **LLM Orchestration** | 7 LLM provider integrations (Gemini, LM Studio, OpenAI-compatible, etc.) with real-time SSE streaming. |
| **Artist Profiles** | Per-artist configuration with adapter presets, reference tracks, style summaries, and computed generation statistics. |
| **Lyric Profiler** | Statistical analysis engine — contraction rates, rhyme schemes, meter patterns, perspective tracking — computed locally without LLM calls. |
| **Streaming Generation** | Real-time SSE streaming of lyrics with live UI updates as the LLM writes. |
| **Audio Generation Queue** | Integrated music generation from lyrics with full parameter parity to Custom-Gen. |
| **Bulk Operations** | "Fill to N" mode — auto-calculates how many generations each profile needs to reach a target count, with progress badges. |
| **Send to Custom-Gen** | Transfers artist context, adapter path, reference track, key signature, and all metadata to Custom-Gen in one click. |
| **Artist Sidebar** | Persistent sidebar with artist list, scroll position memory, and per-artist song counts. |
| **Album Pages** | Browse by album with header bars, generated songs tab, and inline audio playback. |
| **Database Migration** | Import tool for migrating from HOT-Step 9000’s `hotstep_lyrics.db` — artists, profiles, and generations. |
| **Dynamic LLM Model List** | Fetches available models from provider APIs instead of using a hardcoded list. |
| **Profile Stats Recalculation** | One-click re-run of all local statistical analysis without making any LLM calls. |
| **LRC Synced Lyrics** | Timestamped lyric display synced to audio playback with seeking support. |
| **Track Cropping** | Destructive IN/OUT point editing for trimming generated tracks to clean boundaries. |
| **Subject Field** | Optional subject field for guiding lyric generation — sets the topic without dictating specific content. |
---
## Cover Studio
Full-featured cover generation workspace with audio analysis, stem manipulation, and artist-specific generation:
| Feature | Description |
|---------|-------------|
| **Audio Analysis** | Essentia-based extraction of BPM, key, energy, and timbre characteristics from source tracks. |
| **Source Upload** | Upload and analyse reference audio for style-matched cover generation. Drag-and-drop with format auto-detection. |
| **BPM Correction** | ÷2 / Detected / ×2 buttons to fix Essentia’s common tempo halving/doubling errors. |
| **Key Override** | Manual key correction dropdown when Essentia’s detection is wrong — shows both detected and overridden keys. |
| **Style Description** | Editable caption field for describing the target style. Auto-filled from artist profile when available, freely editable. |
| **Artist-Optional Generation** | Generate covers using just a style description — no artist or adapter required. |
| **Pitch Shift** | ±12 semitone slider with real-time key transposition preview (e.g. “+3 st → F Major”). |
| **Tempo Scale** | 0.5x–2.0x tempo slider with computed BPM preview. |
| **Structure Fidelity** | Controls how closely the output follows the source’s arrangement and structure. |
| **Source Timbre** | Controls how much of the original artist’s sonic character is preserved in the output. |
| **Timbre Reference Conditioning** | Uses the target artist’s reference track as a DiT timbre conditioner to influence sonic character. |
| **Stem Separation + Recombination** | Advanced mode: split source into stems via SuperSep, configure the stem mix, then generate from the recombined audio. |
| **Album Adapter Presets** | Per-album adapter presets with bound reference tracks — select an album to auto-load the matching adapter and reference. |
| **Cover Generation UI** | Full workspace with metadata extraction, artist grid, cover-specific sliders, progress tracking, and recent covers list. |
| **Persistent State** | All settings, selections, and analysis results persist across navigation and reloads. |
---
## Repaint Studio
Region-based audio regeneration with waveform selection and synchronized lyrics editing:
| Feature | Description |
|---------|-------------|
| **Waveform Region Selector** | Visual waveform display with click-drag region selection for choosing which section of a track to regenerate. |
| **LRC Lyrics Editor** | Synchronized lyrics editor showing timestamped lyrics aligned to the selected region. |
| **Selective Regeneration** | Regenerate only the selected portion of a track while preserving the rest — fix problematic sections without re-generating the entire song. |
| **WIP Status** | Includes a dismissable notice banner indicating the feature is under active development. |
---
## Stem Studio
Neural audio source separation with a 4-stage ONNX pipeline and interactive stem mixer:
| Feature | Description |
|---------|-------------|
| **SuperSep Pipeline** | 4-stage cascaded separation using specialised ONNX models for different instrument groups. |
| **Stage 1: BS-RoFormer** | Primary 6-stem split (Vocals, Drums, Bass, Guitar, Piano, Other) using Band-Split RoFormer with full-track chunking. |
| **Stage 2: Mel-Band RoFormer** | Vocal sub-separation into Lead Vocals and Backing Vocals. Full-track processing with late-vocal detection. |
| **Stage 3: MDX23C** | Drum sub-separation into Kick, Snare, Toms, Hi-Hat, Cymbals, and Other Percussion via STFT-based MDX processing. |
| **Stage 4: HTDemucs** | Hybrid transformer for “Other” refinement — dual-input model taking both STFT spectrograms and raw waveforms, with dual-output combination. |
| **4 Separation Levels** | Basic (6 stems), Vocal Split (+ lead/backing), Full (+ drum sub-stems), Maximum (+ other sub-stems). |
| **Interactive Stem Mixer** | Multi-solo, mute, and per-stem volume sliders with real-time Web Audio playback. |
| **Chunking + Overlap-Add** | Full-length audio processing with 1-second crossfade windows for seamless chunk boundaries. |
| **Sequential VRAM Management** | Models loaded and released strictly sequentially — peak GPU usage stays under 3 GB. |
| **Per-Stage WAV Exports** | All stages generate raw WAVs in `stage-N/` directories for diagnostics, regardless of downstream routing. |
| **Hidden Intermediate Stems** | Debug stems (e.g. raw Vocals before lead/backing split) saved to disk but filtered from the UI mixer. |
| **MDX STFT Preprocessing** | Generic engine function for MDX23C and HTDemucs models with stripped STFT layers — handles the [1,4,dim_f,T] tensor layout. |
| **Source Library Browser** | Pick source audio from the song library with search, source filtering, and mastered/unmastered toggle. |
| **ZIP Download** | Download all stems as a single ZIP archive, or download individual stems. |
| **Persistent Source Selection** | Source audio URL and filename persist across sessions via localStorage. |
---
## Stem Builder
Generatively create new instrument stems for source tracks using the DiT engine:
| Feature | Description |
|---------|-------------|
| **Generative Stem Creation** | Select a source audio file and generate new AI-created instrument layers (vocals, drums, bass, guitar, piano) that complement the original track. |
| **Instrument Layer Selection** | Choose which stems to generate — add missing instruments or create alternative takes for existing ones. |
| **Per-Stem Preview** | Real-time audio preview of generated stems alongside the source track with per-stem volume controls. |
| **Source Audio Browser** | Browse source audio from the song library with search and mastered/unmastered toggle. |
| **Iterative Layering** | Build up arrangements by generating stems one at a time — each new layer is created in the context of the existing mix. |
| **Full Pipeline Integration** | Generated stems pass through the complete engine pipeline including post-processing, mastering, and format export. |
---
## MIDI Studio
Audio-to-MIDI transcription on HOT-Step's **native `ace-midi` engine** — a C++/GGML port of [MuScriptor](https://github.com/muscriptor/muscriptor) (Kyutai & Mirelo — code MIT, model weights CC BY-NC 4.0, non-commercial), validated byte-for-byte against the reference implementation. GPU-accelerated (a 3.5-min track transcribes in ~50 s on an RTX 5090), zero Python.
| Feature | Description |
|---------|-------------|
| **Multi-Instrument Transcription** | Convert any library track — or a WAV/MP3 uploaded from your PC — into a multi-track `.mid` file: drums, bass, guitar, keys, and more (34 instrument groups + drums). |
| **Model Choice + In-App Weight Download** | `small` (103M), `medium` (307M), or `large` (1.4B). The weights are **gated** on Hugging Face: request access via the in-app links (free), save your read token, and download each model with live progress — all inside MIDI Studio. |
| **Live Event Stream** | The engine streams note events over SSE as it transcribes (chunk progress + notes-so-far in the UI; live playable piano roll planned). |
| **Piano-Roll Preview** | Built-in SVG piano roll with per-channel instrument coloring and GM family legend, rendered from a native MIDI parser. |
| **History** | Completed transcriptions persist to `data/midi/` and survive restarts. |
---
## StableStep
Post-processing refiner that re-renders the instrumental of a generated track through **Stable Audio 3** (SDEdit-style partial re-noising) running natively in the C++ engine — no Python. Replaces autoencoder fizz with real spectral detail while vocals are separated, cleaned, and remixed byte-untouched. *Powered by Stability AI* (models under the [Stability AI Community License](https://stability.ai/community-license-agreement)).
| Feature | Description |
|---------|-------------|
| **SDEdit Instrumental Refine** | The instrumental is encoded into SAME-L latent space, partially re-noised at the chosen strength, and denoised by the SA3 DiT (8-step distilled rectified flow) conditioned on a prompt derived from the track's own caption (vocal descriptors stripped, length appended). |
| **Vocal-Safe Pipeline** | BS-RoFormer splits vocals (lead + backing) from the mix; the instrumental is derived as the exact complement so no content is lost. Vocals get a PP-VAE polish and are remixed over the refined instrumental — lyrics and performance are never re-generated. |
| **Refine Strength** | 0.10–0.60 slider (default 0.30). Low = cleanup; high = re-interpretation of the instrumentation. |
| **Dual Engine Backends** | GGML (CUDA / Vulkan / CPU, 4 GGUF files ~5.8 GB — fastest option on NVIDIA in current testing) or ONNX Runtime with TensorRT (NVIDIA, ~12 GB). Auto mode picks whichever is installed. |
| **In-App Model Download** | Model Manager → StableStep tab, with license acceptance and optional Hugging Face token. Both backend sets from [scragnog/HOT-Step-CPP-StableStep](https://huggingface.co/scragnog/HOT-Step-CPP-StableStep). |
| **Level Matching** | The refined instrumental and cleaned vocals are RMS-matched to their pre-processing levels, preserving the original vocal/instrumental balance through the chain. |
## AI Assistant
In-app LLM-powered assistant with full context awareness:
| Feature | Description |
|---------|-------------|
| **Streaming Chat Sidebar** | Toggleable chat panel with SSE-streamed responses, markdown rendering, and thinking/response separation. |
| **Full Settings Awareness** | Every message includes a JSON snapshot of all engine parameters, content fields (lyrics, caption, BPM, duration, key, time signature, language), and active mode. |
| **Mode-Aware Guidance** | Automatically detects which studio the user is in (Auto-Gen, Custom-Gen, Lyric Studio, Cover Studio, Stem Studio, Stem Builder) and tailors advice to that workflow. |
| **Actionable Suggestions** | LLM responses can include structured action blocks that the user can preview as diffs and apply individually or in bulk — settings update reactively. |
| **Content Editing** | Can write, rewrite, or update lyrics, style descriptions, and other content fields directly via action blocks with one-click apply. |
| **Per-Action Apply** | Each suggested change has its own Apply button — cherry-pick individual settings without accepting the full batch. Applied items show a checkmark and dim out. |
| **Thinking Separation** | LLM chain-of-thought is separated from the response and displayed in a collapsible "💭 Thought process" block — visible but visually distinct. |
| **Multi-Provider Support** | Uses the same LLM provider registry as Lyric Studio — supports Gemini, LM Studio, OpenAI-compatible endpoints, etc. Provider and model selection persisted independently. |
| **Knowledge Base** | Static knowledge base covering all engine parameters, solvers, schedulers, guidance modes, adapters, post-processing, troubleshooting, and lyric formatting rules. |
| **Markdown Rendering** | Lightweight built-in renderer for headers, bold, italic, inline code, fenced code blocks, lists, and horizontal rules — no external dependencies. |
---
## Timbre & Audio Conditioning
| Feature | Description |
|---------|-------------|
| **Timbre Reference** | Use a reference track as a DiT timbre conditioner to influence the sonic character of generations. |
| **FLAC Decoding** | Native dr_flac support for FLAC reference files alongside WAV and MP3. |
| **LM Code Cache** | Cache LM-generated audio codes for deterministic re-generation with consistent structure. |
| **LM Codes Strength** | Slider controlling how strongly cached LM codes influence the generation — from subtle guidance to exact reproduction. |
| **Co-Resident Models** | Run DiT and VAE from different model files simultaneously (e.g. turbo DiT with full VAE). |
---
## Settings & Configuration
| Feature | Description |
|---------|-------------|
| **Settings Page** | Central configuration hub for models, adapters, mastering references, and download preferences. |
| **Smart Defaults** | Works out of the box without a `.env` file — auto-discovers engine binary and model paths. |
| **Selectable VAE Decoder** | Choose between standard and alternative VAE decoders at runtime. |
| **LM / Thinking Toggle** | Skip or enable the LM inference phase entirely — useful for speed when you don't need metadata generation. |
| **Nuke Generations** | One-click wipe of all generated content and database entries. |
| **Configurable Download Defaults** | Set preferred export format, filename prefix, and download behaviour. |
| **Environment Editor** | Read and edit the server's `.env` file directly from the Settings UI with categorised sections, masked API keys, and save confirmation. |
| **Runtime Config Reload** | Hot-reload LLM provider settings and API keys without restarting the server. Engine-level changes show a restart notification. |
| **VAE Chunk/Overlap Settings** | Exposed VAE chunk size and overlap parameters for tuning memory usage on Vulkan/low-VRAM GPUs. |
| **OpenAI-Compatible Provider** | Generic OpenAI-compatible LLM provider supporting oMLX, vLLM, LocalAI, and similar endpoints. Configurable base URL and API key. |
---
## Model Manager
| Feature | Description |
|---------|-------------|
| **In-App Model Downloads** | Browse and download 100+ GGUF and safetensors models directly from the app — no manual file management needed. |
| **Curated Starter Packs** | 4 pre-configured bundles (Quick Start, Minimal, XL Quality, Blackwell Optimized) with one-click download of the full pipeline. |
| **Tabbed Model Catalogue** | Browse all available models organised by role (DiT, LM, Text Encoder, VAE, PP-VAE) with descriptions and quantisation badges. |
| **Concurrent Resumable Downloads** | Multiple simultaneous downloads with HTTP Range-based resumption — interruptions resume from where they left off. |
| **Real-Time Progress** | SSE-streamed download progress with speed, ETA, and per-file status tracking. |
| **Installed Status Tracking** | The catalogue shows which models you already have installed, with per-pack completion indicators. |
| **Model Deletion** | Remove installed models directly from the UI with confirmation prompts. |
| **5 HuggingFace Repos** | Models sourced from Serveurperso/ACE-Step-1.5-GGUF, scragnog/ace-step-1.5-gguf-merge-models, scragnog/Ace-Step-1.5-MXFP4-Quants, scragnog/Ace-Step-1.5-ScragVAE, and scragnog/HOT-Step-CPP-PP-VAE. |
---
## Build & Developer Tools
| Feature | Description |
|---------|-------------|
| **dev-rebuild.bat** | Graceful HTTP shutdown of the running app before engine rebuild — prevents the supervisor's auto-restart from causing a respawn loop. |
| **MSVC Build Compatibility** | Automatic Visual Studio discovery via vswhere, Ninja binary fallback, and Node.js version guard. |
| **File-Based Logging** | Structured logging system mirroring HOT-Step 9000 patterns for consistent debugging. |
| **Quantize Tool** | Experimental GGUF quantisation with IQ, NVFP4, MXFP4, and ternary format support. |
| **Quant Benchmark** | Automated inference benchmarking with peak VRAM tracking and results logging. |
| **MXFP4 Tensor Core Tests** | Blackwell GPU stress tests demonstrating 22–33% speedup with MXFP4 quantisation. |
| **Graceful Shutdown** | Proper Windows process cleanup with a "you can close this page" confirmation screen. |
| **Smart Update Scripts** | `update-and-build.bat` / `.sh` scripts for source builders — pulls latest changes, rebuilds engine, and reinstalls dependencies in one step. |
LAUNCH.bat
+45
View File
@@ -0,0 +1,45 @@
@echo off
echo =============================================
echo HOT-Step 9000 CPP - Production
echo =============================================
echo.
REM Check dependencies are installed
if not exist "%~dp0server\node_modules" (
echo ERROR: Server dependencies not installed.
echo Run install.bat first, or: cd server ^& npm install
echo.
pause
exit /b 1
)
REM Build UI if dist doesn't exist
if not exist "%~dp0ui\dist" (
if not exist "%~dp0ui\node_modules" (
echo ERROR: UI dependencies not installed.
echo Run install.bat first, or: cd ui ^& npm install
echo.
pause
exit /b 1
)
echo Building UI...
cd /d "%~dp0ui"
call npm run build
)
REM Start server (which spawns ace-server) with restart loop
cd /d "%~dp0server"
echo Starting server...
REM Open browser if no existing tab is found
start /MIN "" powershell -ExecutionPolicy Bypass -File "%~dp0open-browser-if-needed.ps1" "http://localhost:3001/" 4
:loop
call npx tsx src/index.ts
if exist "%~dp0.restart-requested" (
del "%~dp0.restart-requested"
echo.
echo [HOT-Step] Restarting server...
timeout /t 2 /nobreak > nul
goto loop
)
Makefile
+119
View File
@@ -0,0 +1,119 @@
# Convenience wrapper for Linux / WSL development.
# Windows-native development uses dev.bat / LAUNCH.bat / dev-rebuild.bat instead.
# Configuration
REPO_DIR = .
BACKEND ?=
# Detect if running inside WSL
IS_WSL := $(shell grep -qi microsoft /proc/version 2>/dev/null && echo 1 || echo 0)
# Auto-detect backend if not explicitly provided: CUDA if a driver is visible,
# else Vulkan if the SDK's shader compiler is installed, else CPU.
ifeq ($(BACKEND),)
ifeq ($(IS_WSL),1)
# In WSL, check if NVIDIA CUDA driver bridge is present
ifneq ($(wildcard /usr/lib/wsl/lib/libcuda.so*),)
BACKEND = cuda
endif
else
# Native Linux detection
ifneq ($(wildcard /usr/local/cuda*),)
BACKEND = cuda
endif
endif
endif
ifeq ($(BACKEND),)
ifneq ($(shell command -v glslc 2>/dev/null),)
BACKEND = vulkan
else
BACKEND = cpu
endif
endif
# Determine CMake flags based on backend selection
ifeq ($(BACKEND),cuda)
CMAKE_FLAGS = -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
else ifeq ($(BACKEND),vulkan)
CMAKE_FLAGS = -DGGML_VULKAN=ON -DCMAKE_BUILD_TYPE=Release
else
CMAKE_FLAGS = -DCMAKE_BUILD_TYPE=Release
endif
NODE_MAJOR := $(shell node -v 2>/dev/null | sed 's/^v\([0-9]*\).*/\1/')
.PHONY: check-config check-node setup submodules build install run clean help
check-config: ## Check current configuration before building
@echo "========================================"
@echo " BUILD CONFIGURATION "
@echo "========================================"
@echo " Environment : $(if $(filter 1,$(IS_WSL)),WSL (Windows Subsystem for Linux),Native Linux)"
@echo " Backend : $(BACKEND)"
@echo " CMake Flags : $(CMAKE_FLAGS)"
@echo " Node.js : $(if $(NODE_MAJOR),v$(NODE_MAJOR) (need 18-22),not found)"
@echo " Repository : $(REPO_DIR)"
@echo "========================================"
check-node: ## Verify Node.js 18-22 LTS is installed
@if [ -z "$(NODE_MAJOR)" ]; then \
echo "ERROR: Node.js not found. Install Node 18-22 LTS, e.g. with nvm:"; \
echo " https://github.com/nvm-sh/nvm then: nvm install 22 && nvm use 22"; \
exit 1; \
fi
@if [ "$(NODE_MAJOR)" -lt 18 ] || [ "$(NODE_MAJOR)" -gt 22 ]; then \
echo "ERROR: Node v$(NODE_MAJOR) detected, but HOT-Step requires Node 18-22 LTS"; \
echo " (Node 24+ breaks native dependencies - see README)."; \
echo " With nvm: nvm install 22 && nvm use 22"; \
exit 1; \
fi
setup: ## Install system dependencies (except Node - use nvm for that)
@echo "==> Installing system dependencies..."
sudo apt update && sudo apt install -y build-essential cmake git
@echo ""
@echo "==> NOTE: Node.js 18-22 LTS is also required (apt's version is often wrong)."
@echo " Recommended: install via nvm - https://github.com/nvm-sh/nvm"
@echo " Then run 'make check-node' to verify."
submodules: ## Initialise git submodules (required before first build)
@echo "==> Initialising git submodules..."
git submodule update --init --recursive
build: submodules ## Build the C++ engine
ifeq ($(BACKEND),vulkan)
@command -v glslc >/dev/null 2>&1 || { \
echo "ERROR: Vulkan backend selected but glslc not found."; \
echo " Install the Vulkan SDK: https://vulkan.lunarg.com/sdk/home"; \
exit 1; }
endif
@echo "==> Building C++ engine ($(BACKEND) backend)..."
cd $(REPO_DIR)/engine && mkdir -p build && cd build && \
cmake .. $(CMAKE_FLAGS) && \
cmake --build . -j $$(nproc)
install: check-node ## Install Node.js dependencies for server and UI
@echo "==> Installing Node.js dependencies for server..."
cd $(REPO_DIR)/server && npm install
@echo "==> Installing Node.js dependencies for UI..."
cd $(REPO_DIR)/ui && npm install
run: ## Launch the application
@echo "==> Launching application..."
cd $(REPO_DIR) && ./launch.sh
clean: ## Clean build artifacts (requires CONFIRM=1 - CUDA rebuild takes 20+ min)
ifneq ($(CONFIRM),1)
@echo "This deletes engine/build and all node_modules."
@echo "A CUDA engine rebuild from scratch takes 20+ minutes."
@echo "Run 'make clean CONFIRM=1' if you really want this."
@exit 1
else
@echo "==> Cleaning build files..."
rm -rf $(REPO_DIR)/engine/build
rm -rf $(REPO_DIR)/server/node_modules
rm -rf $(REPO_DIR)/ui/node_modules
endif
help: ## Show this help message
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
README.md
+568
View File
@@ -0,0 +1,568 @@
# HOT-Step CPP
A feature-rich UI for [acestep.cpp](https://github.com/ServeurpersoCom/acestep.cpp) — local AI music generation powered by GGML, with native safetensors support.
Describe a song with a text caption and lyrics, and get stereo 48kHz audio generated entirely on your local hardware. No cloud, no API keys, no subscriptions.
[![Discord](https://img.shields.io/badge/Discord-Join%20the%20community-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/ezVtmg9GKX)
[![Buy Me a Coffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-ffdd00?style=for-the-badge&logo=buy-me-a-coffee&logoColor=black)](https://buymeacoffee.com/scragnog)
[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-scragnog-FFD21E?style=for-the-badge)](https://huggingface.co/scragnog)
💬 **Questions, feedback, or want to share what you've made?** [Join the Discord](https://discord.gg/ezVtmg9GKX) — it's where I'm most active for HOT-Step discussion and support.
> ### 🎓 New: Training Studio *(highly experimental — for the adventurous!)*
> Train your own **style adapters entirely inside HOT-Step** — no Python, no external tools. Point it at a folder of songs and it walks you through the whole pipeline: **dataset creation** (local BPM/key analysis, lyrics from Genius, AI captions that actually listen to the audio), **tensor preprocessing**, and native **training** of both planner (LM LoRA, 0.6B/1.7B/4B) and **DiT LoRA** adapters — all in C++/GGML on your own GPU. There's even a pure-LM **audition mode** that lets you hear what the planner learned, A/B against the base model, with zero DiT influence.
>
> This is **very much experimental right now** — it's brand new, GPU-hungry (16 GB+ recommended, 24 GB+ for full-depth DiT training), and rough edges are guaranteed. If you try it, we'd love to hear how it goes on the Discord. Find it in the sidebar as **Training**.
## Download
Pre-built portable releases — no installation required. Extract, run, done.
**[📥 Download the latest release →](https://github.com/scragnog/HOT-Step-CPP/releases/latest)**
| Platform | Variants |
|----------|----------|
| **Windows** (x64) | CUDA (NVIDIA), Vulkan (AMD/Intel/NVIDIA), CPU |
| **Linux** (x64) | CUDA (NVIDIA), Vulkan (AMD/Intel/NVIDIA), CPU |
| **macOS** (Apple Silicon) | Metal (M1/M2/M3/M4) |
**Which variant?**
- **CUDA** — Best performance. Use this if you have an NVIDIA GPU (RTX 2060 or newer recommended).
- **Vulkan** — Cross-vendor GPU support. Use this if you have an AMD or Intel GPU, or an older NVIDIA card.
- **CPU** — No GPU needed. Works on any machine but generation will be significantly slower.
### Quick Start
**Windows:**
1. Download and extract the zip for your hardware
2. Run **`HOT-Step.bat`**
3. Your browser opens to `http://localhost:3001`
4. On first launch, go to **Models → Get More Models** to download the AI models (~7 GB)
**Linux:**
1. Download and extract the `.tar.gz` for your hardware
2. Run **`./HOT-Step.sh`**
3. Open `http://localhost:3001` in your browser
4. On first launch, go to **Models → Get More Models** to download the AI models (~7 GB)
**macOS:**
1. Download and extract the `.tar.gz`
2. Open Terminal in the extracted folder and run **`./HOT-Step.sh`**
3. Your browser opens to `http://localhost:3001`
4. On first launch, the **Model Manager** opens automatically — download the AI models (~7 GB)
> **Windows requirements:** Windows 10/11 (64-bit), ~10 GB free disk space. CUDA variant needs NVIDIA drivers. Vulkan variant needs Vulkan 1.1+ capable drivers.
> **Linux requirements:** Ubuntu 22.04+ or equivalent (x86_64), ~10 GB free disk space. CUDA variant needs NVIDIA drivers 525+. Vulkan variant needs Vulkan 1.1+ capable drivers and `libvulkan1`.
> **macOS requirements:** macOS 13+ (Apple Silicon M1/M2/M3/M4), ~10 GB free disk space. No other software needed — Node.js is bundled. If macOS blocks the app (unsigned binary), run: `xattr -cr /path/to/HOT-Step-CPP/`
---
## Highlights
HOT-Step CPP extends the base acestep.cpp engine with 100+ features across inference, audio processing, and creative tooling. Here are the big ones:
🎛️ **17 Solvers, 9 Schedulers, 7 Guidance Modes, Postprocess Plugins** — Fully extensible Lua plugin architecture for ODE/SDE solvers, noise schedulers, guidance modes, and postprocess pipelines. Drop a `.lua` file into `engine/plugins/` and it appears in the UI at next launch — no C++ rebuild needed. Includes research-derived modes like CFG-MP (manifold projection), SMC-CFG (sliding mode control), and CFG-Zero⋆ (zero-init). Each plugin can expose its own user-facing parameters (sliders, toggles, dropdowns). **[Create your own →](docs/PLUGINS.md)**
🎸 **LoRA Adapters with Runtime Mode** — Per-group scale controls (self_attn, cross_attn, mlp, cond_embed), K-quant GPU support via custom CUDA kernels, and a runtime LoRA mode that applies deltas in the forward pass without permanently merging weights.
🎚️ **Matchering Mastering Engine** — Loudness, EQ, and dynamics matching to a reference track with instant mastered/unmastered A/B toggle. Operates at native 48kHz — no resample round-trip.
🤖 **Auto-Gen** — AI-driven song creation. Pick genres, optionally set a subject and language, and the LM handles everything — lyrics, style caption, metadata, and title. Three lyric modes: fully AI-generated, AI-written from your subject, or instrumental. Preview mode lets you review and edit AI-generated lyrics before committing to generation. Serial queue ensures one job at a time with live progress tracking.
🎹 **Custom-Gen** — Full manual control over every generation parameter. Write your own lyrics (or go instrumental), set a style caption, title, artist, BPM, duration, key signature, and time signature. Direct access to all engine settings with queue-based generation. The power-user mode for when you know exactly what you want.
🔌 **VST3 Host** — Scan, load, and run your existing VST3 plugins directly in the generation pipeline. Offline processing and real-time WASAPI monitor mode with transport controls. **Note:** VST plugins run in a single-input pipeline with no external sidechain bus. Plugins that require an external key signal (sidechain compressors, keyed gates, duckers) will not trigger — use plugins in their internal detection mode instead.
✍️ **Lyric Studio** — A complete AI-powered lyrics and music workspace. 7 LLM providers (Gemini, LM Studio, OpenAI-compatible), artist profiles with adapter presets, statistical lyric analysis, bulk generation with "Fill to N" mode, and full parameter parity with the Create page.
🎤 **Cover Studio** — Upload a reference track, get Essentia-based analysis (BPM, key, energy, timbre), and generate style-matched covers. Artist-optional workflow with editable style descriptions, pitch shift with key transposition preview, tempo scaling, stem separation + recombination, and per-album adapter presets.
🔪 **Stem Studio** — 4-stage neural stem separation powered by SuperSep. BS-RoFormer for primary 6-stem splits, Mel-Band RoFormer for lead/backing vocal isolation, MDX23C for drum sub-separation, and HTDemucs for instrument refinement. Interactive mixer with multi-solo, per-stem volume controls, and ZIP export. Sequential VRAM management keeps peak usage under 3 GB.
🧱 **Stem Builder** — Generatively create new instrument stems for source tracks using the DiT engine. Select a source audio file, choose which instrument layers to generate (vocals, drums, bass, guitar, piano), and the engine creates fresh stems that complement the original. Build up arrangements by iteratively adding AI-generated layers.
🎼 **MIDI Studio** — Audio-to-MIDI transcription on a native C++/GGML port of [MuScriptor](https://github.com/muscriptor/muscriptor) (Kyutai & Mirelo), validated byte-for-byte against the reference and GPU-accelerated — a 3.5-minute track transcribes in under a minute. Convert any library track or an uploaded WAV/MP3 into multi-track MIDI (34 instrument groups + drums), watch the piano roll fill in live while transcription runs, and hit play immediately with a crossfade slider between the original audio and the MIDI rendition, plus per-instrument mute/solo. Small/medium/large models with in-app weight download (gated on Hugging Face; weights CC BY-NC 4.0 — non-commercial).
✨ **StableStep** — Post-processing refiner that re-renders the instrumental of a finished track through **Stable Audio 3** (SDEdit-style partial re-noising) to replace VAE fizz with genuine spectral detail. Vocals are split out via BS-RoFormer (lead + backing), cleaned with PP-VAE, and remixed untouched — lyrics stay intact. Adjustable refine strength, per-track prompt derived from the generation caption, and two engine backends: GGML (CUDA/Vulkan/CPU, ~5.8 GB, fastest in testing) or ONNX/TensorRT (~12 GB). Models download in-app under the Stability AI Community License. *Powered by Stability AI.*
🔊 **Audio Post-Processing** — Spectral denoiser (Wiener-filter), Spectral Lifter (native C++), PP-VAE neural audio polish, Vocal Naturalizer (5-stage DSP humanization, experimental — may affect downstream processing), duration buffer with auto-trim for clean endings, and configurable fade-out.
📊 **Audio Quality Evaluator** — Automatic post-generation quality scoring using spectral analysis. Three weighted metrics — metallic sound detection (spectral rolloff), word cut detection (spectral flux discontinuities), and noise/hiss analysis (zero-crossing rate) — produce a 0–100% score per track. Choose to evaluate unmastered, mastered, or both for direct comparison. Scores display as colour-coded badges in the Library. Ported from [JK-AceStep-Nodes](https://github.com/jeankassio/JK-AceStep-Nodes) (MIT License).
🤖 **AI Assistant** — In-app LLM-powered assistant with full awareness of your current settings, lyrics, mode, and engine state. Ask it to review your configuration, write or rewrite lyrics, suggest optimizations, or directly apply setting changes — all via a streaming chat sidebar. Supports any configured LLM provider (local or cloud) with per-action apply controls and thinking/response separation.
🧪 **Latent Space Controls** — Latent shift, latent rescale, custom timestep scheduling, DCW (Differential Correction in Wavelet domain) sampling, and auto-shift for adaptive noise scaling.
📦 **Lossless Pipeline** — WAV32 throughout the processing chain, with export to WAV, MP3, or FLAC.
📥 **In-App Model Manager** — Browse 100+ GGUF models across 5 HuggingFace repos, download with curated starter packs, and manage your model library without leaving the app. Concurrent resumable downloads with real-time progress.
🧬 **PP-VAE & ScragVAE** — Two custom VAE models. PP-VAE runs a neural encode→decode polish pass on generated audio to smooth spectral artifacts. ScragVAE is a fine-tuned decoder with improved high-frequency energy and dynamic range — both selectable at runtime.
📦 **Safetensors Model Support** — Load HuggingFace-format safetensors models alongside GGUF. Drop a model folder into the models directory and it appears in the UI with a format badge. Supports DiT, LM, Text Encoder, and VAE. BF16 safetensors produce bit-perfect output vs BF16 GGUF. Adapters (LoRA) work with both base model formats.
🎨 **Repaint Studio** — Region-based audio regeneration. Select a section of a track via waveform click-drag, edit synchronized lyrics, and regenerate just that portion while preserving the rest. Fix problematic sections without re-generating the entire song.
🔄 **A/B Comparison** — Dual-track playback for comparing two generations side by side. Global A/B mini-bar above the player persists across views for quick cross-page comparison.
👉 **[See the full feature list →](FEATURES.md)**
## Gallery
### Library
Browse your generated songs as a cover art grid with AI-generated artwork, quality scores, and audio metadata. The right sidebar shows a live playlist and engine terminal output. The bottom bar features a waveform visualizer with section markers (verse, chorus, bridge) and real-time synced lyrics.
![Library — song grid with AI cover art, playlist sidebar, waveform visualizer, and synced lyrics playback](docs/images/hot-step-library.webp)
### Auto-Gen
AI-driven music creation — pick a genre, set a vocal mode, and the LLM handles everything else. The song details panel shows full generation metadata: models used, solver, scheduler, CFG scale, key signature, time signature, and duration.
![Auto-Gen — AI-driven song creation with genre picker, generation queue, and detailed song parameter panel](docs/images/hot-step-auto-gen.webp)
### Lyric Studio
A complete AI-powered lyrics workspace. Browse artists and albums on the left, view and edit AI-generated lyrics with structural section tags in the centre, and manage your generation queue on the right. Supports multiple LLM providers for lyric generation and refinement.
![Lyric Studio — artist browser, AI-generated lyrics editor with section tags, and generation queue](docs/images/hot-step-lyric-studio.webp)
### Cover Studio
Upload a reference track for automatic BPM and key detection via Essentia analysis. The engine extracts style descriptions, lyrics, and structural metadata. Fine-tune cover settings including structure fidelity, source preservation, pitch shift with key transposition, and tempo scaling.
![Cover Studio — reference track analysis with BPM/key detection, style matching, and cover generation controls](docs/images/hot-step-cover-studio.webp)
### Model Manager
Browse curated starter packs tailored to different hardware tiers — from minimal setups to Blackwell-optimized configurations. Download individual GGUF models, stem separation networks, and CUDA/cuDNN runtime libraries directly from HuggingFace without leaving the app.
![Model Manager — starter packs, individual model downloads, and runtime dependency management](docs/images/hot-step-model-manager.webp)
---
## Architecture
HOT-Step CPP is three components working together:
| Component | Tech | Purpose |
|-----------|------|---------|
| **Engine** | C++ / CUDA / GGML | The acestep.cpp inference engine — runs the AI models |
| **Server** | Node.js / TypeScript | Orchestrates the engine, manages songs, serves the UI |
| **UI** | React / Vite / TypeScript | The browser-based frontend |
## Platform Support
| Platform | Status |
|----------|--------|
| Windows + NVIDIA (CUDA) | ✅ Pre-built release available |
| Windows + AMD/Intel (Vulkan) | ✅ Pre-built release available |
| Windows CPU-only | ✅ Pre-built release available |
| macOS Apple Silicon (Metal) | ✅ Pre-built release available |
| Linux + NVIDIA (CUDA) | ✅ Pre-built release available |
| Linux + AMD/Intel (Vulkan) | ✅ Pre-built release available |
| Linux CPU-only | ✅ Pre-built release available |
---
## Building from Source
If you prefer to build from source (or want to contribute), follow the instructions below. **Most users should use the [pre-built releases](#download) instead.**
### Windows
#### Prerequisites
| Requirement | Version | Notes |
|-------------|---------|-------|
| [Visual Studio 2022 Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) | 2022 | Select "Desktop development with C++" workload |
| [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) | 12.x+ | For NVIDIA GPU acceleration. **Select "Visual Studio Integration" during install.** |
| [CMake](https://cmake.org/download/) | 3.14+ | Usually included with VS Build Tools |
| [Node.js](https://nodejs.org/) | 18–22 LTS | **Node 24+ is not supported** — use nvm to install 22 LTS if needed |
| [Git](https://git-scm.com/) | Any | For cloning |
#### 1. Clone the repo
```cmd
git clone --recursive https://github.com/scragnog/HOT-Step-CPP.git
cd HOT-Step-CPP
```
> **Already cloned without `--recursive`?** Run `git submodule update --init --recursive` to fetch the ggml and vst3sdk submodules.
#### 2. Build the engine
The easiest way:
```cmd
engine\build.cmd
```
This automatically finds your Visual Studio installation (any edition) and builds with CUDA.
Alternatively, open a **Developer Command Prompt for VS 2022** and build manually:
```cmd
cd engine
mkdir build
cd build
cmake .. -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=native
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
cd ..\..
```
> **Note:** If you use **Ninja** as your CMake generator (`-G Ninja`), binaries will be placed directly in `engine/build/` rather than `engine/build/Release/`. The server auto-detects both locations.
#### 3. Download models
Download four GGUF model files from [Hugging Face](https://huggingface.co/Serveurperso/ACE-Step-1.5-GGUF/tree/main) and place them in a `models/` directory at the repo root:
```
HOT-Step-CPP/
├── models/ ← create this, put GGUFs here
│ ├── acestep-5Hz-lm-4B-Q8_0.gguf
│ ├── Qwen3-Embedding-0.6B-Q8_0.gguf
│ ├── acestep-v15-turbo-Q8_0.gguf
│ └── vae-BF16.gguf
├── engine/
├── server/
└── ui/
```
| Type | Recommended File | Size |
|------|-----------------|------|
| LM | `acestep-5Hz-lm-4B-Q8_0.gguf` | 4.2 GB |
| Text Encoder | `Qwen3-Embedding-0.6B-Q8_0.gguf` | 748 MB |
| DiT | `acestep-v15-turbo-Q8_0.gguf` | 2.4 GB |
| VAE | `vae-BF16.gguf` | 322 MB |
Smaller LM variants available: 0.6B (fast) and 1.7B (balanced).
#### Optional (recommended)
| Type | File | Size | Source |
|------|------|------|--------|
| ScragVAE | `scragvae-BF16.gguf` | 322 MB | [scragnog/Ace-Step-1.5-ScragVAE](https://huggingface.co/scragnog/Ace-Step-1.5-ScragVAE) |
| PP-VAE | `pp-vae-F32.gguf` | 644 MB | [scragnog/HOT-Step-CPP-PP-VAE](https://huggingface.co/scragnog/HOT-Step-CPP-PP-VAE) |
| StableStep (GGML) | `sa3-*.gguf` (4 files) | 5.8 GB | [scragnog/HOT-Step-CPP-StableStep](https://huggingface.co/scragnog/HOT-Step-CPP-StableStep) |
| StableStep (ONNX) | `onnx/sa3/` (9 files) | 12 GB | [scragnog/HOT-Step-CPP-StableStep](https://huggingface.co/scragnog/HOT-Step-CPP-StableStep) |
**ScragVAE** is a fine-tuned VAE decoder with improved high-frequency energy and dynamic range — drop-in replacement for the standard VAE. **PP-VAE** enables neural audio polish via an encode→decode round-trip in the post-processing chain.
> **💡 Tip:** You can also download models directly from the app! Click **Models → Get More Models** to browse 100+ models across 5 HuggingFace repos, with curated starter packs for quick setup.
#### 4. Install UI & server dependencies
```cmd
install.bat
```
Or manually (PowerShell):
```powershell
cd server; npm install; cd ..
cd ui; npm install; cd ..
```
#### 5. Run
```cmd
LAUNCH.bat
```
Open `http://localhost:3001` in your browser. That's it!
> **No `.env` file needed** for the standard setup. The server automatically finds the engine binary (checks `engine/build/Release/`, `engine/build/`, and `engine/build/Debug/`) and models at `models/`. See `.env.example` if you need to override paths for a custom setup.
**Development mode** (with hot-reload):
```cmd
dev.bat
```
Then open `http://localhost:3000`.
---
### macOS (Apple Silicon)
#### Prerequisites
| Requirement | Version | Notes |
|-------------|---------|-------|
| Xcode Command Line Tools | 16+ | `xcode-select --install` |
| CMake | 3.14+ | `brew install cmake` |
| Node.js | 18–22 LTS | `brew install node@22` — **Node 24+ is not supported** |
| Git | Any | Included with Xcode CLI tools |
> **Note:** Xcode provides the Metal SDK and C++ compiler. No separate GPU toolkit is needed — Metal support is built into macOS.
#### 1. Clone the repo
```bash
git clone --recursive https://github.com/scragnog/HOT-Step-CPP.git
cd HOT-Step-CPP
```
> **Already cloned without `--recursive`?** Run `git submodule update --init --recursive` to fetch the ggml and vst3sdk submodules.
#### 2. Build the engine
```bash
cd engine
mkdir build && cd build
cmake .. -DGGML_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_BUILD_TYPE=Release
cmake --build . --config Release -j $(sysctl -n hw.ncpu)
cd ../..
```
> This builds with Metal GPU acceleration. The Metal shader library is embedded into the binary so no external `.metallib` file is needed at runtime.
#### 3. Download models
Same as Windows — place GGUF files in `models/`. See [model list above](#3-download-models). Or skip this and download from the in-app Model Manager on first launch.
#### 4. Install UI & server dependencies
```bash
cd server && npm install && cd ..
cd ui && npm install && cd ..
```
#### 5. Run
```bash
./launch.sh
```
Open `http://localhost:3001` in your browser.
> **No `.env` file needed** for the standard setup. The server automatically finds the engine binary and models. See `.env.example` if you need to override paths.
**Development mode** (with hot-reload):
```bash
./launch.sh # In one terminal
cd ui && npx vite # In another terminal
```
Then open `http://localhost:3000`.
---
### Linux (x86_64)
#### Prerequisites
| Requirement | Version | Notes |
|-------------|---------|-------|
| GCC / Clang | GCC 11+ | `sudo apt install build-essential` |
| CMake | 3.14+ | `sudo apt install cmake` |
| Node.js | 18–22 LTS | **Node 24+ is not supported** |
| Git | Any | `sudo apt install git` |
| CUDA Toolkit (optional) | 12.x+ | For NVIDIA GPU acceleration |
| Vulkan SDK (optional) | Latest | For AMD / Intel GPU acceleration |
#### 1. Clone the repo
```bash
git clone --recursive https://github.com/scragnog/HOT-Step-CPP.git
cd HOT-Step-CPP
```
> **Already cloned without `--recursive`?** Run `git submodule update --init --recursive` to fetch the ggml and vst3sdk submodules.
#### 2. Build the engine
**CUDA (NVIDIA GPU):**
```bash
cd engine
mkdir -p build && cd build
cmake .. -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
cmake --build . -j $(nproc)
cd ../..
```
**Vulkan (AMD / Intel / NVIDIA):**
```bash
# Install Vulkan SDK first: https://vulkan.lunarg.com/sdk/home
cd engine
mkdir -p build && cd build
cmake .. -DGGML_VULKAN=ON -DCMAKE_BUILD_TYPE=Release
cmake --build . -j $(nproc)
cd ../..
```
**CPU-only:**
```bash
cd engine
mkdir -p build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
cmake --build . -j $(nproc)
cd ../..
```
#### 3. Download models
Same as Windows — place GGUF files in `models/`. See [model list above](#3-download-models). Or skip this and download from the in-app Model Manager on first launch.
#### 4. Install UI & server dependencies
```bash
cd server && npm install && cd ..
cd ui && npm install && cd ..
```
#### 5. Run
```bash
./launch.sh
```
Open `http://localhost:3001` in your browser.
> **No `.env` file needed** for the standard setup. The server automatically finds the engine binary and models. See `.env.example` if you need to override paths.
---
### Building a Portable Release
You can package a self-contained, zero-prerequisite release for distribution. The resulting archive bundles everything — engine binaries, Node.js runtime, server, UI, and plugins — so end users just extract and run.
#### macOS
```bash
./package-release.sh
```
This will:
1. Build the C++ engine with Metal GPU acceleration
2. Install production server dependencies
3. Build the optimised production UI
4. Download and bundle a Node.js 22 runtime (~40 MB)
5. Package everything into a `.tar.gz`
Options:
```bash
./package-release.sh --skip-build # Skip engine build (use existing binaries)
./package-release.sh --version=1.2.0 # Set version number
```
The output archive is fully portable — no brew, no npm, no Xcode needed on the target machine. The bundled `launch.sh` auto-detects and uses the included Node.js runtime.
---
## Troubleshooting
<details>
<summary><b>MSVC error C2589: illegal token on right side of '::'</b></summary>
This happens when `Windows.h` defines `min`/`max` as macros, which collide with `std::min`/`std::max`. The CMakeLists.txt should already define `NOMINMAX` — if you're seeing this, pull the latest version.
If building manually, add `-DCMAKE_CXX_FLAGS="/DNOMINMAX /DWIN32_LEAN_AND_MEAN"` to your cmake command.
</details>
<details>
<summary><b>npm install fails on Node.js 24+</b></summary>
Node.js 24 is too new for some dependencies. Use Node.js 22 LTS:
```cmd
nvm install 22
nvm use 22
```
</details>
<details>
<summary><b>build.cmd can't find vcvars64.bat</b></summary>
The build script uses `vswhere.exe` to find Visual Studio automatically. If it fails:
1. Make sure you have **Visual Studio 2022** (any edition) or **Build Tools** installed
2. Ensure the **"Desktop development with C++"** workload is selected
3. As a fallback, open a **Developer Command Prompt for VS 2022** and build manually (see Build the Engine above)
</details>
<details>
<summary><b>"ace-server.exe not found" after building with Ninja</b></summary>
Ninja is a single-config generator — binaries go directly in `engine/build/` instead of `engine/build/Release/`. The server auto-detects both locations. If you still see this error, pull the latest version or set `ACESTEPCPP_EXE` in your `.env` file to point to the binary.
</details>
<details>
<summary><b>CUDA error: "The CUDA Toolkit directory does not exist"</b></summary>
MSBuild can't find the CUDA Toolkit. Check:
1. The `CUDA_PATH` environment variable is set (e.g. `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.x`)
2. You selected **"Visual Studio Integration"** during the CUDA Toolkit install — without this, MSBuild has no `$(CudaToolkitDir)` macro
3. Restart your terminal after installing or modifying CUDA paths
</details>
<details>
<summary><b>"The input line is too long" when running build.cmd</b></summary>
Running `build.cmd` multiple times in the same terminal causes `vcvars64.bat` to append duplicate entries to `%PATH%` until it exceeds the Windows 8,192-character limit.
**Fix:** Close the terminal and open a fresh one. The build scripts now guard against this, but older versions don't — pull latest.
</details>
<details>
<summary><b>Build errors persist after fixing environment</b></summary>
If you changed CUDA versions, VS editions, or environment variables, the CMake cache may contain stale configuration:
```cmd
rd /s /q engine\build
engine\build.cmd
```
The `CMakeCache.txt` is only generated once — `build.cmd` skips reconfiguration if it already exists.
</details>
<details>
<summary><b>macOS: "operation not permitted" or app blocked by Gatekeeper</b></summary>
Since the release binaries are unsigned, macOS may quarantine them. Remove the quarantine flag:
```bash
xattr -cr /path/to/HOT-Step-CPP-v1.0.0-macOS-arm64/
```
This only needs to be done once after extraction.
</details>
<details>
<summary><b>macOS: Metal compilation errors during engine build</b></summary>
Ensure you have Xcode (not just Command Line Tools) and run the first-launch setup:
```bash
sudo xcodebuild -runFirstLaunch
```
If you see errors about Metal Toolchain, these can usually be ignored — the embedded Metal library (`-DGGML_METAL_EMBED_LIBRARY=ON`) does not require a separate Metal Toolchain download.
</details>
## Credits
- **[ACE-Step 1.5](https://github.com/ace-step/ACE-Step-1.5)** — The AI music generation model by ACE Studio and StepFun
- **[acestep.cpp](https://github.com/ServeurpersoCom/acestep.cpp)** — The C++ GGML inference engine by ServeurpersoCom
- **[HOT-Step 9000](https://github.com/scragnog/HOT-Step-9000)** — The Python-based sister project with full feature support
- **Alexander Allan ([MDMAchine](https://github.com/MDMAchine))** — STORM solver plugin (adaptive STORK/DPM++3M hybrid) and MD Audio Tiled Core postprocess plugin (advanced tiled VAE decode with OLA crossfading, dual-pass merge, and DSP chain)
- **[ComfyUI_MusicTools](https://github.com/jeankassio/ComfyUI_MusicTools)** — Vocal Naturalizer DSP algorithm by Jean Kassio (MIT License)
- **[JK-AceStep-Nodes](https://github.com/jeankassio/JK-AceStep-Nodes)** — Audio Quality Evaluator metrics by Jean Kassio (MIT License)
- **[Stability AI](https://stability.ai)** — Stable Audio 3 (diffusion transformer + SAME-L autoencoder + T5Gemma text encoder); powers the StableStep refiner via our native ONNX/GGML conversions, distributed under the [Stability AI Community License](https://stability.ai/community-license-agreement). *Powered by Stability AI.*
- **[MuScriptor](https://github.com/muscriptor/muscriptor)** — Multi-instrument music transcription model by Kyutai and Mirelo (Simon Rouard, Michael Krause, Axel Roebel, Carl-Johann Simon-Gabriel, Alexandre Défossez — [arXiv:2607.08168](https://arxiv.org/abs/2607.08168)); powers MIDI Studio via our native GGML port (code MIT, model weights CC BY-NC 4.0)
## License
The engine component (`engine/`) is licensed under MIT. See [engine/LICENSE](engine/LICENSE) for details.
---
> ### 💜 Special Thanks
>
> A heartfelt thank you to **Alexander Allan ([MDMAchine](https://github.com/MDMAchine))** for his ongoing and generous contributions to HOT-Step — from the STORM solver and MD Audio Tiled Core postprocess plugins to the real-time VST3 monitoring UX (chain presets, live monitor transport, pause/resume/restart) and a slew of JUCE VST3 hosting crash fixes in the engine. Your work has made this project meaningfully better. 🙏
---
## ⭐ Star History
If HOT-Step is useful to you, consider giving it a star — it really helps!
[![Star History Chart](https://api.star-history.com/svg?repos=scragnog/HOT-Step-CPP&type=Date)](https://star-history.com/#scragnog/HOT-Step-CPP&Date)
dev.sh
Executable
+86
View File
@@ -0,0 +1,86 @@
#!/bin/bash
# HOT-Step CPP — macOS Development Mode
#
# Starts Vite dev server (hot-reload UI) + Node.js server (tsx watch).
# Automatically finds a compatible Node.js.
#
# Usage:
# ./dev.sh
set -e
cd "$(dirname "$0")"
ROOT_DIR="$(pwd)"
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║ HOT-Step 9000 ⚡ DEV MODE ║"
echo "║ Vite HMR + tsx watch ║"
echo "╚══════════════════════════════════════════╝"
echo ""
# ── Find Node.js ────────────────────────────────────────────────────
find_node() {
if [ -x "${ROOT_DIR}/runtime/node" ]; then
echo "${ROOT_DIR}/runtime"; return 0
fi
for bp in "/opt/homebrew/opt/node@22/bin" "/usr/local/opt/node@22/bin"; do
if [ -x "${bp}/node" ]; then echo "$bp"; return 0; fi
done
if command -v node &>/dev/null; then
local ver; ver="$(node --version 2>/dev/null | tr -d 'v')"
local major="${ver%%.*}"
if [ "$major" -ge 18 ] && [ "$major" -lt 24 ] 2>/dev/null; then
echo "$(dirname "$(command -v node)")"; return 0
fi
fi
return 1
}
NODE_DIR=""
if NODE_DIR=$(find_node); then
export PATH="${NODE_DIR}:${PATH}"
echo " Node.js: $(node --version)"
else
echo "❌ No compatible Node.js found (need 18-22)."
echo " Fix: brew install node@22"
exit 1
fi
# ── Install deps if needed ───────────────────────────────────────────
if [ ! -d "server/node_modules" ]; then
echo "📦 Installing server dependencies..."
cd server && npm install --loglevel=warn && cd ..
fi
if [ ! -d "ui/node_modules" ]; then
echo "📦 Installing UI dependencies..."
cd ui && npm install --loglevel=warn && cd ..
fi
# ── Cleanup on exit ──────────────────────────────────────────────────
cleanup() {
echo ""
echo "[dev.sh] Shutting down..."
if [ -n "${VITE_PID}" ] && kill -0 "${VITE_PID}" 2>/dev/null; then
kill "${VITE_PID}" 2>/dev/null || true
echo "[dev.sh] Vite stopped"
fi
}
trap cleanup EXIT INT TERM
# ── Start Vite (background) ─────────────────────────────────────────
echo "🚀 Starting Vite dev server..."
cd ui
npx vite &
VITE_PID=$!
cd ..
sleep 2
# ── Start Node server (foreground) ──────────────────────────────────
echo "🚀 Starting Node.js server..."
echo " UI: http://localhost:3000 (Vite HMR)"
echo " API: http://localhost:3001"
echo ""
cd server
npx tsx watch src/index.ts
docker-compose.yml
+47
View File
@@ -0,0 +1,47 @@
# ============================================================================
# HOT-Step 9000 CPP — Docker Compose
#
# Usage:
# docker compose up # Start (build if needed)
# docker compose up --build # Force rebuild
# docker compose down # Stop
# docker compose logs -f # Follow logs
# ============================================================================
services:
hot-step:
build:
context: .
args:
# Dev: Blackwell only (~3 min build)
# Distribution: "75;80;86;89;90;120a" (~15-20 min)
CUDA_ARCHS: "120a"
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
ports:
- "3001:3001"
volumes:
# All I/O on host — accessible from Windows Explorer
- ./models:/app/models # GGUF model files
- "D:/Ace-Step-Latest/All LoKR Files/sidestep/xl-base-turbo-05:/app/adapters:ro"
- "D:/Ace-Step-Latest/Datasets-LoRA-LoKR:/app/datasets:ro"
- ./server/data:/app/server/data # Shared SQLite DB + audio (same as Windows app)
# Lua plugins (bind-mounted so edits are instant, no rebuild needed)
- ./engine/plugins:/app/engine/plugins:ro # Built-in (solvers, schedulers, etc.)
- ./plugins:/app/plugins:ro # Community/custom plugins
# Server source (live from host — `docker compose restart` to apply changes)
# node_modules stays baked in the image; only src/ is overlaid
- ./server/src:/app/server/src:ro
# TRT engine cache (GPU-specific, persists across container restarts)
- trt_engine_cache:/app/trt_cache
env_file:
- .env.docker
restart: unless-stopped
volumes:
trt_engine_cache:
docker/entrypoint.sh
+59
View File
@@ -0,0 +1,59 @@
#!/bin/bash
# ============================================================================
# HOT-Step 9000 CPP — Docker Entrypoint
# Verifies GPU access and starts the Node.js server (which spawns ace-server)
# ============================================================================
set -e
echo ""
echo "╔══════════════════════════════════════════╗"
echo "║ HOT-Step 9000 ⚡ Docker ║"
echo "║ High-Performance Music Generation ║"
echo "╚══════════════════════════════════════════╝"
echo ""
# ── Verify GPU access ───────────────────────────────────────────────
if command -v nvidia-smi &>/dev/null; then
GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1)
GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader 2>/dev/null | head -1)
GPU_DRIVER=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1)
echo " GPU: ${GPU_NAME} (${GPU_MEM})"
echo " Driver: ${GPU_DRIVER}"
else
echo " ⚠ nvidia-smi not found — GPU may not be available"
echo " Check that Docker has GPU access (--gpus=all or deploy.resources in compose)"
fi
# ── Verify engine binary exists ─────────────────────────────────────
if [ -f /app/engine/ace-server ]; then
echo " Engine: /app/engine/ace-server ✓"
else
echo " ⚠ ace-server binary not found at /app/engine/"
echo " The build may have failed — check Docker build logs"
fi
# ── Ensure bind-mount directories exist (in case host dirs are empty) ──
mkdir -p /app/models /app/adapters /app/server/data
# ── Check for models ────────────────────────────────────────────────
MODEL_COUNT=$(find /app/models -name '*.gguf' 2>/dev/null | wc -l)
if [ "$MODEL_COUNT" -gt 0 ]; then
echo " Models: ${MODEL_COUNT} GGUF file(s) found"
else
echo " ⚠ No .gguf models found in /app/models"
echo " Place model files in the ./models/ directory on the host"
fi
echo ""
echo " Server: http://localhost:${SERVER_PORT:-3001}"
echo " Engine: http://localhost:${ACESTEPCPP_PORT:-8085}"
echo ""
echo " Starting server..."
echo ""
# ── Start the Node.js server ────────────────────────────────────────
# The server spawns ace-server as a child process automatically.
# Use exec to replace the shell — proper signal handling for graceful shutdown.
cd /app/server
exec node --import tsx/esm src/index.ts
docs/PLUGINS.md
+453
View File
@@ -0,0 +1,453 @@
# Plugin Authoring Guide
How to create custom solvers, schedulers, and guidance modes for the HOT-Step CPP engine.
---
## Quick Start
1. Create a `.lua` file in the appropriate directory:
- `engine/plugins/solvers/` — ODE/SDE solvers
- `engine/plugins/schedulers/` — noise schedules
- `engine/plugins/guidance/` — CFG guidance modes
2. Declare a metadata table (`solver`, `scheduler`, or `guidance`)
3. Implement the required function (`step`, `schedule`, or `guide`)
4. Restart the app — your plugin appears in the UI automatically
No C++ rebuild required. The engine hot-loads all `.lua` files at startup.
---
## Plugin Types
### Solver
Solvers advance the latent state `xt` by one step along the ODE/SDE trajectory.
**Metadata table:**
```lua
solver = {
name = "my_solver", -- unique internal ID (lowercase, underscores)
display = "My Solver (2 NFE)", -- name shown in UI dropdown
description = "A custom solver", -- tooltip text
nfe = 2, -- number of function evaluations per step
order = 2, -- solver order (informational)
needs_model = true, -- true if step() needs model_fn callback
stateful = false, -- true if solver carries state across steps
stochastic = false, -- true if solver uses randomness (SDE)
}
```
**Required function — single-eval solver:**
```lua
function step(xt, vt, t_curr, t_prev, n)
-- xt: mutable FloatArray — current latent state (modify in-place)
-- vt: read-only FloatArray — velocity at (xt, t_curr)
-- t_curr: float — current timestep
-- t_prev: float — next timestep (we step FROM t_curr TO t_prev)
-- n: int — total elements in xt/vt
local dt = t_curr - t_prev
for i = 0, n - 1 do
xt[i] = xt[i] - vt[i] * dt
end
end
```
**Required function — multi-eval solver** (when `needs_model = true`):
```lua
function step(xt, vt, t_curr, t_prev, n, model_fn, vt_buf)
-- Additional args when needs_model = true:
-- model_fn(xt_array, t_val): evaluates the model at (xt_array, t_val),
-- writes velocity output to vt_buf
-- vt_buf: mutable FloatArray — receives model_fn output
local dt = t_curr - t_prev
local t_mid = t_curr - 0.5 * dt
-- Save state
local k1 = {}
local xt_orig = {}
for i = 0, n - 1 do
k1[i] = vt[i]
xt_orig[i] = xt[i]
end
-- Evaluate at midpoint
for i = 0, n - 1 do
xt[i] = xt_orig[i] - 0.5 * k1[i] * dt
end
model_fn(xt, t_mid) -- result appears in vt_buf
-- Final update using midpoint velocity
for i = 0, n - 1 do
xt[i] = xt_orig[i] - vt_buf[i] * dt
end
end
```
---
### Scheduler
Schedulers produce a timestep sequence for the denoising trajectory.
**Metadata table:**
```lua
scheduler = {
name = "my_schedule",
display = "My Schedule",
description = "Custom noise schedule",
}
```
**Required function:**
```lua
function schedule(output, num_steps, shift)
-- output: mutable FloatArray — write num_steps timestep values
-- num_steps: int — number of timesteps to generate
-- shift: float — noise shift parameter from UI
for i = 0, num_steps - 1 do
output[i] = 1.0 - i / num_steps
end
apply_shift(output, num_steps, shift)
end
```
Timesteps go from `1.0` (pure noise) to `~0.0` (clean signal). The engine appends a final `0.0` step automatically — your schedule should produce `num_steps` values, not `num_steps + 1`.
**Common helper — shift warp:**
```lua
function apply_shift(ts, n, shift)
if shift == 1.0 then return end
for i = 0, n - 1 do
local t = ts[i]
ts[i] = shift * t / (1.0 + (shift - 1.0) * t)
end
end
```
---
### Guidance
Guidance modes control how the conditional and unconditional model predictions are combined.
**Metadata table:**
```lua
guidance = {
name = "my_guidance",
display = "My Guidance",
description = "Custom guidance mode",
}
```
**Required function:**
```lua
function guide(pred_cond, pred_uncond, guidance_scale, result, Oc, T, norm_threshold)
-- pred_cond: read-only FloatArray — conditional velocity prediction
-- pred_uncond: read-only FloatArray — unconditional velocity prediction
-- guidance_scale: float — the guidance scale (w) from the UI
-- result: mutable FloatArray — write the guided velocity here
-- Oc: int — output channels per timestep frame
-- T: int — number of timestep frames (n = Oc * T)
-- norm_threshold: float — APG norm threshold from the UI
-- Route through APG for stability (STRONGLY RECOMMENDED)
apg(pred_cond, pred_uncond, guidance_scale, result, Oc, T, norm_threshold)
end
```
> **⚠️ Important:** Always route through `apg()` for the base guidance computation. Raw linear interpolation (`result = uncond + w * (cond - uncond)`) produces severe audio artifacts (static, underwater sound, frequency distortion). The `apg()` function provides momentum smoothing, perpendicular projection, and norm thresholding that are essential for stable audio output. If your guidance mode needs custom math, apply it as a correction on top of the APG result.
**Available globals in guidance plugins:**
| Global | Type | Description |
|--------|------|-------------|
| `step_idx` | int | Current step index (0-based) |
| `total_steps` | int | Total number of denoising steps |
| `dt` | float | Current timestep delta (t_curr - t_next) |
| `t_curr` | float | Current timestep value |
| `params` | table | User-configured parameter values |
---
## The `apg()` Bridge
The `apg()` function is the native C++ APG (Analytical Perpendicular Guidance) implementation, exposed to Lua guidance plugins. It handles:
1. **Perpendicular projection** — removes the component of `(cond - uncond)` parallel to `uncond`, keeping only the steering signal
2. **Momentum smoothing** — exponential moving average across steps to prevent jitter
3. **Norm thresholding** — caps per-channel magnitudes to prevent blowup
**Signature:**
```lua
apg(pred_cond, pred_uncond, guidance_scale, result, Oc, T, norm_threshold)
```
All guidance plugins have access to this function. It is registered automatically on first use.
---
## Advanced: The `post_step()` Hook
For guidance modes that need to run extra model evaluations *after* the solver step (e.g., manifold projection), guidance plugins can declare a `post_step()` function. The engine detects this at load time and calls it after each solver step.
**When to use this:**
- Your guidance technique requires iterative refinement of the latent state
- You need to evaluate the model at positions different from the main solver trajectory
- The technique calls the model with conditioning and unconditioning separately
**Performance warning:** Each call to `eval_cond` or `eval_uncond` runs a full model forward pass. This is expensive — use sparingly.
```lua
function post_step(xt, t, n, eval_cond, eval_uncond, vt_cond, vt_uncond)
-- xt: mutable FloatArray — latent state after solver step (modify in-place)
-- t: float — timestep we just stepped TO (t_next)
-- n: int — total elements in xt
-- eval_cond: function(xt_arr, t) — runs model with conditioning, writes to vt_cond
-- eval_uncond: function(xt_arr, t) — runs model without conditioning, writes to vt_uncond
-- vt_cond: mutable FloatArray — output buffer for conditional velocity
-- vt_uncond: mutable FloatArray — output buffer for unconditional velocity
-- Example: one iteration of manifold projection
local a = math.abs(dt) / 2.0
eval_uncond(xt, t) -- fills vt_uncond
for i = 0, n - 1 do
xt[i] = xt[i] - a * vt_uncond[i] -- push away from uncond manifold
end
eval_cond(xt, t) -- fills vt_cond
for i = 0, n - 1 do
xt[i] = xt[i] + a * vt_cond[i] -- pull toward cond manifold
end
end
```
The `post_step` hook has access to the same globals as `guide()` (`step_idx`, `total_steps`, `dt`, `t_curr`, `params`).
The hook is **not called on the final step** (the latent is about to be decoded, so further projection is pointless).
---
## Parameter Schema
Plugins can declare user-facing parameters that appear in the UI. Parameters are defined in the `params` array of the metadata table.
### Slider
```lua
{ key = "strength", type = "slider", label = "Strength",
default = 0.5, min = 0.0, max = 1.0, step = 0.01,
hint = "Controls the effect intensity" }
```
### Select (Dropdown)
```lua
{ key = "mode", type = "select", label = "Mode",
default = "fast",
options = {
{ value = "fast", label = "Fast" },
{ value = "quality", label = "Quality" },
},
hint = "Choose between speed and quality" }
```
### Toggle
```lua
{ key = "enabled", type = "toggle", label = "Enable Feature",
default = false,
hint = "Turn this feature on or off" }
```
### Conditional Visibility
Parameters can be shown/hidden based on another parameter's value:
```lua
{ key = "sub_param", type = "slider", label = "Sub-Parameter",
default = 1.0, min = 0.0, max = 5.0, step = 0.1,
visible_when = { key = "mode", equals = "quality" },
hint = "Only visible when Mode is set to Quality" }
```
### Transform Expressions
The `transform` field allows the UI to apply a mathematical transformation to the displayed value before sending it to the plugin. This is useful when the internal value differs from what the user sees:
```lua
{ key = "sigma", type = "slider", label = "Noise σ",
default = 5, min = 0, max = 100, step = 1,
transform = "value * 0.05",
hint = "Displayed as 0-100, sent to plugin as 0-5" }
```
### Reading Parameters
Parameters are available via the `params` global table, keyed by their `key` field:
```lua
local strength = (params and params.strength) or 0.5
local mode = (params and params.mode) or "fast"
local enabled = (params and params.enabled) or false
```
Always provide a fallback default with `or` — `params` may be `nil` if no parameters have been set.
---
## FloatArray
All array data passes between C++ and Lua via the `FloatArray` userdata type. This is a **zero-copy** bridge — Lua reads and writes the same memory that the C++ engine uses.
**Indexing:** 0-based (matching C++ convention, not Lua's typical 1-based).
```lua
-- Read
local val = xt[i]
-- Write (only on mutable arrays)
xt[i] = val
-- Length
local n = #xt
```
Read-only arrays (like `pred_cond` and `pred_uncond` in guidance) will raise an error if you try to write to them.
---
## Available Globals
### Solver globals
| Global | Type | Description |
|--------|------|-------------|
| `step_index` | int | Current step index |
| `batch_n` | int | Number of batch elements |
| `n_per` | int | Elements per batch element |
| `params` | table | Plugin parameters |
### Guidance globals
| Global | Type | Description |
|--------|------|-------------|
| `step_idx` | int | Current step index (0-based) |
| `total_steps` | int | Total denoising steps |
| `dt` | float | Timestep delta |
| `t_curr` | float | Current timestep |
| `params` | table | Plugin parameters |
---
## Sandbox
Each plugin runs in an isolated Lua 5.4 VM with:
**Available:** `math`, `string`, `table`, `print`, `type`, `pairs`, `ipairs`, `tonumber`, `tostring`, `require` (for companion data files)
**Blocked:** `os`, `io`, `debug`, `dofile`, `loadfile` — no filesystem access, no shell commands, no process control.
The `require()` function works for loading companion Lua data files (e.g., precomputed constants in a separate `.lua` file in the same directory), but cannot load C modules.
---
## Complete Examples
### Minimal Solver
```lua
-- my_solver.lua
solver = {
name = "my_solver",
display = "My Solver",
description = "Simple Euler variant",
nfe = 1,
order = 1,
needs_model = false,
}
function step(xt, vt, t_curr, t_prev, n)
local dt = t_curr - t_prev
for i = 0, n - 1 do
xt[i] = xt[i] - vt[i] * dt
end
end
```
### Scheduler with Custom Curve
```lua
-- my_schedule.lua
scheduler = {
name = "my_schedule",
display = "Quadratic",
description = "Quadratic timestep spacing",
params = {
{ key = "power", type = "slider", label = "Power",
default = 2.0, min = 1.0, max = 4.0, step = 0.1 },
},
}
function schedule(output, num_steps, shift)
local p = (params and params.power) or 2.0
for i = 0, num_steps - 1 do
local frac = i / num_steps
output[i] = (1.0 - frac) ^ p
end
-- Apply shift warp
if shift ~= 1.0 then
for i = 0, num_steps - 1 do
local t = output[i]
output[i] = shift * t / (1.0 + (shift - 1.0) * t)
end
end
end
```
### Guidance with APG + Custom Logic
```lua
-- my_guidance.lua
guidance = {
name = "my_guidance",
display = "My Guidance",
description = "Warm-up guidance with linear ramp",
params = {
{ key = "warmup_steps", type = "slider", label = "Warm-Up Steps",
default = 3, min = 0, max = 10, step = 1 },
},
}
function guide(pred_cond, pred_uncond, guidance_scale, result, Oc, T, norm_threshold)
local warmup = (params and params.warmup_steps) or 3
local progress = math.min((step_idx or 0) / math.max(warmup, 1), 1.0)
local effective_scale = 1.0 + (guidance_scale - 1.0) * progress
apg(pred_cond, pred_uncond, effective_scale, result, Oc, T, norm_threshold)
end
```
---
## Tips
- **Test with Euler + Linear first.** The simplest solver/scheduler combination isolates your plugin's behaviour.
- **Use `print()` for debugging.** Output goes to the terminal panel in the app.
- **Lua tables for scratch space.** If you need temporary arrays, use Lua tables: `local tmp = {}; for i = 0, n-1 do tmp[i] = 0 end`. They're slower than FloatArrays but work for intermediate calculations.
- **Be careful with the loop range.** FloatArrays are 0-indexed: `for i = 0, n - 1 do ... end`.
- **Guidance plugins: always use `apg()`.** Raw math without APG produces audio artifacts. Apply your custom logic as a delta on top.
- **Stateful plugins** can use file-level `local` variables to carry state across steps (e.g., previous velocity buffers, error accumulators). These reset when the plugin is reloaded.
docs/RELEASING.md
+116
View File
@@ -0,0 +1,116 @@
# Releasing HOT-Step CPP — agent runbook
How to cut and publish a release, plus the non-obvious gotchas. Written for an
agent (or human) driving the process with the `gh` CLI on Windows/Git-Bash.
## TL;DR
Releases are cut by **pushing a `vX.Y.Z` tag**. The `Release` workflow builds
every platform and creates a **draft** GitHub Release; you review and publish it.
A separate `Cache Warm` workflow keeps the engine build cache on `master` so
release builds take ~10–15 min instead of ~1.5h for the CUDA jobs.
## Prerequisites
- `gh` authenticated (as `scragnog`).
- On `master`, working tree clean, everything committed **and pushed**.
- Pick a semver version **without a hyphen**: `vX.Y.Z` (hyphens are reserved for
test/pre-release tags — see gotchas).
## 1. (Optional) Compile-test before releasing
To verify CI compiles without cutting a real release, push a throwaway
**hyphenated** tag — it triggers the same build pipeline but is ignored by the
changelog logic:
```bash
git tag -a vX.Y.Z-CI-Test -m "compile test" && git push origin vX.Y.Z-CI-Test
# ...watch it (section 3)... then delete when done:
gh release delete vX.Y.Z-CI-Test --cleanup-tag --yes # removes draft + remote tag
git tag -d vX.Y.Z-CI-Test
```
Re-pushing the **same** `-CI-Test` name is free (delete remote+local, recreate,
push). Tags cannot be renamed.
## 2. Cut the release
```bash
git tag -a vX.Y.Z -m "vX.Y.Z — <one-line summary>"
git push origin vX.Y.Z
```
This triggers `Release` → builds Windows (cuda13.1 / cuda12.8 / vulkan / cpu),
Linux (same four), macOS (Metal) → creates a **draft** release with **22 assets**
(11 archives + 11 `.sha256`).
To change the commit or re-run: delete + re-push the tag (it rebuilds).
## 3. Monitor
```bash
gh run list --limit 5
gh run view <run-id> # per-job status + timings
```
To read a **failed/cancelled job's** log while the run is still in progress
(`gh run view --log` won't show it yet), pull it from the API:
```bash
MSYS_NO_PATHCONV=1 gh api repos/scragnog/HOT-Step-CPP/actions/jobs/<job-id>/logs > log.txt
```
`MSYS_NO_PATHCONV=1` stops Git-Bash rewriting the leading-slash API path into a
filesystem path.
## 4. Publish
The workflow leaves the release as a **draft**. Verify the asset count (18) and
the `What's Changed` notes, then:
```bash
gh release view vX.Y.Z --json assets --jq '.assets | length' # expect 22
gh release edit vX.Y.Z --draft=false --latest
```
## 5. Cleanup
Delete any leftover test tags and their drafts (see section 1).
## Build caching — why releases are fast (and how it breaks)
- **GitHub Actions caches are ref-scoped.** A cache saved by one tag run is NOT
visible to a different tag run — only **default-branch (`master`) caches** are
visible to every run, including release tags. So releases can only reuse a
cache that was created on `master`.
- **`.github/workflows/cache-warm.yml`** builds the engine on `master` (when
`engine/ggml` or `engine/CMakeLists.txt` change, or via manual dispatch) and
saves the build dir under the **same cache keys** `release.yml` uses. Release
runs restore it and skip the CUDA compile (the long part).
- **Timings:** cold (no master cache) CUDA jobs ≈ 1.5h each; warm ≈ 7–13 min.
- **If CUDA suddenly rebuilds slow:** the master cache is missing/stale. Re-warm
it: GitHub → Actions → **Cache Warm** → *Run workflow* (on `master`). It also
auto-runs when `engine/ggml`/`CMakeLists.txt` change.
- Cache reuse depends on git-restored source mtimes (incl. the **ggml submodule**
— its `.cu` files live in the submodule's own history, not the superproject).
## Gotchas / lessons learned
- **Windows runner is pinned to `windows-2022`.** Do NOT switch to
`windows-latest` — that's windows-2025, whose MSVC (`_MSC_VER >= 1950`) is
rejected by CUDA 12.8/13.1 `nvcc` (`host_config.h`: VS 2017–2022 only).
- **Any pushed `v*` tag triggers the Release pipeline.** Use `vX.Y.Z` for
releases and `-CI-Test` (or other hyphenated) tags for throwaway checks; delete
them afterward. Don't push local feature tags that match `v*`.
- **Changelog range** = commits since the previous **non-hyphenated** tag
(`git describe ... --exclude '*-*'`). This is why a stray `vX-CI-Test` tag must
not be treated as a release; the exclude guard handles it, but still clean up.
- **The release is a draft** — it does not auto-publish. Review before going live.
- **Cache key** = `cmake-<runner>-<variant>-<hash(engine/ggml, CMakeLists)>`.
Changing the runner image invalidates it (compiler abs-paths bake into
`CMakeCache.txt`); the key includes the runner image to prevent stale restores.
## Reference
- Workflows: [`.github/workflows/release.yml`](../.github/workflows/release.yml),
[`.github/workflows/cache-warm.yml`](../.github/workflows/cache-warm.yml)
docs/TRAINING.md
+160
View File
@@ -0,0 +1,160 @@
# Training System — Architecture & Continuation Guide
*The complete map of HOT-Step's native training system (Training Studio). Written as the handoff/continuation doc: everything an agent or contributor needs to keep working on this subsystem in a fresh session. Built 2026-07-27/28; all measurements from an RTX 5090 (32 GB, sm_120).*
## What it is
End-to-end adapter training **entirely in C++/GGML — no Python anywhere**:
```
Dataset creation → Tensor preprocessing → LM LoRA training (0.6B/1.7B/4B)
→ DiT LoRA training (LoKR planned)
→ Pure-LM audition (A/B, no DiT influence)
```
Side-Step (`D:\Ace-Step-Latest\Side-Step`, local) is the Python reference implementation this system reaches parity with; its dev sanctioned porting. Dataset/sidecar formats are byte-compatible with Side-Step; tensor caches deliberately are not (safetensors, not pickle).
## Component map
| Piece | Engine | Server | UI |
|---|---|---|---|
| Dataset creation | — (uses ace-server `/understand` optionally, legacy) | `routes/training.ts` + `services/training/{sidecarIO,datasetScan,labelStore,essentiaClient,enhanceService,captionPrompt,datasetBuilder,labelingQueue}.ts` | `training-studio/{DatasetList,NewDatasetWizard,SampleGrid,SampleDrawer,LabelPanel,EnhancePanel,BuildPanel}.tsx` |
| Preprocess | `ace-train preprocess` (`engine/src/train/preprocess-*.h`, `st-write.h`) | `preprocessRunner.ts`, `preprocessStatus.ts`, `aceTrain.ts` | `Preprocess{Panel,OptionsForm,VariantCard}.tsx` |
| LM training | `ace-train train-lm` (`lm-*.h`: graph/optim/ckpt/bf16/vram/data/extract/export/selftest/train-run) | `trainLmRunner.ts`, `trainLmStatus.ts` | `TrainPanel.tsx`, `TrainLmForm.tsx`, `TrainingChart.tsx`, `TrainingRunStats.tsx` |
| DiT training | `ace-train train-dit` (`dit-*.h`: same family) | `trainDitRunner.ts`, `trainDitStatus.ts` | `TrainDitForm.tsx` (+ shared chart/stats) |
| Audition | `POST /codes-decode` on **ace-server** (`hot-step-server.cpp`, ~174 lines) | `audition{Service,Runner,Store}.ts` | `AuditionCard.tsx`, `AuditionPlayer.tsx`, `LmAdapterPicker.tsx` |
| Engine lifecycle | — | `services/aceEngineProcess.ts` (stop/restart with epoch-guarded respawn cancellation) | engine-paused banners |
Frozen contracts (types, routes, JSONL event schemas, CLI) are duplicated verbatim between `server/src/services/training/types.ts` and `ui/src/services/trainingApi.ts` — **kept in sync by hand, deliberately**.
Job model: one global promise-chain queue (`labelingQueue.ts`), SSE streams with replayable capped buffers, `_meta.json` persistence, `TrainingMetricEvent` for training numbers. GPU jobs stop the ace-server child first (`stopEngine`, default on) and restart it in a `finally`.
## Key design facts (the ones that bite)
**Data & labeling**
- Sidecars (`<stem>.txt`) live **next to the audio**; `dataset.json` in the source folder — Side-Step-compatible on purpose. Studio-private state (audio_codes, provenance, raw analyzer results) lives in `server/data/training/datasets/<slug>/labels/`.
- Labeling is engine-free: Essentia (BPM/key, cached by size+mtime) ∥ Genius lyrics (relaxed collab matching — "Electric Callboy & BABYMETAL" broke exact primary-artist equality) → Gemini captions **with the audio attached** (96 kbps MP3 inline; the local caption is *omitted* from the prompt to avoid anchoring). `ace-understand` is out of the default flow (weak captions, hallucinated lyrics, J-pop prior); still reachable via API `useUnderstand:true`.
- Dataset language is **declared, not detected** (`default_language`, forced-write on every label pass).
- Lyrics *absence* never writes `is_instrumental` — only lyrics presence writes `false`.
**FSQ (critical correctness area)**
- `engine/src/fsq-quant.h` is the single source of truth. The reference path is **ResidualFSQ `preserve_symmetry`**: soft clamp `c=1+1/(L−1)` → `tanh(z/c)·c` → hard clamp → `floor((L−1)(w+1)/2+0.5)`. NOT `FSQ.bound` (that branch never executes in vqp). Encode verified 13046/13046 vs the checkpoint's own tokenizer.
- `verify-hooks.ps1` has 2 FSQ hooks — upstream syncs must not revert this.
- Side-Step's stored `lm_codes.jsonl` are a **bf16 ceiling** (~85% identity); engine-F32 vs fp32-reference is the honest gate (99.0%).
**LM trainer**
- Hand-rolled loop (`lm-optim.h`): ggml-opt can't grad-clip and its grad accessor crashes with dynamic graphs. Persistent grad accumulators indexed by forward-node order, **in-graph** global-norm clip at 1.0, AdamW nodes, Side-Step-exact LR schedule (`lr(0)=0`, 5% warmup, cosine→0.1).
- ggml gap catalogue: `out_prod` F32-only (the central constraint), no backward for flash-attn / `SET_ROWS` (KV cache) / fused swiglu; CE labels are dense one-hot; `CONCAT` has no backward (use `ACC`); rms_norm backward is wrong on **non-contiguous (permuted) inputs** — norm before permute.
- 4B fits ~12 GB via per-layer checkpoint segments + a transient per-layer F32 window + chunked-CE head (BF16 embedding). Naive path preserved byte-identical for 0.6B/1.7B.
- `--weights bf16` (experimental): backward surgery rewrites `out_prod(W_f32,gT)` → `mul_mat(cont(transpose(W_bf16)),g)`. **1.256× at 4B** measured; gradient cos ~0.9992 vs F32 (BF16 rounding compounds sub-linearly with depth; partly ggml-cuda's dst-BF16 rounding, only fixable by a vendor patch). Zero vendor patches shipped; `engine/ggml` submodule is clean upstream.
- `--batch` intentionally **not built** — measured amortisable overhead below the bar where it matters (4B: 9.3% < 10%); flag exits 2 citing the numbers.
- Defaults (Side-Step parity, from Rob's real runs): target loss **4.0** (not 0.4!), GA 2, epochs 75, milestone step 1.0, lr 1e-4.
- Self-test: `ace-train train-lm --self-test` — T1–T13 (+T14–T17 bf16, report-not-gate). T5/T4 finite differences run **TF32-off in a child process** (cuBLAS TF32 noise defeats central differences; gradients were always right). Known pre-existing 4B rough edges: T11e/T12 slightly over bars (BF16-dtype-resolution class).
**DiT trainer**
- Trainable graph is **bit-identical** to the inference forward (17 debug-tensor diff = 0.0). Unfused load via `g_dit_load_no_fuse` (mirrors `g_qwen3_load_no_fuse`; the zero-stub-adapter trick is dead).
- F32 mirror streamed from the GGUF via CPU-backend load (a GPU-copy mirror transiently costs +5–8 GB and OOMs cards the steady state fits).
- Crop-window training (random window/step). No flash-attn backward ⇒ O(S²·Nh) retained softmax ⇒ full-song full-depth ≈ 68 GB: **impossible**; crop ~1000–1250 frames max at 24 GB full depth; `--layers` top-K ladder for smaller cards; <16 GB refused.
- Product loss = flow_snr (window-normalized — batch-1 mean normalization is silently a no-op, the trap) + channel_balance from `channel_stats.json`; lr default **5e-4** (1e-4 measurably does not train).
- Defaults: **LoRA** epochs 400, r128/α256, lr 5e-4, GA 4; **LoKR** epochs **250**, dim512/α512/factor 6, lr **2e-3**, GA **4**, target-loss 0.6, loss-weighting `none`. Genre ratio 30%, target-MLP **on** for both. DiT target-loss 0.4 requires Side-Step-length runs; epoch cap is the practical stop.
- **LoKR lr and grad-accum move together** (2026-07-30 five-run A/B on gunship_unicorn). The old 1e-2 @ GA 20 reproduced Side-Step's effective batch of 20 by accumulation; 2e-3 @ GA 4 is the same effective LR per sample under linear scaling and measured *identical* epochs-to-target (227 vs 228) with better-behaved gradients (median grad-norm 0.062 vs 0.031 — the √5 a 5× smaller batch predicts — and no warmup spike, where GA 20 peaked at 13.5 on epoch 1). Setting one without the other is an untested config.
- **The 250-epoch LoKR horizon is a schedule fix, not a shortcut.** Every 400-horizon run stopped with the cosine only halfway down (LR still ~50% of peak), so it never decayed *into* the target. 250 cut epochs-to-0.6 from 228 → 203 and audio-seconds-to-target by 14% — the only one of five knobs tested that reduced the work required rather than rearranging it. Don't go much lower: that run used 203 of its 250, and a horizon under the epochs actually needed strands the LR at its 10% floor with the tail unfinished.
- **Time-to-target is set by audio-seconds seen, not epochs or update count.** ~85,000 audio-seconds to reach 0.6, stable to ±5% across a 5× change in update count and a 1.8× change in window length. Changing grad-accum or crop length only re-slices it; only the schedule shape (above) and engine throughput move it.
- Adapter output: PEFT dir, loads through **both** `adapter-merge.h` and runtime paths (round-trip reproduces trainer loss; that check is a permanent gate). Exported **BF16** since 2026-07-30 (LyCORIS/Side-Step parity; halves a dim-512 census from 872 → 436 MB). LK5 gates the dtype — it compares alpha as a *bit pattern* against the writer's own rounding, so an alpha that is not BF16-exact still fails honestly.
**LoKR `apply()` — never let the token axis reach `ne2` of a mul_mat** (2026-07-30, `docs/plans/2026-07-30-dit-trainer-step-profile.md`)
- **The single largest perf defect found in this trainer: 9.7×.** LoKR training ran at 1902 ms/step where the same run with a LoRA took 116 ms — 16.4× — and the whole gap was `DitLokrAdapter::apply()`.
- **Mechanism.** Both kron contractions have a 2-D trainable factor (`ne2 == 1`) as src0. If src1 carries the token count in `ne2`, ggml emits the weight gradient as `out_prod(src1, grad)` with `dst->ne[2] == S`, and ggml-cuda's `out_prod` takes its `dps2 > 1` **fallback**: one `cublasSgemm` per token (`out-prod.cu:96-108`). One `OUT_PROD [512,5,344,1]` node = 344 launches of a 512×5 GEMM; ×189 such nodes ≈ 65,000 micro-GEMMs per micro-step from that shape alone, plus a `repeat_back` to reduce the per-token gradient slabs. Op histogram: 80% `OUT_PROD` + 8% `REPEAT_BACK`.
- **Fix (free).** `[in_n, in_m, S]` and `[in_n, in_m·S]` are the SAME BYTES, so folding the token axis into the column count is a pure `ggml_reshape_2d` — it leaves `ne2 == 1`, takes the strided-batched fast path in one call, and removes the `repeat_back` entirely. Restore 3-D only around the `permute`s, which genuinely need it.
- **Generalise this.** Any adapter or side-path whose trainable factor is 2-D must keep tokens out of `ne2`, or it silently buys the per-token fallback. A LoRA never hit it because its A/B matmuls are already 2-D.
- Gated by **LK3** (kron-matvec vs materialized-kron reference, 2.86e-08) and **LK4** (finite differences). A loss-level A/B is the *wrong* check here and misled once: the VRAM auto-fit picked crop 692 vs 694 between the two runs, so the losses differed for reasons unrelated to the change. Pin `--crop` **and** `--layers` if you ever want one.
- **`--profile-step <n>` / `--profile-ops`** are the tools that found it, and they stay: per-micro-step buckets (assemble/upload/build/backward/alloc/compute/readback/free) plus graph node and scheduler split counts, and a per-node op/shape histogram for one warmed-up step. `dit_train_log.json` now carries `runtime.backend` / `graph_nodes` / `sched_splits` / `profile_ms`, so a finished run explains itself — before this, "was that run even on CUDA?" was unanswerable after the fact.
**Muon optimizer** (2026-07-30, `--optimizer adamw|muon`, shared by both trainers via `lm-optim.h`)
- **DEFAULT for the DiT** since the ear test. Measured on gunship_unicorn (LoKR dim512+MLP, 32 layers, crop/depth pinned): **161 epochs to ma5 0.6 vs AdamW's 227** — 1.41× more sample-efficient — and ~1.23× on wall-clock once bucketed. Rob's own run reached 0.6 in ~5 min and the adapter was judged perfect by ear. AdamW is unchanged and one dropdown away.
- **Per PARAMETER, not global.** 2-D parameters with a short side ≥ `--muon-min-dim` (16) get orthogonalised-momentum (Newton-Schulz) updates; everything else stays on AdamW. A LoKR `w1` is `[4,5]` — orthogonalising a 4-dimensional subspace is vacuous — so on a DiT LoKR run the split is **448 Muon / 352 AdamW**. Every published Muon uses this hybrid.
- **The LR does NOT mean AdamW's.** Muon's update is normalised by construction, so per element it is ~20× smaller; `--muon-lr-scale` multiplies the shared schedule for Muon parameters only. Sweep on the DiT: 5 undershoots, **20 matches**, 50 overshoots. `--muon-lr-scale` and the base `--lr` are not independent.
- **Free VRAM:** one momentum buffer instead of two — 16315 → 15443 MB trainer-owned at LoKR dim512 (**872 MB**). `dit-vram.h` still charges for both, so the auto-fit is conservative, not wrong.
- **Bucketing is what makes it affordable, and its knob is non-monotone.** Same-shape parameters are batched into one Newton-Schulz. Optimizer step, LoKR dim512+MLP: **298.6 ms** (bucket 1) → 136.1 (8) → **138.0 (16, default)** → 187.3 (64) → 201.2 (128), against AdamW's **19.2 ms**. Larger buckets are WORSE because the gather is a left-folded `ggml_concat`, quadratic in bucket size. ~55 ms of what remains is irreducible NS FLOPs (~2.8 TFLOP/step); killing the concat (allocate accumulators/momenta as slabs of one per-bucket tensor) is the remaining win.
- **LM trainer: wired and functional, NOT yet defaulted.** All 392 LoRA parameters qualify at rank 16 (**the short side of a LoRA `A` is the rank** — a rank-8 adapter would fall entirely to AdamW; the startup line reports the split). 8-epoch smoke: loss 7.193 vs AdamW's 7.581 at +14% step time. That window is far too short to conclude anything — the DiT looked like parity at 10 epochs and was 1.41× ahead at 200. A run to target 4.0 is the honest test.
- Gated by **MU1**: update vs a double-precision host reference (CPU 5.7e-05, bar 1e-4; CUDA reported not gated — cuBLAS TF32 floors five chained iterations at ~4e-3), semi-orthogonality of **the graph's own** O, and a bucket case with grads spread ×8/×1/×⅛ that catches a global-instead-of-per-slab Frobenius norm.
**DiT micro-batching + gradient checkpointing** (2026-07-29, `docs/plans/2026-07-29-dit-batching-checkpointing.md`)
- `--batch <n>` (default **1** = OFF, range 1-16): crops per micro-batch, drawn from that many DIFFERENT songs (Side-Step DataLoader parity — B same-song crops was never on the table). Clamped down to the dataset's own song count with a `warn` when the variant has fewer songs than requested; further clamped by ggml's CUDA `REPEAT_BACK` cap on `Nkv·max(S,enc_S)·B` (GQA head-expansion backward — **both** attentions expand, and cross-attention's token axis is the dataset's padded `enc_S`, not the crop), and by the C4 auto-fit's own VRAM order (crop shrinks first, then checkpoint segments rise, then B itself gives with a warn — **before** depth is ever touched, since depth is the quality axis). `--grad-accum` counts **micro-batches**, not samples — effective samples/optimizer-step is `batch × grad-accum`.
- **Why the batch default is 1** (changed 2026-07-29 from 5, on the measurement below): batching is ~2.5× SLOWER at full depth on a 32 GB card and ~2.4× faster on shallow / partial-depth runs, so it ships off and is opt-in for the shallow case. The same default is mirrored in `DitTrainArgs`, the `ace-train` usage text, the `/api/training/train-dit` route fallback, `TrainDitOptions` (both mirror files) and `TRAIN_DIT_DEFAULTS` (which `TRAIN_DIT_LOKR_DEFAULTS` spreads, so LoKR inherits it). `--ckpt` stays at 1 (auto), and auto resolves to a single unsegmented run whenever full depth fits — `seg_candidates()` tries 1 first — so the default pair is the monolithic graph.
- **Mixed-length batches mask the pad in ATTENTION, not just in the loss.** When any element of a micro-batch is padded (a song shorter than the crop), `dit_batch_assemble` switches the self-attention mask from the shared `[S,S]` broadcast to a per-element `[S,S,1,B]` pair: `sa` (sliding window + padded KV columns) for `layer_type 0` and `sa_pad` (padded KV columns only, no window) for `layer_type 1`, which otherwise takes no self-attention mask at all. Without this the pad reaches every valid query as an attention KEY and from there the adapter gradients — the design-B4 loss mask cannot stop it, since it only zeroes the padded frames' own loss contribution. Padded QUERY rows are deliberately left unmasked (a fully-masked softmax row is NaN); they are inert anyway, because a zero output gradient contributes exactly zero to dQ/dK/dV. Self-test rungs SB3/SC2 gate it by dirtying the padded INPUT and requiring loss and every adapter gradient to move by EXACTLY 0. The unpadded case keeps the `[S,S]` fast path, and B=1 can never pad.
- **Grad-accum weighting is an element share.** A micro-batch's `t_lossgrad` is `elems_in_micro_batch / elems_in_window`, not `1/n_micro_batches`: the in-graph `t_lw` already normalises within a micro-batch, so `sum_mb (nb/wlen) * loss_mb` telescopes to Side-Step's global mean-of-per-sample-means over the whole optimizer window. The two agree while every micro-batch is full; a short tail of `nb < B` elements used to be over-weighted by `B/nb`. At B=1 the two forms are algebraically identical, so the `--batch 1 --ckpt 0` anchor is untouched.
- `--ckpt <n>` (default **1** = auto): hand-rolled segmented-recompute checkpointing — `0` disables it (byte-identical to the pre-batching monolithic graph), `1` lets the VRAM fit pick a segment count, `2-32` pins it. Mechanism: one no-grad forward over the whole trained stack to capture segment-boundary activations, then segments rebuilt LAST→FIRST with grads, each feeding the next via a `spike-s3`-style surrogate loss (`sum(seg_out ⊙ G)`) seeded from the boundary gradient above it; adapter grads accumulate in-place across every segment's graph compute (`lm-optim.h` untouched — it never needed to change, since PARAM-flagged boundary tensors get their own grad slot through the existing `ggml_build_backward_expand` machinery). Bit-exact vs `--ckpt 0` at the same seed/batch (self-test SC1-SC3).
- JSONL: `step.micro` counts MICRO-BATCHES (not samples — samples is `micro × batch`, modulo a short tail). `start` cannot know the resolved batch/segment count (it is emitted before the model loads), so it carries `batchRequested` / `ckptRequested`; the RESOLVED pair is `vram.batch` / `vram.ckptSegments` (+ `ckptSource`), emitted after the fit.
- Server/UI: `TrainDitOptions.batch` / `.ckptSegments` (route validation mirrors the CLI ranges; `ckptSegments` is the CLI's `--ckpt` value verbatim — 0/1/2-32). TrainDitForm's Advanced drawer has "Batch size" and a "Checkpointing" select (Auto/Off/2/4/8/16).
- VRAM model (`dit-vram.h`): the arena/KV-expand terms scale with a straight `×B`, validated (not re-derived — the pre-existing arena polynomial was already the fitted part) against a measured {B 1,5} × {ckpt 0,4} × {crop 375,750} grid at `--layers 8`: every cell landed within **+0.1% to +6.9%** of the trainer-owned high-water peak, always over- never under-predicting. The boundary-buffer term is exact arithmetic (matched every grid cell's reported `boundaryMb` to the byte). See the dated comment block in `dit-vram.h` for the full 8-cell table.
- Throughput (RTX 5090, 32 GB, LoKR dim512+MLP, full available depth = 32 layers, `--mirror bf16`, 10 epochs, same seed, tiny 5-song smoke tensors): `--batch 1 --ckpt 0` (yesterday's behavior) auto-fit a 674-frame crop and ran ~13.5 audio-seconds of training content per wall-clock second; `--batch 5 --ckpt auto` was forced to 4 checkpoint segments to fit batch 5 at full depth at all (batch 5 does **not** fit full-depth on this card with `--ckpt 0` — the auto-fit correctly reduces B to 1 rather than touch depth, confirming the C4 order), auto-fit a smaller 524-frame crop, and ran ~5.3 audio-seconds/wall-second — **~2.5× SLOWER**, not the ≥3× hoped for. Layers=8 (shallow, LoRA rank-16, no MLP) tells the opposite story: the grid's own probe timings show batching there is ~2.4× **faster** per crop. Read together: batching's ROI is depth/shape-dependent — a already-compute-bound full-depth 32-layer graph on a fast card gets little from wider batches while paying checkpointing's ~30-40% extra-forward tax and a forced-smaller crop; a shallow/small graph that under-uses the GPU at batch 1 gets a real win. Neither Side-Step's 0.223 s/(60 s crop) anchor nor the ≥3× target is met at full depth on this card/config — reported honestly per the design's own instruction to do so; smaller-depth or larger-VRAM configurations are the likelier place batching pays off and are untested here.
- bf16-mirror A/B (same-seed loss-curve diff, `--mirror f32` vs `--mirror bf16`) repeated at `--batch 5` (LoRA rank-16, `--layers 8`, `--ckpt 0`, 10 epochs): max abs loss delta **7.9e-5**, max relative delta **5.8e-5** — essentially unchanged from the original batch-1 measurement (7.6e-5), so amendment A2's CUDA GEMM-route change (batched-cublas at B>1) does not measurably widen the f32/bf16 drift.
- Loss curves at the throughput bench's Side-Step-parity LR (0.01) do **not** cleanly descend over just 10 toy epochs on a 5-song dataset — the cosine schedule (tuned for hundreds of production epochs) compresses its whole warmup+decay into 10-20 steps, producing a noisy rise-then-plateau in both the batch-1 and batch-5 runs. A same-config re-run at LR 0.001 descends cleanly in both (ma5 1.27→1.04 at batch 1, 1.34→1.09 at batch 5) confirming the mechanism itself is sound — the non-descent at LR 0.01 is a toy-bench artifact (LR schedule too aggressive for a 10-epoch/5-song smoke run), not a regression.
**`--bwd <outprod|mm>` — the mul_mat-backward reformulation** (2026-07-29, `engine/patches/mm-backward.patch`)
- **The constraint it removes.** ggml emits the MUL_MAT *activation* gradient as `out_prod(W, transpose(grad))`, and ggml-cuda's `OUT_PROD` is F32-only (`cublasSgemm`). That forces the frozen weight to be F32, which in turn drags the **forward** `mul_mat` into `ggml_cuda_op_mul_mat_cublas`'s F32 branch → TF32 tensor cores. Upstream `ggml.c` has always carried the alternative, commented out: `mul_mat(cont(transpose(W)), grad)`. It is dtype-agnostic, so a BF16 weight rides real BF16 tensor cores in **both** directions with no dequant and no F32 window. `--bwd mm` turns it on.
- **The two arms are provably the same op**, not an approximation: `out_prod(W[n,m,q1,r1], transpose(grad)[p,m,qq,rr]) -> [n,p,qq,rr]` and `mul_mat(cont(transpose(W))[m,n,q1,r1], grad[m,p,qq,rr]) -> [n,p,qq,rr]`, and `ggml_can_out_prod` / `ggml_can_mul_mat` reduce to the identical `b->ne[2] % a->ne[2] == 0, b->ne[3] % a->ne[3] == 0` broadcast pair. This is **not** a quality trade like `--weights bf16`, which genuinely changes the quantity computed — `--bwd` only changes how the same quantity is reached.
- **Per-layer GEMM measurement** (`ace-train spike gemmbench`, RTX 5090, Qwen3-4B projection shapes, reproduced 2026-07-29): `bf16_pre` beats the shipped `cur_lowvr` mix by **1.67–1.78×** per layer per step (1.41–1.64× vs `cur_naive`). Gradient parity vs the TF32 `out_prod` reference: **cosine 0.999996, max relative ~3–4e-3**. `bf16_cont` ≈ `bf16_pre`, i.e. the per-use `cont(transpose(W))` is free at these shapes — so the implementation needs **no pre-transposed weight cache**, which is why it lives entirely in graph-construction code and stays backend-agnostic.
- **End-to-end it is much smaller than that — 1.10–1.17×, and you must interleave to see it at all.** Same-seed `train-dit` A/B (LoKR dim512+MLP, `--mirror bf16 --layers 8 --batch 1 --ckpt 0 --order fixed`, 3 interleaved rounds per arm, min/median over epochs 2-6): crop 1250 → outprod median 4179 ms/epoch vs mm 3785 ms = **1.10×**; crop 375 → 1435 vs 1229 ms = **1.17×**. The gap vs the per-layer figure is Amdahl, not a broken lever: at `--layers 8` only 8 of 32 layers have a backward at all, and the step is dominated by the retained-softmax O(S²·Nh) attention (no flash-attn backward) plus LoKR kron reconstruction — which is exactly why the smaller crop, with less attention per projection, scores higher. **Do not trust a single non-interleaved A/B here**: run-to-run contention on a working machine is ±10%, the same size as the effect, and the first two one-shot runs gave 1.47× and 0.82× purely on which arm caught the interference.
- **Loss drift is negligible and deterministic.** Over 10 same-seed epochs: max absolute delta **7.4e-5**, max relative **6.7e-5** (6-epoch runs: 4.3e-5 at crop 1250, 2.7e-5 at crop 375); epoch 1 is bit-identical to 9 decimals, and each arm reproduces its own curve exactly across 3 repeats. Same order as the `--mirror f32`/`bf16` A/B (7.9e-5) — this is GEMM-reassociation rounding, not a different training run.
- **Self-test:** `ace-train train-dit --self-test` gives the identical **21/22** rung-for-rung with the env unset and with `GGML_BACKWARD_MM=1`, T9 (the known `E[v^2]` fingerprint drift) the only failure in both. The FD gate T4 reports the same max rel 2.3142e-04 on the same loss 1.659584 either way; SC1's graph grows 4386→4505 nodes under mm (the extra `cont` per converted site) while its loss and its segmented-vs-monolithic gradient delta stay bit-identical.
- **Contiguity guard (found by the self-test, not by reading).** `grad` becomes `mul_mat`'s src1, and the CUDA mul_mat kernels require it row-contiguous — `ggml-cuda/mmf.cu:28` asserts `nb10 == ts_src1` and **aborts** otherwise, where `out_prod` happily takes an arbitrary transposed view. The patch therefore takes the mm arm only when `ggml_is_contiguous(grad)` and otherwise falls back to out_prod. Without the guard the LoKR rung SC3 hard-aborts. `ggml_cont` on grad is not the fix — grad is activation-sized, so copying it costs more than the GEMM saves.
- **How it is wired.** `engine/ggml` is a submodule kept upstream-clean, so this ships as a vendored patch alongside `bf16-out-prod.patch` — disjoint files (`ggml.c` vs `ggml-cuda/*.cu`), applied by a `for p in engine/patches/*.patch` loop in CI's "Apply engine patches" step and guarded by **Hook 8** in `engine/verify-hooks.ps1`. The patch is **env-gated**: with `GGML_BACKWARD_MM` unset the emitted graph is byte-identical to upstream, so a stock build regresses nothing. `ace-train`'s `--bwd mm` sets the variable in `cmd_train_lm`/`cmd_train_dit` **before** `ggml_time_init()` and any model load — the patch latches it into a static on the first backward it builds, so it must be set before graph construction, and setting it there covers `--self-test` too. `--bwd outprod` deliberately does *not* clear an externally-set variable, so exporting `GGML_BACKWARD_MM=1` still A/Bs the whole self-test battery.
- **`--weights bf16` (LM) and `--bwd mm` COLLIDE — refused, not coerced.** Lever A reaches the same mul_mat backward by a different route: it lets ggml build the out_prod form and then rewrites those nodes in place, asserting **exactly 7 rewrites / 0 skipped / 0 residual** per segment (its S18 tripwire). Under `--bwd mm` ggml emits mul_mat directly, the surgery finds nothing, and the tripwire `GGML_ABORT`s — correct behaviour, but mid-run, after the model load. `ace-train train-lm` now exits 2 on the pair and the route answers 400. There is no version of that pair worth building: on the `f32-window` path `--bwd mm` gains nothing either, because the weight it transposes is the **F32 window**, so the GEMM stays TF32 and the extra `cont` is pure cost. **The LM already solved this problem its own way; `--bwd mm` is a train-dit lever.**
- **Defaults are deliberately split three ways.** The engine default is `outprod` (a bare `ace-train` invocation is unchanged). The **server** defaults **train-dit to `mm`** and **train-lm to `outprod`** — the LM's `weights` already defaults to `bf16`, so an LM default of `mm` would brick the *default* LM job on the collision above. `buildTrainLmArgs`/`buildTrainDitArgs` always emit `--bwd` so an older `ace-train.exe` rejects it loudly instead of silently running the slow path. Surfaced as "Backward GEMM" in both Advanced drawers; recorded in the `start` JSONL event and in `dit_train_log.json` / `lm_train_log.json`'s `config.bwd`.
**Trigger words (embedded in the adapter)**
- The tag was always trained in — `preprocess-run.h:192-204` bakes `custom_tag` into the caption before text encoding, and `lm-extract.h` re-applies it — but export used to drop it, leaving inference to guess the trigger from the filename (wrong for our adapters: the dir is `<name>-<size>`, not the tag).
- Both trainers now write `hot_step_trigger`, `hot_step_trigger_position` and `modelspec.trigger_phrase` into the adapter's safetensors `__metadata__`. Tensors and `adapter_config.json` are untouched, so ComfyUI/PEFT/Side-Step load it exactly as before. **Not** `adapter_config.json` — PEFT does `LoraConfig(**json)` and unknown keys are a version-dependent TypeError.
- Source of the value: `--trigger`/`--trigger-position` flags, else the variant's `preprocess_meta.json` (`custom_tag`/`tag_position`). The **variant meta, not the dataset row**, is authoritative — a dataset's tag can be edited after preprocessing, in which case the tensors (and the adapter) carry the old one. `tag_position: replace` embeds nothing: that path never puts the tag in the caption at all.
- Generation-side resolution lives in `services/generation/triggerWords.ts` and runs **server-side** for every caller: per adapter, manual override → embedded → filename fallback → none. A stack can mix prepend and append. The gate widened from "a DiT adapter is loaded" to "any adapter", so planner-LM adapters now contribute their trigger too.
- Legacy corpus: `server/scripts/stamp-adapter-triggers.mjs`, dry-run by default. Verified on a copy — 132 MB payload SHA-256 identical, header stays 8-byte aligned, `.bak` kept.
**Adapter layout (per-base + per-run, 2026-07-28)**
- `<adapters>/lm-06b|lm-17b|lm-4b/<artist>/<run>/` and `<adapters>/dit-<shorthand>/<artist>/<run>/`, where `<run>` = `YYYY-MM-DD_HH-MM-SS` (logs/ convention) — retraining an artist never overwrites an earlier adapter. Artist names carry **no** `-<size>` suffix; the parent folder says the base.
- Single source of truth: `server/src/services/training/adapterLayout.ts` (size slugs, the confirmed DiT shorthand map `xl-thirds`/`xl-base-turbo`/`xl-sft-turbo`/…, run stamps, latest-run resolution). `migrate-adapter-layout.mjs` moves an old corpus; its shorthand map must stay in sync.
- Writes go through `lmRunDirFor`/`ditRunDirFor` (fresh stamped dir); reads through `adapterDirFor`/`adapterDitDirFor` (newest run → unversioned artist dir → legacy flat `lm/<name>-4B` / root DiT dir). Two legacy forms are read-everywhere, written-never.
- Scanners: `GET /api/adapters/lm` walks all lm-* roots + legacy `lm/` (entries carry `lmSize`, `run`, `trigger`) — this is what the global-bar planner list, Lyric Studio's preset picker and the Training Studio picker all consume. `POST /api/adapters/scan` descends into `dit-*` folders (and their run subdirs) only.
- The audition's base-LM pinning now derives the size from the `lm-<size>` parent folder (suffix kept as legacy fallback) — do not reintroduce suffix-only derivation.
**Audition (pure-LM preview)**
- `POST /codes-decode`: codes → `detok_ggml_decode` → tail-call the existing VAE decode worker (layouts are byte-identical; zero changes to `/vae`). Deterministic; full song ≈ 3.1 s warm.
- A/B = two `/lm` calls, same explicit `lm_seed`, base side **must** byte-match an adapterless run (V5 hard gate). Base LM auto-derived from the adapter's `-<size>` suffix (`pickLmFor`) — never let the engine's `resolve_name` fallback pick (sticky 0.6B met a 4B adapter: "36 layers but model has 28").
- LM-echo sideband trap: never forward the `/lm` reply into another request — build requests fresh (see `generation-request-flow` skill).
## Build & verification rules
- `ace-train`-only changes: `cmake --build engine/build --config Release --target ace-train` — safe with the app running **if no ace-train.exe process is live** (training runs spawn it). New `train/*.h` headers need **no CMake change**.
- `hot-step-server.cpp` (ace-server) changes: full `dev-rebuild.bat` cycle (app goes down; Rob restarts with `dev.bat`). Never `build.cmd` directly, never `--clean-first`.
- Self-tests are the regression net: `ace-train train-lm --self-test`, `ace-train train-dit --self-test`, `ace-train spike …` (gemmbench, bf16layer, dit2 — the measurement harnesses that justified every design call; keep them).
- GPU discipline for agents: `nvidia-smi` before runs, stay under ~29 GB total, never kill Rob's python/node/ace-server, bounded runs while he's working.
- TypeScript: `server` `npx tsc --noEmit`; `ui` `npx tsc -p tsconfig.app.json --noEmit` (one pre-existing error in `globalParamsStore.ts:449` — `lmAdapter` vs `lmAdapters` in `types.ts:347` — is NOT ours).
## Measured reference numbers
| Thing | Number |
|---|---|
| LM 0.6B full E2E (17 songs, 16 epochs) | ~70 s wall |
| LM 4B, checkpointed | ~12.4 GB trainer-owned, ~1.06–1.34 s/micro-step; a real run auto-stops ~epoch 29 at target 4.0 ≈ 12 min (bf16: ~9.5 min) |
| DiT full-depth, LoRA r16 attn-only, crop 750 frames (**S=375**) | ~147 ms/step, ~23 GB; crop ceiling ~1000–1250 frames @24 GB. **Frames, not tokens: `S = frames / patch_size` and patch is 2, so "crop 750" is 750 frames = 30 s = S 375.** |
| DiT full-depth **LoKR dim512 + MLP**, crop ~690 frames (S≈344), RTX 5090 | **~195 ms/step, ~2.7 s/epoch at 14 songs** (was 1902 ms/step before the 2026-07-30 `apply()` fix). A 14-song adapter reaches ma5 0.6 in **~9 min / ~196 epochs**, vs ~90 min before |
| Preprocess | ~6 GB VRAM (vae-chunk 384; 1024 costs ~17 GB), ~3 s/song |
| Codes decode (audition) | ~200 ms warm 30 s clip; ~3.1 s full song |
| TF32 vs BF16 GEMM (5090) | ~95 vs ~165 TFLOP/s; end-to-end lever 1.256× at 4B |
## Pending / open decisions (Rob's list)
1. ~~One `dev-rebuild` owed~~ — **done**: the 2026-07-28 17:05 rebuild (during the SuperSep work) postdates every one of those source edits; `ace-train train-dit --help` shows `--no-target-mlp` and `ace-server.exe` carries the audition fixes.
1b. **Trigger stamper awaits Rob's approval**: `node server/scripts/stamp-adapter-triggers.mjs --datasets D:\Ace-Step-Latest\Datasets-LoRA-LoKR` prints the proposed table for 179 adapters (28 from real dataset.json ground truth, 151 from the filename). Nothing is written without `--apply`. Also outstanding: the per-adapter **manual trigger override** UI — the server already accepts `triggerSpecs` entries with `source:'override'` and a `path`, but no control emits them yet.
2. **bf16 listen test**: train twin LM adapters (f32-window vs `--weights bf16`), A/B via audition; ship-call by ear.
3. **LoKR for DiT** (`dit-adapter.h` has the parameterization seam ready) — Rob's preferred adapter type.
4. **ConvRot bases**: refused by the trainer pending a convrot spike (`…-convrot-*` GGUFs exist).
5. **BF16 `out_prod` vendor patch**: would ~2.5× DiT crop @24 GB, unlock 12 GB cards, and remove the bf16-lever's dst-rounding error — requires forking the ggml submodule (CI re-inits from upstream; see the rejected-P1 analysis in the levers plan).
6. **Top-K DiT adapter quality** on small cards: runs, but musical usefulness unmeasured.
7. Micro-batching: closed with measurements; revisit only if the graph-build overhead picture changes.
8. UI `minVram` hint string may be optimistic at r128+MLP defaults.
## Where the deep documents live
`docs/plans/` is **gitignored (local-only)** — on Rob's machine the full design/implementation plans exist: `2026-07-27-training-studio-design.md` (master design), `-dataset-studio-implementation.md`, `-preprocess-implementation.md`, `-lm-trainer-implementation.md`, `2026-07-28-lm-4b-training.md`, `-dit-trainer-implementation.md`, `-lm-speed-levers.md`, `-codes-preview.md`. Each contains the frozen contracts and verification ladders. The commit history (`git log --oneline --grep=training`) carries the measured numbers per milestone.
docs/images/hot-step-auto-gen.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 244 KiB

docs/images/hot-step-cover-studio.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 265 KiB

docs/images/hot-step-library.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 569 KiB

docs/images/hot-step-lyric-studio.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 283 KiB

docs/images/hot-step-model-manager.webp
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 140 KiB

docs/release-notes-v1.2.0.md
+88
View File
@@ -0,0 +1,88 @@
# v1.2.0
The StableStep release. Two flagship features: **StableStep**, a Stable Audio 3-powered
instrumental refiner that finally kills VAE fizz at the source, and **MIDI Studio**,
audio-to-MIDI transcription running on our own native GGML port of MuScriptor. Plus
timestep-gated adapters, parameter profiles, two new MDMAchine plugins, and a stack of fixes.
## ✨ StableStep — Stable Audio 3 instrumental refining
Generated tracks carry a characteristic high-frequency "fizz" from the ACE-Step
autoencoder. StableStep re-renders the instrumental through **Stable Audio 3**
(SDEdit-style partial re-noising, 8-step distilled rectified flow) so the fizz band is
*regenerated* with real spectral detail instead of filtered. Vocals are never touched:
BS-RoFormer splits them out (lead **and** backing), PP-VAE polishes them, and they're
remixed over the refined instrumental — the exact complement split guarantees nothing
is lost, and lyrics stay bit-identical.
- **One toggle + a strength slider** (0.10–0.60) in Post-Processing. The refine prompt is
derived from each track's own caption automatically (vocal descriptors stripped).
- **Runs natively in the C++ engine** — no Python. Two backends, selectable in-app:
- **GGML** (4 GGUF files, ~5.8 GB): CUDA, Vulkan, and CPU. The fastest option on
NVIDIA in our testing — ~2 s of compute for a 30 s clip on an RTX 5090.
- **ONNX Runtime / TensorRT** (~12 GB): NVIDIA alternative path.
- **Numerically validated end to end**: every ported component matches the reference
implementation at cosine > 0.9998; the two backends agree with each other at 0.9999.
- **Model Manager → StableStep tab** downloads either backend set from
[scragnog/HOT-Step-CPP-StableStep](https://huggingface.co/scragnog/HOT-Step-CPP-StableStep),
with license acceptance built in (Stability AI Community License — free for individuals
and orgs under $1M revenue). *Powered by Stability AI.*
- New engine surface for tinkerers: `POST /sa3-refine` (strength/steps/sampler/backend)
and SuperSep `level=4` — a dedicated vocals+instrumental split.
## 🎼 MIDI Studio — audio-to-MIDI on the native engine (#80)
Transcribe any library track or uploaded WAV/MP3 into multi-track MIDI (34 instrument
groups + drums) using **MuScriptor** (Kyutai & Mirelo) — ported phase by phase to our own
C++/GGML `ace-midi` engine and validated **byte-exact** against the reference
implementation. A 3.5-minute track transcribes in ~50 s on an RTX 5090; no Python anywhere.
- **Live piano roll** fills in while transcription runs, with per-channel instrument
coloring; crossfade playback slider between the original audio and the MIDI rendition,
plus per-instrument mute/solo.
- **Three model sizes** (small 103M / medium 307M / large 1.4B) with in-app gated-weight
download flow (weights are CC BY-NC 4.0 — non-commercial).
- Engine work along the way: exact-parity mel frontend + KV-cache greedy decode, F16 KV
cache to fix CUDA decode corruption, byte-exact event decode + MIDI writer.
- `ace-midi` ships in every platform bundle.
## 🎛 Adapter system
- **Timestep-dependent adapter gating** (interval experts / MoE) — restrict any stacked
adapter to a step window, e.g. one adapter shapes structure early, another handles
detail late. UI windows are evaluated **per step** (not raw t), and they now compose
correctly with Adapter VRAM quant (previously a silent 32 GB blowup).
- **PEFT DoRA support** + per-module `alpha_pattern`.
- **`runtime_lowrank` mode** — factor-apply without materialized deltas: the lowest-VRAM
way to run big adapter stacks.
- **Merge (low VRAM)** — opt-in requant of merged weights back to the base's native
quant type (~¼ merged-DiT VRAM on a Q8 base).
- LoKr Kronecker-apply self-test (`HOTSTEP_KRON_TEST`); `gain_domain` parse fix.
## 📋 Parameter profiles
Save, apply, and delete named full-state snapshots of your generation parameters —
in-app, with JSON import/export, inline rename, and click-to-inspect.
## 🔌 Plugins (MDMAchine)
- **MD HT Scheduler V3** and **MD Trajectory Anchor V5**, with user manuals.
- Max inference steps raised 200 → 300 (UI + engine clamp).
## 🔧 Fixes & polish
- **Storm streaming**: stream died instantly (premature `close` on body consumption);
"Keep DiT & VAE loaded" is now honored between stream slots.
- **In-app restart** was dead on Windows (ping-as-sleep hang meant taskkill never fired).
- **Queue**: deleted songs pruned from the persisted queue on load; Nuke All Generations
clears the queue store and recent-songs cache.
- **Logs**: GGML debug spam dropped at source with digit-insensitive dedup; terminal
stick-to-bottom pin survives layout shifts.
- MCP: generating model name appended to song titles.
---
**Models:** StableStep sets are on
[Hugging Face](https://huggingface.co/scragnog/HOT-Step-CPP-StableStep) or one click away
in the Model Manager. MuScriptor weights are gated (free) — request access via the links
inside MIDI Studio.
engine/.clang-format
+170
View File
@@ -0,0 +1,170 @@
---
Language: Cpp
AlignAfterOpenBracket: Align
AlignArrayOfStructures: Left
AlignConsecutiveAssignments: AcrossComments
AlignConsecutiveBitFields: AcrossComments
AlignConsecutiveDeclarations: AcrossComments
AlignConsecutiveMacros: AcrossComments
# AlignConsecutiveShortCaseStatements: AcrossComments
AlignEscapedNewlines: Left # LeftWithLastLine
AlignOperands: Align
AlignTrailingComments:
Kind: Always
OverEmptyLines: 1
AllowAllArgumentsOnNextLine: true
AllowAllParametersOfDeclarationOnNextLine: false
# AllowBreakBeforeNoexceptSpecifier: OnlyWithParen
AllowShortBlocksOnASingleLine: Never
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: Never
AllowShortLambdasOnASingleLine: Inline
AllowShortLoopsOnASingleLine: false
AlwaysBreakBeforeMultilineStrings: true
# Treat CUDA keywords/attributes as "attribute macros" and avoid breaking lines inside them
AttributeMacros:
- __host__
- __device__
- __global__
- __forceinline__
- __launch_bounds__
BinPackArguments: true
BinPackParameters: false # OnePerLine
BitFieldColonSpacing: Both
BreakBeforeBraces: Custom # Attach
BraceWrapping:
AfterCaseLabel: true
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: false
AfterNamespace: false
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: false
BeforeElse: false
BeforeLambdaBody: false
BeforeWhile: false
IndentBraces: false
SplitEmptyFunction: false
SplitEmptyRecord: false
SplitEmptyNamespace: false
# BreakAdjacentStringLiterals: true
BreakAfterAttributes: Never
BreakBeforeBinaryOperators: None
BreakBeforeInlineASMColon: OnlyMultiline
BreakBeforeTernaryOperators: false
# BreakBinaryOperations: Never
BreakConstructorInitializers: AfterColon
# BreakFunctionDefinitionParameters: false
BreakInheritanceList: AfterComma
BreakStringLiterals: true
# BreakTemplateDeclarations: Yes
ColumnLimit: 120
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: false
DerivePointerAlignment: false
DisableFormat: false
EmptyLineBeforeAccessModifier: Leave
EmptyLineAfterAccessModifier: Never
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
IncludeBlocks: Regroup
IncludeCategories:
- Regex: '".*"'
Priority: 1
SortPriority: 0
- Regex: '^<.*\.h>'
Priority: 2
SortPriority: 0
- Regex: '^<.*'
Priority: 3
SortPriority: 0
- Regex: '.*'
Priority: 4
SortPriority: 0
IncludeIsMainRegex: '([-_](test|unittest))?$'
IncludeIsMainSourceRegex: ''
IndentAccessModifiers: false
IndentCaseBlocks: true
IndentCaseLabels: true
IndentExternBlock: NoIndent
IndentGotoLabels: false
IndentPPDirectives: AfterHash
IndentWidth: 4
IndentWrappedFunctionNames: false
InsertBraces: true # NOTE: may lead to incorrect formatting
InsertNewlineAtEOF: true
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
LambdaBodyIndentation: Signature
LineEnding: LF
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCBinPackProtocolList: Auto
ObjCBlockIndentWidth: 4
ObjCSpaceAfterProperty: true
ObjCSpaceBeforeProtocolList: true
PPIndentWidth: -1
PackConstructorInitializers: CurrentLine
PenaltyBreakAssignment: 2
PenaltyBreakBeforeFirstCallParameter: 1
PenaltyBreakComment: 300
PenaltyBreakFirstLessLess: 120
PenaltyBreakString: 1000
PenaltyBreakTemplateDeclaration: 10
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Middle
QualifierAlignment: Left
#QualifierOrder: ['static', 'inline', 'friend', 'constexpr', 'const', 'volatile', 'type', 'restrict']
RawStringFormats:
- Language: Cpp
Delimiters:
- cc
- CC
- cpp
- Cpp
- CPP
- 'c++'
- 'C++'
CanonicalDelimiter: ''
ReferenceAlignment: Middle
ReflowComments: false # IndentOnly
SeparateDefinitionBlocks: Always
SortIncludes: CaseInsensitive
SortUsingDeclarations: LexicographicNumeric
SpaceAfterCStyleCast: true
SpaceAfterLogicalNot: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: false
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatements
SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyBlock: false
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: Never
SpacesInContainerLiterals: true
SpacesInLineCommentPrefix:
Minimum: 1
Maximum: -1
SpacesInParentheses: false
SpacesInSquareBrackets: false
SpaceBeforeSquareBrackets: false
Standard: c++17
TabWidth: 4
UseTab: Never
WhitespaceSensitiveMacros: ['STRINGIZE']
...
engine/.gitattributes
+1
View File
@@ -0,0 +1 @@
* text=auto eol=lf
engine/.github/workflows/ci-build.yml
Vendored
+77
View File
@@ -0,0 +1,77 @@
# Validate that the project builds on Ubuntu and macOS (no model download).
name: CI Build
on:
pull_request:
workflow_dispatch:
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Build (Ubuntu)
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq cmake build-essential pkg-config libopenblas-dev
mkdir build && cd build
cmake .. -DGGML_BLAS=ON
cmake --build . --config Release -j$(nproc)
- name: Build (macOS)
if: matrix.os == 'macos-latest'
run: |
mkdir build && cd build
cmake ..
cmake --build . --config Release -j$(sysctl -n hw.ncpu)
- name: Smoke test
run: |
./build/ace-lm --help 2>&1 | head -5
./build/ace-synth --help 2>&1 | head -5
./build/quantize --help 2>&1 | head -3
lint:
name: Lint & Static Analysis
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v4
- name: Install lint tools
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq clang-format clang-tidy cppcheck
- name: Run clang-format (check mode)
run: |
find . \
\( -path './.git' -o -path './ggml' -o -path './build' -o -path './vendor' -o -path './mp3' \) -prune -o \
-type f \( -name '*.c' -o -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.hpp' \) \
-print0 | xargs -0 clang-format --dry-run --Werror
- name: Run cppcheck
run: |
cppcheck --enable=all --error-exitcode=1 --inline-suppr \
--suppress=missingIncludeSystem \
--suppress=missingInclude \
--suppress=cstyleCast \
--suppress=constVariable \
--suppress=constVariablePointer \
--suppress=constParameterPointer \
--suppress=variableScope \
--suppress=uselessCallsSubstr \
--suppress=useStlAlgorithm \
--suppress=shiftNegativeLHS \
-i ggml -i build -i .git -i mp3 \
.
engine/.github/workflows/release.yml
Vendored
+263
View File
@@ -0,0 +1,263 @@
name: Build & Release Binaries
on:
release:
types: [published]
workflow_dispatch:
inputs:
release_tag:
description: 'Release tag to attach binaries to (e.g. v0.1.0)'
required: true
type: string
skip_linux:
description: 'Skip Linux build'
required: false
type: boolean
default: false
skip_mac:
description: 'Skip macOS build'
required: false
type: boolean
default: false
skip_windows:
description: 'Skip Windows build'
required: false
type: boolean
default: false
permissions:
contents: write
jobs:
build-linux:
name: Build · linux-x64
runs-on: ubuntu-22.04
if: github.event_name != 'workflow_dispatch' || !inputs.skip_linux
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Install build tools
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq cmake build-essential pkg-config libopenblas-dev
- name: ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
create-symlink: true
key: build-linux-ubuntu-22.04
- name: Install CUDA toolkit
uses: Jimver/cuda-toolkit@v0.2.30
with:
log-file-suffix: 'ubuntu-22.04.txt'
- name: Install Vulkan SDK
uses: humbletim/install-vulkan-sdk@v1.2
with:
version: 1.4.309.0
cache: true
- name: Configure & Build
run: |
./buildall.sh
- name: Smoke test
continue-on-error: true
shell: bash
run: |
BIN="build"
"$BIN/ace-lm" 2>&1 | head -5
"$BIN/ace-synth" 2>&1 | head -5
"$BIN/ace-understand" 2>&1 | head -5
"$BIN/neural-codec" 2>&1 | head -5
"$BIN/quantize" 2>&1 | head -3
"$BIN/mp3-codec" 2>&1 | head -3
- name: Resolve release tag
id: tag
shell: bash
run: |
if [ "${{ github.event_name }}" = "release" ]; then
echo "value=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "value=${{ inputs.release_tag }}" >> $GITHUB_OUTPUT
fi
- name: Package binaries
run: |
mkdir -p dist
cp build/ace-* \
build/quantize build/neural-codec build/mp3-codec build/*.so dist/
tar -C dist -czf "acestep-linux-x64.tar.gz" .
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "${{ steps.tag.outputs.value }}" \
"acestep-linux-x64.tar.gz" \
--clobber
build-mac:
name: Build · macos-arm64-metal
runs-on: macos-latest
if: github.event_name != 'workflow_dispatch' || !inputs.skip_mac
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
create-symlink: true
key: build-mac-macos-latest
- name: Configure & Build
run: |
mkdir build
cd build
cmake ..
cmake --build . --config Release -j "$(nproc)"
- name: Smoke test
continue-on-error: true
shell: bash
run: |
BIN="build"
"$BIN/ace-lm" 2>&1 | head -5
"$BIN/ace-synth" 2>&1 | head -5
"$BIN/ace-understand" 2>&1 | head -5
"$BIN/neural-codec" 2>&1 | head -5
"$BIN/quantize" 2>&1 | head -3
"$BIN/mp3-codec" 2>&1 | head -3
- name: Resolve release tag
id: tag
shell: bash
run: |
if [ "${{ github.event_name }}" = "release" ]; then
echo "value=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "value=${{ inputs.release_tag }}" >> $GITHUB_OUTPUT
fi
- name: Package binaries
run: |
mkdir -p dist
cd build
for bin in ace-* quantize neural-codec mp3-codec; do
install_name_tool -add_rpath @executable_path "$bin"
done
cp -P ace-* quantize neural-codec mp3-codec libacestep*.a libggml*.dylib ../dist/
cd ..
tar -C dist -czf "acestep-macos-arm64-metal.tar.gz" .
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "${{ steps.tag.outputs.value }}" \
"acestep-macos-arm64-metal.tar.gz" \
--clobber
build-windows:
name: Build · windows-x64
runs-on: windows-latest
if: github.event_name != 'workflow_dispatch' || !inputs.skip_windows
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- name: Cache CUDA toolkit
id: cache-cuda
uses: actions/cache@v4
with:
path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
key: cuda-12-windows-latest
- name: Install CUDA toolkit
if: steps.cache-cuda.outputs.cache-hit != 'true'
uses: Jimver/cuda-toolkit@v0.2.30
with:
log-file-suffix: 'windows-latest.txt'
- name: Install Vulkan SDK
uses: humbletim/install-vulkan-sdk@v1.2
with:
version: 1.4.309.0
cache: true
- name: Cache build directory
uses: actions/cache@v4
with:
path: build-msvc
key: build-msvc-${{ github.sha }}
restore-keys: |
build-msvc-
- name: Configure & Build
shell: pwsh
run: |
# Configure — only print errors
cmake -S . -B build-msvc `
-DGGML_CPU_ALL_VARIANTS=ON `
-DGGML_CUDA=ON `
-DGGML_VULKAN=ON `
-DGGML_BACKEND_DL=ON `
--log-level=ERROR 2>&1 | Where-Object { $_ -notmatch '^--' }
# Build — suppress per-file progress, only show warnings/errors
cmake --build build-msvc --config Release -j $env:NUMBER_OF_PROCESSORS `
-- /v:minimal /consoleloggerparameters:ErrorsOnly 2>&1 `
| Where-Object { $_ -match '(error|warning|FAILED|fatal)' -or $_ -eq '' } `
| Select-Object -Last 50
- name: Smoke test
continue-on-error: true
shell: bash
run: |
BIN="build-msvc/Release"
"$BIN/ace-lm.exe" 2>&1 | head -5
"$BIN/ace-synth.exe" 2>&1 | head -5
"$BIN/ace-understand.exe" 2>&1 | head -5
"$BIN/neural-codec.exe" 2>&1 | head -5
"$BIN/quantize.exe" 2>&1 | head -3
"$BIN/mp3-codec.exe" 2>&1 | head -3
- name: Resolve release tag
id: tag
shell: bash
run: |
if [ "${{ github.event_name }}" = "release" ]; then
echo "value=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "value=${{ inputs.release_tag }}" >> $GITHUB_OUTPUT
fi
- name: Package binaries
shell: pwsh
run: |
New-Item -ItemType Directory -Path dist | Out-Null
Copy-Item "build-msvc\Release\*.exe" dist\ -ErrorAction SilentlyContinue
Copy-Item "build-msvc\Release\*.dll" dist\ -ErrorAction SilentlyContinue
Compress-Archive -Path dist\* -DestinationPath "acestep-windows-x64.zip"
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: pwsh
run: |
gh release upload "${{ steps.tag.outputs.value }}" `
"acestep-windows-x64.zip" `
--clobber
engine/.gitignore
+13
View File
@@ -0,0 +1,13 @@
build/
*.wav
*.bf16
tests/*/
checkpoints/
models/*.gguf
adapters/*/
adapters/*.safetensors
__pycache__/
node_modules/
tools/webui/dist/
engine/.gitmodules
+3
View File
@@ -0,0 +1,3 @@
[submodule "ggml"]
path = ggml
url = https://github.com/ServeurpersoCom/ggml.git
engine/CMakeLists.txt
+742
View File
@@ -0,0 +1,742 @@
cmake_minimum_required(VERSION 3.21)
project(acestep-ggml LANGUAGES C CXX)
# CI cache generation: 2 (2026-07-16). This file is hashed into the GitHub
# Actions build-cache key — bump this comment to force cold builds when the
# cached objects themselves are suspect (v1.1.3 stale-cache mixed-ABI crash,
# issues #82/#83). Routine drift is handled by the .built-commit stamp guard
# in release.yml/cache-warm.yml; this is the manual override.
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# version.h: embed git commit hash into all binaries.
# runs on every build, only rewrites if the hash changed.
set(VERSION_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/version.h")
add_custom_target(version ALL
COMMAND "${CMAKE_COMMAND}" "-DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DOUTPUT=${VERSION_OUTPUT}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/tools/version.cmake"
BYPRODUCTS "${VERSION_OUTPUT}"
COMMENT "Checking git version"
)
# pthread: required explicitly on older glibc (< 2.34) where libpthread
# is not merged into libc. Modern distros link it implicitly but aarch64
# and older x86_64 toolchains need the explicit dependency.
find_package(Threads REQUIRED)
# Suppress MSVC fopen/sprintf deprecation warnings and Windows.h macro pollution.
# NOMINMAX: prevents Windows.h from defining min/max macros that collide with
# std::min/std::max (causes C2589 errors in solvers/schedulers).
# WIN32_LEAN_AND_MEAN: reduces Windows.h header bloat.
if(MSVC)
add_compile_definitions(_CRT_SECURE_NO_WARNINGS NOMINMAX WIN32_LEAN_AND_MEAN)
endif()
# Static MSVC runtime (/MT) for portable release builds.
# Eliminates the VC++ Redistributable dependency for end users.
# Only enable during release builds: -DHOT_STEP_STATIC_RUNTIME=ON
option(HOT_STEP_STATIC_RUNTIME "Use static MSVC runtime (/MT) for portable builds" OFF)
if(HOT_STEP_STATIC_RUNTIME AND MSVC)
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
message(STATUS "MSVC runtime: static (/MT)")
endif()
# Put executables and backend .so in the same directory (build root).
# Without this, ggml defaults to bin/ for .so but executables stay in root,
# and ggml_backend_load_all() can't find the backends at runtime.
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# macOS rpath: make binaries relocatable (portable release support).
# Without this, CMake bakes the absolute build directory into LC_RPATH,
# which breaks on any machine other than the one that built it.
# @executable_path tells dyld to look for dylibs next to the binary.
if(APPLE)
set(CMAKE_INSTALL_RPATH "@executable_path")
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_MACOSX_RPATH TRUE)
endif()
# Linux rpath: make binaries relocatable (portable release support).
# $ORIGIN tells the dynamic linker to search for .so files next to the binary.
if(UNIX AND NOT APPLE)
set(CMAKE_INSTALL_RPATH "$ORIGIN")
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
endif()
# DiT tensor names can exceed default GGML_MAX_NAME of 64
add_compile_definitions(GGML_MAX_NAME=128)
# Harden: mark fread/fwrite/etc with warn_unused_result on all platforms
if(NOT MSVC)
add_compile_definitions(_FORTIFY_SOURCE=2)
endif()
# CUDA architectures: cover Turing to Blackwell for distributed binaries (CI: CUDA 13.1 / 12.8).
# The CUDA 12.8 build additionally targets legacy Pascal/Volta (see below).
# Users can override with -DCMAKE_CUDA_ARCHITECTURES=native for local builds.
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
find_package(CUDAToolkit QUIET)
# Base arch list: Turing through Lovelace/Ada
set(CMAKE_CUDA_ARCHITECTURES "75-virtual;80-virtual;86-real;89-real;90-real")
if(CUDAToolkit_FOUND)
# Legacy GPUs — Pascal only (sm_60 P100, sm_61 GTX 10xx / P40 / P4).
# CUDA 12.x can still compile real SASS for these; CUDA 13.0 removed
# offline compilation support, so they go only into the cuda12.8 variant.
# Real SASS is required — the 75-virtual PTX above can't JIT backwards
# onto pre-Turing cards.
#
# NOTE: Volta (sm_70) is deliberately EXCLUDED. ggml's mma-based MMQ and
# flash-attention kernels have no device code for sm_70 (they need
# Turing+), so a sm_70 build crashes on Volta. Worse, sm_70 SASS is
# binary-compatible upward to Turing (sm_75), so the driver loads it on
# 75 cards in preference to JIT-ing the compute_75 PTX — which regressed
# all Turing users on the cuda12.8 bundle in v1.1.1 (#63). Pascal (major
# 6) is unaffected since Turing can't load major-6 SASS.
if(CUDAToolkit_VERSION VERSION_LESS "13.0")
list(APPEND CMAKE_CUDA_ARCHITECTURES "60-real;61-real")
endif()
# Blackwell sm_120a: CUDA 12.8+
if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.8")
list(APPEND CMAKE_CUDA_ARCHITECTURES "120a-real")
endif()
# Blackwell sm_121a: CUDA 12.9+ (same as upstream GGML)
if(CUDAToolkit_VERSION VERSION_GREATER_EQUAL "12.9")
list(APPEND CMAKE_CUDA_ARCHITECTURES "121a-real")
endif()
endif()
endif()
list(APPEND CMAKE_PREFIX_PATH /opt/rocm)
find_package(hip CONFIG QUIET)
if(hip_FOUND)
message(STATUS "building with AMD ROCm support")
add_compile_definitions(__HIP_PLATFORM_AMD__)
endif()
# Disable flash attention (cuda12-volta variant). ggml's mma flash-attention has
# no device code for Volta (sm_70); with this defined the engine takes the manual
# attention path instead. Pair with -DGGML_CUDA_FORCE_CUBLAS=ON (ggml) which
# likewise replaces the mma MMQ kernels. See engine/src/hot-step-build-flags.h.
option(HOT_STEP_DISABLE_FA "Disable flash attention (Volta / pre-Turing GPUs)" OFF)
if(HOT_STEP_DISABLE_FA)
add_compile_definitions(HOT_STEP_DISABLE_FA)
message(STATUS "[HOT-Step] Flash attention DISABLED (HOT_STEP_DISABLE_FA)")
endif()
# ggml as subdirectory, inherits GGML_CUDA, GGML_METAL, etc. from cmake flags
# CUDA graphs default on: standalone ggml ships them off. Overridable with
# -DGGML_CUDA_GRAPHS=OFF or at runtime with GGML_CUDA_DISABLE_GRAPHS=1.
if(NOT DEFINED GGML_CUDA_GRAPHS)
set(GGML_CUDA_GRAPHS_DEFAULT ON)
endif()
add_subdirectory(ggml)
# cpp-httplib (HTTP server library, used by ace-server)
add_subdirectory(vendor/cpp-httplib)
# Shared compile options and ggml linkage
macro(link_ggml_backends target)
target_include_directories(${target} PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
target_include_directories(${target} SYSTEM PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/ggml/include
)
if(MSVC)
target_compile_options(${target} PRIVATE /W4 /wd4100 /wd4505)
else()
target_compile_options(${target} PRIVATE -Wall -Wextra -Wshadow -Wconversion
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
endif()
target_link_libraries(${target} PRIVATE ggml Threads::Threads)
if(TARGET ggml-base)
target_link_libraries(${target} PRIVATE ggml-base)
endif()
foreach(backend cpu blas cuda metal vulkan)
if(TARGET ggml-${backend})
get_target_property(CURRENT_BACKEND_TYPE ggml-${backend} TYPE)
if (CURRENT_BACKEND_TYPE STREQUAL "MODULE_LIBRARY")
# DL mode: backend is loaded at runtime via dlopen,
# skip all link-time deps.
continue()
endif()
target_link_libraries(${target} PRIVATE ggml-${backend})
endif()
endforeach()
add_dependencies(${target} version)
endmacro()
# yyjson (MIT, fast JSON parser/writer)
add_library(yyjson STATIC vendor/yyjson/yyjson.c)
target_include_directories(yyjson PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/yyjson)
if(MSVC)
target_compile_options(yyjson PRIVATE /W0)
else()
target_compile_options(yyjson PRIVATE -w)
endif()
# Lua 5.4 (MIT, embedded scripting for plugin system)
# All .c files except lua.c (standalone interpreter) and luac.c (compiler)
file(GLOB LUA_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/vendor/lua/*.c")
list(FILTER LUA_SOURCES EXCLUDE REGEX "(lua|luac)\\.c$")
add_library(lua54 STATIC ${LUA_SOURCES})
target_include_directories(lua54 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/lua)
if(MSVC)
target_compile_options(lua54 PRIVATE /W0)
else()
target_compile_options(lua54 PRIVATE -w)
endif()
# ─────────────────────────────────────────────────────────────────────────────
# ONNX Runtime (SuperSep stem separation + VAE-ORT TensorRT acceleration)
# ─────────────────────────────────────────────────────────────────────────────
# Pre-built ORT GPU package. Resolution order:
# 1. ORT_ROOT cmake variable
# 2. ONNXRUNTIME_ROOT environment variable
# 3. Auto-detect from engine/deps/onnxruntime/ (populated by buildall.cmd)
#
# CUDAToolkit detection: the find_package at L73 is conditional on
# CMAKE_CUDA_ARCHITECTURES, so on cached re-configures CUDAToolkit_FOUND
# may be unset. Ensure it's always available for CUDA EP support.
find_package(CUDAToolkit QUIET)
# SuperSep itself is pure GGML and always built. This option now only controls
# whether the ONNX Runtime-dependent paths (StableStep's ONNX backend and the
# ONNX VAE/text-encoder) are compiled — they are the last ORT consumers.
option(HOT_STEP_SUPERSEP "Build the ONNX Runtime paths (StableStep ONNX backend, ONNX VAE)" ON)
set(ORT_ROOT "" CACHE PATH "Path to ONNX Runtime pre-built package")
if(NOT ORT_ROOT AND DEFINED ENV{ONNXRUNTIME_ROOT})
set(ORT_ROOT "$ENV{ONNXRUNTIME_ROOT}")
endif()
# Auto-detect from deps directory (buildall.cmd downloads here)
if(NOT ORT_ROOT)
set(_ORT_DEPS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/deps/onnxruntime")
if(EXISTS "${_ORT_DEPS_DIR}/include/onnxruntime_cxx_api.h")
set(ORT_ROOT "${_ORT_DEPS_DIR}")
message(STATUS "ORT auto-detected at ${ORT_ROOT}")
endif()
endif()
set(SUPERSEP_ENABLED FALSE)
if(HOT_STEP_SUPERSEP AND ORT_ROOT)
if(EXISTS "${ORT_ROOT}/include/onnxruntime_cxx_api.h")
set(SUPERSEP_ENABLED TRUE)
else()
message(WARNING "ORT_ROOT set but onnxruntime_cxx_api.h not found at ${ORT_ROOT}/include")
endif()
endif()
# SuperSep library (STFT + ONNX Runtime for stages 1-4, GGML for the
# BS-Roformer-Leap Xe pair used by SUPERSEP_STABLESTEP)
add_library(supersep STATIC src/supersep.cpp)
target_include_directories(supersep PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}/vendor/pocketfft
${CMAKE_CURRENT_BINARY_DIR}
)
# bs-roformer-ggml.h needs ggml headers/symbols.
target_link_libraries(supersep PUBLIC ggml)
if(MSVC)
target_compile_options(supersep PRIVATE /W4 /wd4100 /wd4505 /wd4244 /wd4267)
else()
target_compile_options(supersep PRIVATE -Wall -Wextra -Wno-unused-parameter -Wno-sign-conversion)
endif()
# SuperSep is now pure GGML (bs-roformer-ggml.h / mdx23c-ggml.h) — no ONNX
# Runtime, so it builds and runs on every backend including Vulkan, Metal and
# plain CPU. The definition is kept because supersep.cpp/.h still guard on it.
target_compile_definitions(supersep PUBLIC HOT_STEP_SUPERSEP)
message(STATUS "SuperSep: ENABLED (native GGML)")
if(CUDAToolkit_FOUND)
target_compile_definitions(supersep PUBLIC GGML_USE_CUDA)
endif()
# NOTE: SUPERSEP_ENABLED below is now a misnomer kept for the ORT plumbing that
# OTHER features still need. SuperSep itself no longer touches ONNX Runtime —
# but sa3-refine.h (StableStep's ONNX backend) and model-store.h's
# vae-ort / cond-enc-ort / text-enc-ort / vae-enc-ort do, so ace-server still
# links ORT and the runtime DLLs are still required by those paths.
# TensorRT Native SDK (DiT TRT acceleration + LoRA refitting)
# ─────────────────────────────────────────────────────────────────────────────
# Platform-specific TRT detection:
# Windows: vendored SDK in engine/deps/tensorrt/ (versioned import libs)
# Linux: system-installed TRT packages via find_path/find_library
# The runtime DLLs/SOs are expected on the library search path at runtime.
set(TRT_ENABLED FALSE)
# --- Windows: vendored SDK in engine/deps/tensorrt/ ---
set(_TRT_DEPS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/deps/tensorrt")
if(WIN32 AND EXISTS "${_TRT_DEPS_DIR}/include/NvInfer.h" AND EXISTS "${_TRT_DEPS_DIR}/lib/nvinfer_10.lib")
set(TRT_ENABLED TRUE)
set(_TRT_INCLUDE_DIR "${_TRT_DEPS_DIR}/include")
set(_TRT_LIB_DIR "${_TRT_DEPS_DIR}/lib")
# Windows vendored SDK ships versioned import libs
set(_TRT_NVINFER_LIB nvinfer_10)
set(_TRT_NVONNXPARSER_LIB nvonnxparser_10)
message(STATUS "[TRT] Found vendored SDK at ${_TRT_DEPS_DIR}")
# --- Linux: system-installed TRT packages ---
elseif(NOT WIN32)
find_path(_TRT_INCLUDE_DIR
NAMES NvInfer.h
PATHS /usr/include/x86_64-linux-gnu /usr/local/include /usr/include
)
find_library(_TRT_NVINFER_LIB
NAMES nvinfer
PATHS /usr/lib/x86_64-linux-gnu /usr/local/lib /usr/lib
)
find_library(_TRT_NVONNXPARSER_LIB
NAMES nvonnxparser
PATHS /usr/lib/x86_64-linux-gnu /usr/local/lib /usr/lib
)
if(_TRT_INCLUDE_DIR AND _TRT_NVINFER_LIB AND _TRT_NVONNXPARSER_LIB)
set(TRT_ENABLED TRUE)
message(STATUS "[TRT] Found system TRT: ${_TRT_NVINFER_LIB}")
else()
message(STATUS "[TRT] Not found (install libnvinfer-dev + libnvonnxparsers-dev to enable)")
endif()
endif()
if(NOT TRT_ENABLED AND WIN32)
message(STATUS "[TRT] Not found (set engine/deps/tensorrt/ to enable DiT TRT)")
endif()
# Core library (shared between binaries)
add_library(acestep-core STATIC
src/request.cpp
src/model-store.cpp
src/pipeline-lm.cpp
src/pipeline-synth.cpp
src/pipeline-synth-ops.cpp
src/pipeline-understand.cpp
)
if(hip_FOUND)
target_link_libraries(acestep-core PUBLIC yyjson lua54 supersep hip::host)
else()
target_link_libraries(acestep-core PUBLIC yyjson lua54 supersep)
endif()
link_ggml_backends(acestep-core)
# ONNX Runtime for the paths that still use it: sa3-refine.h (StableStep's ONNX
# backend) and model-store.h's vae-ort / cond-enc-ort / text-enc-ort /
# vae-enc-ort. This used to arrive transitively from supersep, which is now
# pure GGML — so acestep-core declares it directly.
if(SUPERSEP_ENABLED)
target_compile_definitions(acestep-core PUBLIC HOT_STEP_ORT)
target_include_directories(acestep-core PUBLIC "${ORT_ROOT}/include")
target_link_directories(acestep-core PUBLIC "${ORT_ROOT}/lib")
target_link_libraries(acestep-core PUBLIC onnxruntime)
if(APPLE)
target_link_libraries(acestep-core PUBLIC "-framework CoreML" "-framework Foundation")
endif()
message(STATUS "ONNX Runtime paths: ENABLED (ORT at ${ORT_ROOT})")
else()
message(STATUS "ONNX Runtime paths: DISABLED (StableStep uses its GGML backend)")
endif()
# TRT native linkage for DiT acceleration (shared across platforms)
if(TRT_ENABLED)
target_compile_definitions(acestep-core PUBLIC HOT_STEP_TRT)
target_include_directories(acestep-core PUBLIC "${_TRT_INCLUDE_DIR}")
if(_TRT_LIB_DIR)
target_link_directories(acestep-core PUBLIC "${_TRT_LIB_DIR}")
endif()
target_link_libraries(acestep-core PUBLIC ${_TRT_NVINFER_LIB} ${_TRT_NVONNXPARSER_LIB})
# TRT headers include cuda_runtime_api.h — on Linux g++ needs the CUDA
# include path explicitly (nvcc gets it automatically, but acestep-core
# compiles as plain C++). Windows CUDA Toolkit puts headers on PATH.
if(NOT WIN32)
find_package(CUDAToolkit QUIET)
if(CUDAToolkit_FOUND)
target_include_directories(acestep-core PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
target_link_libraries(acestep-core PUBLIC CUDA::cudart)
message(STATUS "[TRT] CUDA include: ${CUDAToolkit_INCLUDE_DIRS}")
endif()
endif()
message(STATUS "[TRT] DiT TRT acceleration: ENABLED")
endif()
# acestep-core compiles as plain C++ (not nvcc), but pipeline-synth.cpp includes
# <cuda_runtime.h> for VRAM instrumentation whenever GGML_USE_CUDA is defined
# (propagated from ggml). The host compiler needs the CUDA Toolkit include path
# explicitly on EVERY CUDA build — Linux g++ and Windows MSVC (under Ninja) both
# fail to find it otherwise. (The TRT block above also adds this, but only when
# TRT is enabled, so plain CUDA release builds were missing it.)
if(GGML_CUDA)
find_package(CUDAToolkit QUIET)
if(CUDAToolkit_FOUND)
target_include_directories(acestep-core PUBLIC ${CUDAToolkit_INCLUDE_DIRS})
target_link_libraries(acestep-core PUBLIC CUDA::cudart)
message(STATUS "[CUDA] acestep-core include: ${CUDAToolkit_INCLUDE_DIRS}")
endif()
endif()
# ─────────────────────────────────────────────────────────────────────────────
# TRT-LLM Executor (C++ Executor API for LM inference)
# ─────────────────────────────────────────────────────────────────────────────
# Pre-built TRT-LLM SDK (tensorrt_llm.dll + plugin DLL).
# Auto-detect from engine/trtllm-libs/.
# This is SEPARATE from HOT_STEP_TRT (raw NvInfer for DiT). Both can coexist.
#
# STATUS: DISABLED (2026-06-02). Native Windows TRT-LLM is not viable:
# - FMHA/XQA cubin embedding requires GCC inline asm (INCBIN), impossible on MSVC
# - Docker-built engines have Linux platform tags, can't deserialize on Windows
# - TRT version mismatch between Docker (10.14) and Windows SDK (10.16)
# - ONNX-rebuilt engines lack TRT-LLM tensor bindings (kv_cache_block_offsets etc.)
# The code remains intact behind #ifdef HOT_STEP_TRTLLM for future WSL2 or
# cross-platform engine support. To re-enable, set HOT_STEP_TRTLLM_ENABLE=ON.
option(HOT_STEP_TRTLLM_ENABLE "Enable TRT-LLM Executor (currently broken on native Windows)" OFF)
set(TRTLLM_ENABLED FALSE)
set(_TRTLLM_LIBS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/trtllm-libs")
set(_TRTLLM_INC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/trtllm-include")
if(HOT_STEP_TRTLLM_ENABLE AND WIN32
AND EXISTS "${_TRTLLM_LIBS_DIR}/tensorrt_llm.lib"
AND EXISTS "${_TRTLLM_INC_DIR}/tensorrt_llm/executor/executor.h")
set(TRTLLM_ENABLED TRUE)
message(STATUS "[TRT-LLM] Found at ${_TRTLLM_LIBS_DIR}")
target_compile_definitions(acestep-core PUBLIC HOT_STEP_TRTLLM)
target_include_directories(acestep-core PUBLIC "${_TRTLLM_INC_DIR}")
target_link_directories(acestep-core PUBLIC "${_TRTLLM_LIBS_DIR}")
target_link_libraries(acestep-core PUBLIC tensorrt_llm)
# NOTE: nvinfer_plugin_tensorrt_llm is loaded dynamically via LoadLibrary
# in lm-trtllm.h to avoid pulling its dependency chain at process startup.
message(STATUS "[TRT-LLM] LM Executor: ENABLED")
else()
if(HOT_STEP_TRTLLM_ENABLE)
message(STATUS "[TRT-LLM] Not found (set engine/trtllm-libs/ + trtllm-include/ to enable)")
else()
message(STATUS "[TRT-LLM] DISABLED (set -DHOT_STEP_TRTLLM_ENABLE=ON to re-enable)")
endif()
endif()
# ace-synth: full pipeline (text-enc + cond + dit + vae + wav)
add_executable(ace-synth tools/ace-synth.cpp)
target_link_libraries(ace-synth PRIVATE acestep-core)
link_ggml_backends(ace-synth)
# CUDA runtime for TRT DiT path (cudaMalloc, cudaMemcpy, etc.)
if(CUDAToolkit_FOUND)
if(GGML_STATIC)
target_link_libraries(ace-synth PRIVATE CUDA::cudart_static)
else()
target_link_libraries(ace-synth PRIVATE CUDA::cudart)
endif()
endif()
# ace-lm: LLM inference (CoT + audio codes)
add_executable(ace-lm tools/ace-lm.cpp)
target_link_libraries(ace-lm PRIVATE acestep-core)
link_ggml_backends(ace-lm)
# CUDA runtime for TRT LM path (cudaMalloc, cudaMemcpy, etc.)
if(CUDAToolkit_FOUND)
if(GGML_STATIC)
target_link_libraries(ace-lm PRIVATE CUDA::cudart_static)
else()
target_link_libraries(ace-lm PRIVATE CUDA::cudart)
endif()
endif()
# webui: convert tools/webui/public/index.html.gz to a C header for embedding.
# the .gz is committed to git so the C++ build works without npm.
# to update: cd tools/webui && npm install && npm run build, then rebuild ace-server.
set(WEBUI_INPUT "${CMAKE_CURRENT_SOURCE_DIR}/tools/public/index.html.gz")
set(WEBUI_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/index.html.gz.hpp")
add_custom_command(
OUTPUT "${WEBUI_OUTPUT}"
COMMAND "${CMAKE_COMMAND}" "-DINPUT=${WEBUI_INPUT}" "-DOUTPUT=${WEBUI_OUTPUT}" -P "${CMAKE_CURRENT_SOURCE_DIR}/tools/xxd.cmake"
DEPENDS "${WEBUI_INPUT}"
COMMENT "Embedding webui into index.html.gz.hpp"
)
set_source_files_properties(${WEBUI_OUTPUT} PROPERTIES GENERATED TRUE)
# hot-step-server: HOT-Step HTTP server (LM + synth endpoints + embedded webui)
# NOTE: upstream ace-server.cpp is kept as reference but NOT compiled.
# Our server binary is hot-step-server.cpp with extension layer support.
add_executable(ace-server tools/hot-step-server.cpp ${WEBUI_OUTPUT})
target_include_directories(ace-server PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if(hip_FOUND)
target_link_libraries(ace-server PRIVATE acestep-core httplib supersep hip::host)
else()
target_link_libraries(ace-server PRIVATE acestep-core httplib supersep)
endif()
link_ggml_backends(ace-server)
# CUDA runtime for GET /vram (cudaMemGetInfo)
find_package(CUDAToolkit QUIET)
if(CUDAToolkit_FOUND)
target_compile_definitions(ace-server PRIVATE GGML_USE_CUDA)
if(GGML_STATIC)
target_link_libraries(ace-server PRIVATE CUDA::cudart_static)
else()
target_link_libraries(ace-server PRIVATE CUDA::cudart)
endif()
# Copy CUDA runtime DLL to build dir (needed for portable release builds —
# end users don't have the CUDA Toolkit, so cudart64_*.dll must ship with the binary)
if(WIN32)
get_target_property(_CUDART_LOC CUDA::cudart IMPORTED_LOCATION)
if(_CUDART_LOC)
add_custom_command(TARGET ace-server POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_CUDART_LOC}"
"$<TARGET_FILE_DIR:ace-server>"
COMMENT "Copying cudart DLL for portable release"
)
endif()
endif()
# Write CUDA version marker next to ace-server (server reads this to
# select the correct runtime DLLs — cuBLAS/cudart/cuDNN for 12 vs 13)
file(GENERATE OUTPUT "$<TARGET_FILE_DIR:ace-server>/.cuda-version"
CONTENT "${CUDAToolkit_VERSION_MAJOR}")
endif()
# Copy ONNX Runtime DLLs next to ace-server at build time
if(SUPERSEP_ENABLED AND WIN32)
set(_ORT_DLL_DIR "${ORT_ROOT}/lib")
foreach(_dll onnxruntime.dll onnxruntime_providers_shared.dll onnxruntime_providers_cuda.dll onnxruntime_providers_tensorrt.dll)
if(EXISTS "${_ORT_DLL_DIR}/${_dll}")
add_custom_command(TARGET ace-server POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_ORT_DLL_DIR}/${_dll}"
"$<TARGET_FILE_DIR:ace-server>/${_dll}"
COMMENT "Copying ${_dll}"
)
endif()
endforeach()
endif()
# Copy ONNX Runtime dylibs next to ace-server at build time (macOS)
if(SUPERSEP_ENABLED AND APPLE)
set(_ORT_LIB_DIR "${ORT_ROOT}/lib")
file(GLOB _ORT_DYLIBS "${_ORT_LIB_DIR}/libonnxruntime*.dylib")
foreach(_dylib ${_ORT_DYLIBS})
add_custom_command(TARGET ace-server POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_dylib}" "$<TARGET_FILE_DIR:ace-server>"
COMMENT "Copying ${_dylib}"
)
endforeach()
endif()
# Copy ONNX Runtime shared libs next to ace-server at build time (Linux)
if(SUPERSEP_ENABLED AND UNIX AND NOT APPLE)
set(_ORT_LIB_DIR "${ORT_ROOT}/lib")
file(GLOB _ORT_SOLIBS "${_ORT_LIB_DIR}/libonnxruntime*.so*")
foreach(_solib ${_ORT_SOLIBS})
add_custom_command(TARGET ace-server POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_solib}" "$<TARGET_FILE_DIR:ace-server>"
COMMENT "Copying ${_solib}"
)
endforeach()
endif()
add_executable(ace-understand tools/ace-understand.cpp)
target_link_libraries(ace-understand PRIVATE acestep-core)
link_ggml_backends(ace-understand)
# quantize: GGUF requantizer (BF16 -> K-quants)
add_executable(quantize tools/quantize.cpp)
link_ggml_backends(quantize)
# neural-codec: Oobleck VAE neural audio codec (encode/decode WAV <-> latent)
add_executable(neural-codec tools/neural-codec.cpp)
link_ggml_backends(neural-codec)
# sa3-ggml-test: parity tests for the StableStep GGML SA3 modules vs goldens
add_executable(sa3-ggml-test tools/sa3-ggml-test.cpp)
target_link_libraries(sa3-ggml-test PRIVATE yyjson)
link_ggml_backends(sa3-ggml-test)
# bs-roformer-test: parity test for the GGML BS-RoFormer (SuperSep StableStep
# separation) vs PyTorch goldens. See scripts/dump_bs_roformer_goldens.py.
add_executable(bs-roformer-test tools/bs-roformer-test.cpp)
link_ggml_backends(bs-roformer-test)
# mdx23c-test: parity test for the GGML MDX23C drum separator (SuperSep stage 3).
add_executable(mdx23c-test tools/mdx23c-test.cpp)
link_ggml_backends(mdx23c-test)
# ace-midi: MuScriptor audio->MIDI transcription (GGML port, in development —
# docs/plans/muscriptor-cpp-port.md). Standalone tool, no acestep-core needed.
add_executable(ace-midi tools/ace-midi.cpp)
link_ggml_backends(ace-midi)
# ace-train: training toolchain (phase 2: dataset tensor preprocessing —
# docs/plans/2026-07-27-preprocess-implementation.md). Standalone tool: every
# engine module it uses is header-only, so no acestep-core link is needed.
add_executable(ace-train tools/ace-train.cpp)
target_link_libraries(ace-train PRIVATE yyjson)
link_ggml_backends(ace-train)
# mp3-codec: MP3 encoder/decoder (standalone, no ggml needed)
# The mp3/ headers are header-only and usable by ace-synth too via #include "mp3/mp3enc.h"
add_executable(mp3-codec tools/mp3-codec.cpp)
target_include_directories(mp3-codec PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
add_dependencies(mp3-codec version)
if(MSVC)
target_compile_options(mp3-codec PRIVATE /W4 /wd4100 /wd4505)
else()
target_compile_options(mp3-codec PRIVATE -Wall -Wextra -Wconversion
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
target_link_libraries(mp3-codec PRIVATE m)
endif()
target_link_libraries(mp3-codec PRIVATE Threads::Threads)
# mastering: reference-based audio mastering (standalone, no ggml needed)
# Implements the matchering algorithm using pocketfft for FFT.
add_executable(mastering tools/mastering.cpp)
target_include_directories(mastering PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}/vendor/pocketfft
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
)
add_dependencies(mastering version)
if(MSVC)
target_compile_options(mastering PRIVATE /W4 /wd4100 /wd4505 /wd4244 /wd4267)
else()
target_compile_options(mastering PRIVATE -Wall -Wextra -Wconversion
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
target_link_libraries(mastering PRIVATE m)
endif()
target_link_libraries(mastering PRIVATE Threads::Threads)
# ─────────────────────────────────────────────────────────────────────────────
# VST3 Hosting Library + vst-host tool
# ─────────────────────────────────────────────────────────────────────────────
# We compile only the necessary VST3 SDK source files for hosting (loading
# plugins, processing audio, state save/restore). We do NOT use the SDK's
# own CMakeLists.txt because it requires cmake 3.25+ and pulls in the full
# build system including plugin examples and VSTGUI.
set(VST3SDK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/vendor/vst3sdk")
# Static library: vst3-hosting
# Platform-specific hosting sources:
# Windows: module_win32.cpp + threadchecker_win32.cpp
# macOS: module_mac.mm (ARC) + threadchecker_mac.mm
# Linux: module_linux.cpp + threadchecker_linux.cpp
if(APPLE)
set(VST3_PLATFORM_SOURCES
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/module_mac.mm
${VST3SDK_ROOT}/public.sdk/source/common/threadchecker_mac.mm
)
# module_mac.mm requires Objective-C ARC
set_source_files_properties(
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/module_mac.mm
${VST3SDK_ROOT}/public.sdk/source/common/threadchecker_mac.mm
PROPERTIES COMPILE_FLAGS "-fobjc-arc"
)
elseif(WIN32)
set(VST3_PLATFORM_SOURCES
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/module_win32.cpp
${VST3SDK_ROOT}/public.sdk/source/common/threadchecker_win32.cpp
)
else()
set(VST3_PLATFORM_SOURCES
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/module_linux.cpp
${VST3SDK_ROOT}/public.sdk/source/common/threadchecker_linux.cpp
)
endif()
add_library(vst3-hosting STATIC
# Base library
${VST3SDK_ROOT}/base/source/baseiids.cpp
${VST3SDK_ROOT}/base/source/fobject.cpp
${VST3SDK_ROOT}/base/source/fdebug.cpp
${VST3SDK_ROOT}/base/source/fstreamer.cpp
${VST3SDK_ROOT}/base/source/fbuffer.cpp
${VST3SDK_ROOT}/base/source/updatehandler.cpp
${VST3SDK_ROOT}/base/source/timer.cpp
${VST3SDK_ROOT}/base/source/fstring.cpp
# Pluginterfaces
${VST3SDK_ROOT}/pluginterfaces/base/funknown.cpp
${VST3SDK_ROOT}/pluginterfaces/base/ustring.cpp
${VST3SDK_ROOT}/pluginterfaces/base/coreiids.cpp
${VST3SDK_ROOT}/pluginterfaces/base/conststringtable.cpp
# Public SDK hosting (cross-platform + platform-specific)
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/module.cpp
${VST3_PLATFORM_SOURCES}
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/hostclasses.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/plugprovider.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/processdata.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/parameterchanges.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/pluginterfacesupport.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/eventlist.cpp
${VST3SDK_ROOT}/public.sdk/source/vst/hosting/connectionproxy.cpp
# Public SDK common (memory streams for state I/O)
${VST3SDK_ROOT}/public.sdk/source/common/memorystream.cpp
${VST3SDK_ROOT}/public.sdk/source/common/commoniids.cpp
${VST3SDK_ROOT}/public.sdk/source/common/pluginview.cpp
${VST3SDK_ROOT}/public.sdk/source/common/commonstringconvert.cpp
# VST interface ID definitions (all DEF_CLASS_IID symbols)
${VST3SDK_ROOT}/public.sdk/source/vst/vstinitiids.cpp
# VST utilities
${VST3SDK_ROOT}/public.sdk/source/vst/utility/stringconvert.cpp
)
target_include_directories(vst3-hosting PUBLIC
${VST3SDK_ROOT}
${VST3SDK_ROOT}/pluginterfaces
${VST3SDK_ROOT}/public.sdk
)
# The SDK defines DEVELOPMENT=1 for debug builds
target_compile_definitions(vst3-hosting PRIVATE
$<$<CONFIG:Debug>:DEVELOPMENT=1>
$<$<CONFIG:Release>:RELEASE=1>
)
if(MSVC)
# Suppress noisy SDK warnings
target_compile_options(vst3-hosting PRIVATE /W0)
else()
target_compile_options(vst3-hosting PRIVATE -w)
endif()
# macOS: VST3 hosting needs Cocoa + CoreFoundation frameworks
if(APPLE)
find_library(COCOA_FRAMEWORK Cocoa)
find_library(COREFOUNDATION_FRAMEWORK CoreFoundation)
target_link_libraries(vst3-hosting PRIVATE
${COCOA_FRAMEWORK}
${COREFOUNDATION_FRAMEWORK}
)
endif()
# vst-host: standalone VST3 host tool (GUI + offline processing + chain)
add_executable(vst-host tools/vst-host.cpp)
target_include_directories(vst-host PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}
${VST3SDK_ROOT}
)
target_link_libraries(vst-host PRIVATE vst3-hosting yyjson Threads::Threads)
if(MSVC)
target_compile_options(vst-host PRIVATE /W4 /wd4100 /wd4505)
target_compile_definitions(vst-host PRIVATE _CRT_SECURE_NO_WARNINGS NOMINMAX WIN32_LEAN_AND_MEAN)
else()
target_compile_options(vst-host PRIVATE -Wall -Wextra -Wconversion
-Wno-unused-parameter -Wno-unused-function -Wno-sign-conversion)
endif()
add_dependencies(vst-host version)
# macOS: vst-host also needs Apple frameworks
if(APPLE)
target_link_libraries(vst-host PRIVATE ${COCOA_FRAMEWORK} ${COREFOUNDATION_FRAMEWORK})
endif()
engine/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023-2026 The acestep.cpp authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
engine/README.md
+185
View File
@@ -0,0 +1,185 @@
# acestep.cpp
Local AI music generation server with browser UI, powered by GGML.
Describe a song, get stereo 48kHz audio. Runs on CPU, CUDA, Metal, Vulkan.
<img width="1704" height="773" alt="Light" src="https://github.com/user-attachments/assets/aeda150a-46a2-4542-a2d6-57d238a7bbb4" />
<img width="1705" height="771" alt="Dark" src="https://github.com/user-attachments/assets/4941cec9-b6ff-4e09-8905-bdc3ee06d222" />
## Download models
Grab one GGUF of each type from Hugging Face and drop them in the `models/` folder:
https://huggingface.co/Serveurperso/ACE-Step-1.5-GGUF/tree/main
| Type | Pick one | Size |
|------|----------|------|
| LM | acestep-5Hz-lm-4B-Q8_0.gguf | 4.2 GB |
| Text encoder | Qwen3-Embedding-0.6B-Q8_0.gguf | 748 MB |
| DiT | acestep-v15-turbo-Q8_0.gguf | 2.4 GB |
| VAE | vae-BF16.gguf (always this one) | 322 MB |
Three LM sizes available: 0.6B (fast), 1.7B, 4B (best quality).
Multiple DiT variants: turbo (8 steps), sft (50 steps, higher quality), base, shift1, shift3, continuous.
Alternative: `./models.sh` downloads the default set automatically (needs `pip install hf`).
## Build
```
git clone --recurse-submodules https://github.com/ServeurpersoCom/acestep.cpp.git
cd acestep.cpp
```
### Windows
Pre-built binaries (until CI is set up): https://www.serveurperso.com/temp/acestep.cpp-win64/
To build from source, install
[Visual C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
(select "Desktop development with C++" workload) and optionally the
[CUDA Toolkit](https://developer.nvidia.com/cuda-downloads) and/or the
[Vulkan SDK](https://vulkan.lunarg.com/sdk/home).
```cmd
buildcuda.cmd # NVIDIA GPU
buildvulkan.cmd # AMD/Intel GPU (Vulkan)
buildall.cmd # all backends (CUDA + Vulkan + CPU, runtime loading)
```
### Linux / macOS
```bash
./buildcuda.sh # NVIDIA GPU
./buildvulkan.sh # AMD/Intel GPU (Vulkan)
./buildcpu.sh # CPU only (with BLAS)
./buildall.sh # all backends (CUDA + Vulkan + CPU, runtime loading)
```
macOS auto-enables Metal and Accelerate BLAS with any of the above.
## Run
```bash
./server.sh # Linux / macOS
server.cmd # Windows
```
Open http://localhost:8085 in your browser. The WebUI handles everything:
write a caption, set lyrics and metadata, generate, play, and download tracks.
Models are loaded on first request (zero GPU at startup) and swapped
automatically when you pick a different one in the UI.
## Adapters
Drop adapters in the `adapters/` folder and restart the server.
Supports LoRA today in two flavours: PEFT directories (with
`adapter_model.safetensors` + `adapter_config.json`) and ComfyUI single
`.safetensors` files. Select the active adapter from the WebUI.
## Server options
```
--models <dir> Model directory (required)
--adapters <dir> Adapter directory (LoRA today, LoKr soon)
--host <addr> Listen address (default: 127.0.0.1)
--port <N> Listen port (default: 8080)
--max-batch <N> LM batch limit 1-9 (default: 1)
--vae-chunk <N> VAE tile size (default: 256, lower = less VRAM)
--mp3-bitrate <N> MP3 kbps (default: 128)
```
<details>
<summary>API endpoints</summary>
The server exposes three POST endpoints and two GET endpoints:
**POST /lm** - Generate lyrics and audio codes from a caption. Returns JSON.
**POST /synth** - Render audio codes into MP3 or WAV (`?wav=1`).
Accepts JSON or multipart (with source audio for cover/repaint modes).
**POST /understand** - Reverse pipeline: audio in, metadata + lyrics + codes out.
Accepts multipart (audio file) or JSON (codes-only).
**GET /health** - Returns `{"status":"ok"}`.
**GET /props** - Available models, server config, default parameters.
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the full API reference
and AceRequest JSON specification.
</details>
<details>
<summary>CLI tools (advanced)</summary>
For scripting without the server, `ace-lm` and `ace-synth` work as a pipe:
```bash
# LM generates lyrics + codes
./build/ace-lm \
--request /tmp/request.json \
--lm models/acestep-5Hz-lm-4B-Q8_0.gguf
# DiT + VAE render to audio
./build/ace-synth \
--request /tmp/request0.json \
--embedding models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit models/acestep-v15-turbo-Q8_0.gguf \
--vae models/vae-BF16.gguf
```
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the full JSON reference,
task types, batching, and understand pipeline.
</details>
## Technical documentation
[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) covers the complete AceRequest JSON
reference, all task types (text2music, cover, repaint, lego, extract, complete),
FSM constrained decoding, custom GGML operators, quantization, and architecture
internals.
## Community
### ACE-Step official documentation
- [A Musician's Guide](https://github.com/ace-step/ACE-Step-1.5/discussions/235) - non-technical guide for music makers
- [Tutorial](https://github.com/ace-step/ACE-Step-1.5/blob/main/docs/en/Tutorial.md) - design philosophy, model architecture, input control, inference hyperparameters
### Third-party UIs for acestep.cpp
- [acestep-cpp-ui](https://github.com/audiohacking/acestep-cpp-ui)
- [acestep.cpp-simple-GUI](https://github.com/Nurb4000/acestep.cpp-simple-GUI)
- [aceradio](https://github.com/IMbackK/aceradio)
## Samples
https://github.com/user-attachments/assets/9a50c1f4-9ec0-474a-bd14-e8c6b00622a1
https://github.com/user-attachments/assets/fb606249-0269-4153-b651-bf78e05baf22
https://github.com/user-attachments/assets/e0580468-5e33-4a1f-a0f4-b914e4b9a8c2
https://github.com/user-attachments/assets/292a31f1-f97e-4060-9207-ed8364d9a794
https://github.com/user-attachments/assets/34b1b781-a5bc-46c4-90a6-615a10bc2c6a
## Acknowledgements
Independent C++ implementation based on
[ACE-Step 1.5](https://github.com/ace-step/ACE-Step-1.5) by ACE Studio and StepFun.
All model weights are theirs, this is just a native backend.
```bibtex
@misc{gong2026acestep,
title={ACE-Step 1.5: Pushing the Boundaries of Open-Source Music Generation},
author={Junmin Gong, Yulin Song, Wenxiao Zhao, Sen Wang, Shengyuan Xu, Jing Guo},
howpublished={\url{https://github.com/ace-step/ACE-Step-1.5}},
year={2026},
note={GitHub repository}
}
```
engine/UPSTREAM_SYNC
+4
View File
@@ -0,0 +1,4 @@
UPSTREAM_REPO=acestep.cpp
UPSTREAM_COMMIT=9d38f00267d71f8a1d7a6d3e325476472ce09179
SYNC_DATE=2026-07-14
SYNC_NOTES=LM perf overhaul (graph arenas, static decode graph replay, set_rows KV, padded attn windows), snake autofuse + fused CUDA/Vulkan kernels via ggml bump to ServeurpersoCom b677b63c, K/V F16 cast before flash_attn_ext, dead full-attention padding mask removed (incl. hot-step-sampler.h), GGML_CUDA_GRAPHS default ON, cpp-httplib 0.44, WAV PCM24 fixes, locale-immune float parser. Upstream src/solvers/ NOT copied (fork has own).
engine/build-mac.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/bin/bash
# HOT-Step CPP — macOS build script (Apple Silicon + Metal)
#
# Prerequisites:
# - Xcode (full install, not just command line tools — needed for Metal)
# - CMake 3.20+ (brew install cmake)
# - Ninja (brew install ninja) — optional but faster
#
# Usage:
# ./build-mac.sh # Release build with Metal
# ./build-mac.sh Debug # Debug build
#
# ONNX Runtime (SuperSep):
# To enable stem separation, download the macOS ONNX Runtime package:
# brew install onnxruntime
# Or download from: https://github.com/microsoft/onnxruntime/releases
# Place in engine/deps/onnxruntime-osx-arm64/ (or set ORT_ROOT)
set -e
cd "$(dirname "$0")"
BUILD_TYPE="${1:-Release}"
BUILD_DIR="build"
echo "╔══════════════════════════════════════════╗"
echo "║ HOT-Step CPP — macOS Build ║"
echo "║ Apple Silicon + Metal ║"
echo "╚══════════════════════════════════════════╝"
echo ""
echo "Build type: ${BUILD_TYPE}"
mkdir -p "${BUILD_DIR}"
cd "${BUILD_DIR}"
# Detect Ninja
GENERATOR=""
if command -v ninja &>/dev/null; then
GENERATOR="-G Ninja"
echo "Generator: Ninja"
else
echo "Generator: Unix Makefiles (install ninja for faster builds: brew install ninja)"
fi
# Auto-detect ONNX Runtime
ORT_FLAG=""
if [ -n "${ORT_ROOT}" ]; then
echo "ONNX Runtime: ${ORT_ROOT}"
elif [ -d "../deps/onnxruntime-osx-arm64" ]; then
export ORT_ROOT="../deps/onnxruntime-osx-arm64"
echo "ONNX Runtime: auto-detected at ${ORT_ROOT}"
else
echo "ONNX Runtime: not found (SuperSep will be disabled)"
echo " → To enable: brew install onnxruntime, or download manually"
fi
cmake .. \
${GENERATOR} \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-DGGML_METAL=ON \
-DGGML_METAL_EMBED_LIBRARY=ON \
-DGGML_BACKEND_DL=OFF
# Build using all available cores
NCPU=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
cmake --build . --config "${BUILD_TYPE}" -j "${NCPU}"
echo ""
echo "═══════════════════════════════════════════"
echo " Build complete! Binaries in: $(pwd)/"
echo ""
echo " Targets built:"
[ -f ace-server ] && echo " ✓ ace-server"
[ -f mastering ] && echo " ✓ mastering"
[ -f mp3-codec ] && echo " ✓ mp3-codec"
[ -f vst-host ] && echo " ✓ vst-host"
echo ""
echo " Next: cd .. && ./launch.sh"
echo "═══════════════════════════════════════════"
engine/build.cmd
+151
View File
@@ -0,0 +1,151 @@
@echo off
REM HOT-Step engine build (CUDA, native arch only)
REM Compiles ONLY for the local GPU — fast dev builds.
REM
REM Automatically finds Visual Studio / Build Tools via vswhere.
REM Automatically downloads ONNX Runtime GPU SDK for SuperSep support.
REM --- Find vcvars64.bat dynamically ---
REM vswhere ships with VS 2017+ and VS BuildTools.
REM
REM IMPORTANT: %ProgramFiles(x86)% contains parentheses which break
REM batch for-loop parsing. We write the vswhere output to a temp file
REM and read from that instead.
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
if not exist "%VSWHERE%" (
echo ERROR: vswhere.exe not found. Is Visual Studio or Build Tools installed?
exit /b 1
)
set "VCVARS_TMP=%TEMP%\vcvars_path.txt"
"%VSWHERE%" -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "VC\Auxiliary\Build\vcvars64.bat" > "%VCVARS_TMP%" 2>nul
set "VCVARS="
for /f "usebackq tokens=*" %%i in ("%VCVARS_TMP%") do set "VCVARS=%%i"
del "%VCVARS_TMP%" 2>nul
if not defined VCVARS (
echo ERROR: Could not find vcvars64.bat via vswhere.
echo Install the "Desktop development with C++" workload.
exit /b 1
)
REM Skip vcvars if already sourced (prevents PATH overflow on repeated runs)
if defined VSCMD_VER (
echo Using cached VS environment ^(VSCMD_VER=%VSCMD_VER%^)
) else (
echo Using: %VCVARS%
call "%VCVARS%"
)
REM ── ONNX Runtime GPU SDK (for SuperSep stem separation) ────────────
REM Auto-downloads from Microsoft's GitHub Releases if not present.
REM Users can skip this by setting ONNXRUNTIME_ROOT env var.
set "ORT_VERSION=1.25.1"
set "ORT_DIR=%~dp0deps\onnxruntime"
set "ORT_MARKER=%ORT_DIR%\include\onnxruntime_cxx_api.h"
if defined ONNXRUNTIME_ROOT (
echo [ORT] Using ONNXRUNTIME_ROOT=%ONNXRUNTIME_ROOT%
goto :cudnn
)
if exist "%ORT_MARKER%" (
echo [ORT] Found at %ORT_DIR%
goto :cudnn
)
echo.
echo [ORT] ONNX Runtime GPU SDK not found. Downloading v%ORT_VERSION%...
echo [ORT] (one-time download for SuperSep stem separation)
echo.
set "ORT_ZIP=%TEMP%\onnxruntime-win-x64-gpu-%ORT_VERSION%.zip"
set "ORT_URL=https://github.com/microsoft/onnxruntime/releases/download/v%ORT_VERSION%/onnxruntime-win-x64-gpu-%ORT_VERSION%.zip"
echo [ORT] Downloading from %ORT_URL%
curl -L -o "%ORT_ZIP%" "%ORT_URL%"
if errorlevel 1 (
echo [ORT] WARNING: Download failed. Building without SuperSep support.
goto :build
)
echo [ORT] Extracting...
mkdir "%~dp0deps" 2>nul
powershell -NoProfile -Command "Expand-Archive -Path '%ORT_ZIP%' -DestinationPath '%~dp0deps' -Force"
if errorlevel 1 (
echo [ORT] WARNING: Extraction failed. Building without SuperSep support.
goto :build
)
REM Rename extracted folder (it has version in the name)
if exist "%~dp0deps\onnxruntime-win-x64-gpu-%ORT_VERSION%" (
ren "%~dp0deps\onnxruntime-win-x64-gpu-%ORT_VERSION%" onnxruntime
)
del "%ORT_ZIP%" 2>nul
if exist "%ORT_MARKER%" (
echo [ORT] Successfully installed to %ORT_DIR%
) else (
echo [ORT] WARNING: Installation may have failed. Check %ORT_DIR%
)
REM ── cuDNN 9 (required for ONNX Runtime CUDA EP) ────────────────────
:cudnn
REM ORT GPU needs cudnn64_9.dll which isn't bundled. We get it from
REM the nvidia-cudnn-cu12 pip package (no NVIDIA login required).
REM Only the runtime DLLs are needed — copied next to the exe.
set "CUDNN_MARKER=%~dp0build\Release\cudnn64_9.dll"
if exist "%CUDNN_MARKER%" (
echo [cuDNN] Found cudnn64_9.dll
goto :build
)
echo.
echo [cuDNN] cudnn64_9.dll not found. Installing via pip...
echo [cuDNN] (one-time download for CUDA-accelerated SuperSep)
echo.
python -m pip install --quiet nvidia-cudnn-cu12 2>nul
if errorlevel 1 (
echo [cuDNN] WARNING: pip install failed. CUDA EP will be disabled.
echo [cuDNN] To fix: pip install nvidia-cudnn-cu12
goto :build
)
REM Find the installed DLLs and copy them to build/Release
for /f "tokens=*" %%d in ('python -c "import nvidia.cudnn; import os; print(os.path.join(nvidia.cudnn.__path__[0], 'bin'))" 2^>nul') do (
if exist "%%d\cudnn64_9.dll" (
echo [cuDNN] Copying DLLs from %%d
mkdir "%~dp0build\Release" 2>nul
copy /y "%%d\cudnn*.dll" "%~dp0build\Release\" >nul 2>nul
echo [cuDNN] Done
) else (
echo [cuDNN] WARNING: Could not find cudnn64_9.dll in pip package
echo [cuDNN] path checked: %%d
)
)
:build
cd /d "%~dp0"
mkdir build 2>nul
cd build
REM Only run cmake configure if not yet configured (avoids invalidating incremental builds)
if not exist "CMakeCache.txt" (
REM HOT_STEP_CMAKE_FLAGS can be set by update.bat for auto-detected backends.
REM When unset, defaults to CUDA-only (native dev build).
if defined HOT_STEP_CMAKE_FLAGS (
cmake .. %HOT_STEP_CMAKE_FLAGS% -DGGML_CPU_ALL_VARIANTS=ON -DGGML_BACKEND_DL=ON
) else (
cmake .. -DGGML_CUDA=ON -DGGML_CUDA_GRAPHS=ON -DCMAKE_CUDA_ARCHITECTURES="75;80;86;89;90;120a" -DGGML_NATIVE=OFF -DGGML_CPU_ALL_VARIANTS=ON -DGGML_BACKEND_DL=ON
)
)
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
cd ..
engine/buildall.cmd
+100
View File
@@ -0,0 +1,100 @@
@echo off
REM HOT-Step engine build (all backends: CPU variants + CUDA + Vulkan)
REM Uses vswhere to find any Visual Studio edition automatically.
REM Automatically downloads ONNX Runtime GPU SDK for SuperSep support.
REM Skip vcvars if already sourced (prevents PATH overflow on repeated runs)
if defined VSCMD_VER goto :deps
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
if not exist "%VSWHERE%" (
echo ERROR: vswhere.exe not found. Is Visual Studio or Build Tools installed?
exit /b 1
)
set "VCVARS_TMP=%TEMP%\vcvars_path.txt"
"%VSWHERE%" -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "VC\Auxiliary\Build\vcvars64.bat" > "%VCVARS_TMP%" 2>nul
set "VCVARS="
for /f "usebackq tokens=*" %%i in ("%VCVARS_TMP%") do set "VCVARS=%%i"
del "%VCVARS_TMP%" 2>nul
if not defined VCVARS (
echo ERROR: Could not find vcvars64.bat via vswhere.
echo Install the "Desktop development with C++" workload.
exit /b 1
)
echo Using: %VCVARS%
call "%VCVARS%"
:deps
REM ── ONNX Runtime GPU SDK (for SuperSep stem separation) ────────────
REM Auto-downloads from Microsoft's GitHub Releases if not present.
REM Users can skip this by setting ORT_ROOT or ONNXRUNTIME_ROOT env var.
set "ORT_VERSION=1.25.1"
set "ORT_DIR=%~dp0deps\onnxruntime"
set "ORT_MARKER=%ORT_DIR%\include\onnxruntime_cxx_api.h"
if defined ONNXRUNTIME_ROOT (
echo [ORT] Using ONNXRUNTIME_ROOT=%ONNXRUNTIME_ROOT%
goto :build
)
if exist "%ORT_MARKER%" (
echo [ORT] Found at %ORT_DIR%
goto :build
)
echo.
echo ══════════════════════════════════════════════════════════════════
echo ONNX Runtime GPU SDK not found. Downloading v%ORT_VERSION%...
echo This is needed for SuperSep stem separation (one-time download).
echo ══════════════════════════════════════════════════════════════════
echo.
set "ORT_ZIP=%TEMP%\onnxruntime-win-x64-gpu-%ORT_VERSION%.zip"
set "ORT_URL=https://github.com/microsoft/onnxruntime/releases/download/v%ORT_VERSION%/onnxruntime-win-x64-gpu-%ORT_VERSION%.zip"
echo [ORT] Downloading from %ORT_URL%
curl -L -o "%ORT_ZIP%" "%ORT_URL%"
if errorlevel 1 (
echo [ORT] WARNING: Download failed. Building without SuperSep support.
echo [ORT] You can manually download and extract to: %ORT_DIR%
goto :build
)
echo [ORT] Extracting...
mkdir "%~dp0deps" 2>nul
powershell -NoProfile -Command "Expand-Archive -Path '%ORT_ZIP%' -DestinationPath '%~dp0deps' -Force"
if errorlevel 1 (
echo [ORT] WARNING: Extraction failed. Building without SuperSep support.
goto :build
)
REM Rename extracted folder (it has version in the name)
if exist "%~dp0deps\onnxruntime-win-x64-gpu-%ORT_VERSION%" (
ren "%~dp0deps\onnxruntime-win-x64-gpu-%ORT_VERSION%" onnxruntime
)
REM Clean up zip
del "%ORT_ZIP%" 2>nul
if exist "%ORT_MARKER%" (
echo [ORT] Successfully installed to %ORT_DIR%
) else (
echo [ORT] WARNING: Installation may have failed. Check %ORT_DIR%
)
:build
cd /d "%~dp0"
rem rd /s /q build 2>nul
mkdir build 2>nul
cd build
REM Build with ORT auto-detection from deps directory
cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON %RELEASE_CMAKE_EXTRA%
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
cd ..
engine/buildall.sh
+10
View File
@@ -0,0 +1,10 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
export PATH=/usr/local/cuda/bin:$PATH
cmake .. -DGGML_CPU_ALL_VARIANTS=ON -DGGML_CUDA=ON -DGGML_VULKAN=ON -DGGML_BACKEND_DL=ON
cmake --build . --config Release -j "$(nproc)"
engine/buildcpu.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
cmake .. -DGGML_BLAS=ON
cmake --build . --config Release -j "$(nproc)"
engine/buildcuda.cmd
+39
View File
@@ -0,0 +1,39 @@
@echo off
REM HOT-Step engine build (CUDA only)
REM Uses vswhere to find any Visual Studio edition automatically.
REM Skip vcvars if already sourced (prevents PATH overflow on repeated runs)
if defined VSCMD_VER goto :build
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
if not exist "%VSWHERE%" (
echo ERROR: vswhere.exe not found. Is Visual Studio or Build Tools installed?
exit /b 1
)
set "VCVARS_TMP=%TEMP%\vcvars_path.txt"
"%VSWHERE%" -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "VC\Auxiliary\Build\vcvars64.bat" > "%VCVARS_TMP%" 2>nul
set "VCVARS="
for /f "usebackq tokens=*" %%i in ("%VCVARS_TMP%") do set "VCVARS=%%i"
del "%VCVARS_TMP%" 2>nul
if not defined VCVARS (
echo ERROR: Could not find vcvars64.bat via vswhere.
echo Install the "Desktop development with C++" workload.
exit /b 1
)
echo Using: %VCVARS%
call "%VCVARS%"
:build
cd /d "%~dp0"
rem rd /s /q build 2>nul
mkdir build 2>nul
cd build
cmake .. -DGGML_CUDA=ON
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
cd ..
engine/buildcuda.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
cmake .. -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc
cmake --build . --config Release -j "$(nproc)"
engine/buildhip.sh
Executable
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
if command -v rocminfo; then export GFX_NAME=$(rocminfo | awk '/ *Name: +gfx[1-9]/ {print $2; exit}'); else echo "rocminfo missing!"; fi
if [ -z "${GFX_NAME}" ]; then
echo "Warn: Couldn't detect AMD GPU for HIP! Using fallback value (gfx1030).";
GFX_NAME="gfx1030";
else
echo "Building for GPU arch: ${GFX_NAME}";
fi
cmake .. -DGGML_HIP=ON -DGPU_TARGETS=$GFX_NAME -DCMAKE_BUILD_TYPE=Release
cmake --build . --config Release -j `nproc --ignore=1`
engine/buildtermux.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
cmake .. -DGGML_BLAS=ON -DBLAS_INCLUDE_DIRS=$PREFIX/include/openblas
cmake --build . --config Release -j "$(nproc)"
engine/buildvulkan.cmd
+39
View File
@@ -0,0 +1,39 @@
@echo off
REM HOT-Step engine build (Vulkan only)
REM Uses vswhere to find any Visual Studio edition automatically.
REM Skip vcvars if already sourced (prevents PATH overflow on repeated runs)
if defined VSCMD_VER goto :build
set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
if not exist "%VSWHERE%" (
echo ERROR: vswhere.exe not found. Is Visual Studio or Build Tools installed?
exit /b 1
)
set "VCVARS_TMP=%TEMP%\vcvars_path.txt"
"%VSWHERE%" -latest -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -find "VC\Auxiliary\Build\vcvars64.bat" > "%VCVARS_TMP%" 2>nul
set "VCVARS="
for /f "usebackq tokens=*" %%i in ("%VCVARS_TMP%") do set "VCVARS=%%i"
del "%VCVARS_TMP%" 2>nul
if not defined VCVARS (
echo ERROR: Could not find vcvars64.bat via vswhere.
echo Install the "Desktop development with C++" workload.
exit /b 1
)
echo Using: %VCVARS%
call "%VCVARS%"
:build
cd /d "%~dp0"
rem rd /s /q build 2>nul
mkdir build 2>nul
cd build
cmake .. -DGGML_VULKAN=ON
cmake --build . --config Release -j %NUMBER_OF_PROCESSORS%
cd ..
engine/buildvulkan.sh
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
rm -rf build
mkdir build
cd build
cmake .. -DGGML_VULKAN=ON
cmake --build . --config Release -j "$(nproc)"
engine/buildwebui.sh
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
cd tools/webui
rm -rf node_modules
npm install
npm run format
npm run lint
npm run check
npm run build
engine/checkpoints.sh
+63
View File
@@ -0,0 +1,63 @@
#!/bin/bash
# Download ACE-Step checkpoints from HuggingFace
# Usage: ./checkpoints.sh [--all]
# default: Qwen3-Embedding-0.6B + acestep-5Hz-lm-4B + acestep-v15-turbo + vae
# --all: + all LM variants + all DiT variants (incl. XL 4B) from ACE-Step registry
set -eu
DIR="checkpoints"
mkdir -p "$DIR"
HF="hf download --quiet"
MAIN="ACE-Step/Ace-Step1.5"
dl_main() {
local name="$1"
local target="$DIR/$name"
if [ -d "$target" ] && [ "$(ls "$target"/*.safetensors "$target"/*.bin 2>/dev/null | wc -l)" -gt 0 ]; then
echo "[OK] $name"
return
fi
echo "[Download] $name <- $MAIN"
$HF "$MAIN" --include "$name/*" --local-dir "$DIR"
}
dl_repo() {
local name="$1" repo="$2"
local target="$DIR/$name"
if [ -d "$target" ] && [ "$(ls "$target"/*.safetensors "$target"/*.bin 2>/dev/null | wc -l)" -gt 0 ]; then
echo "[OK] $name"
return
fi
echo "[Download] $name <- $repo"
$HF "$repo" --local-dir "$target"
}
# Core (required)
dl_main "Qwen3-Embedding-0.6B"
dl_repo "acestep-5Hz-lm-4B" "ACE-Step/acestep-5Hz-lm-4B"
dl_main "acestep-v15-turbo"
dl_main "vae"
# Every model from ACE-Step registry
if [ "${1:-}" = "--all" ]; then
# LM variants (from main repo)
dl_main "acestep-5Hz-lm-1.7B"
# LM variants (separate repos)
dl_repo "acestep-5Hz-lm-0.6B" "ACE-Step/acestep-5Hz-lm-0.6B"
# DiT variants (separate repos)
dl_repo "acestep-v15-turbo-shift3" "ACE-Step/acestep-v15-turbo-shift3"
dl_repo "acestep-v15-turbo-shift1" "ACE-Step/acestep-v15-turbo-shift1"
dl_repo "acestep-v15-turbo-continuous" "ACE-Step/acestep-v15-turbo-continuous"
dl_repo "acestep-v15-sft" "ACE-Step/acestep-v15-sft"
dl_repo "acestep-v15-base" "ACE-Step/acestep-v15-base"
# XL (4B DiT) variants
dl_repo "acestep-v15-xl-turbo" "ACE-Step/acestep-v15-xl-turbo"
dl_repo "acestep-v15-xl-sft" "ACE-Step/acestep-v15-xl-sft"
dl_repo "acestep-v15-xl-base" "ACE-Step/acestep-v15-xl-base"
fi
find "$DIR" -name '.cache' -type d -exec rm -rf {} + 2>/dev/null
echo "[Done] Checkpoints ready in $DIR"
echo "[Done] Run: python3 convert.py"
engine/convert-comfy-int8.py
+258
View File
@@ -0,0 +1,258 @@
#!/usr/bin/env python3
# convert-comfy-int8.py: ComfyUI int8 (comfy_quant) DiT safetensors -> Q8_0 GGUF
#
# ComfyUI int8 checkpoints (convert_to_quant output) store each quantized
# linear as:
# <base>.weight I8 (out, in) -- quantized weight
# <base>.weight_scale F32 scalar or (out,) -- per-tensor / per-row scale
# <base>.comfy_quant U8 json blob -- {"format": "int8_tensorwise",
# "convrot": true?,
# "convrot_groupsize": G?, ...}
# Dequant is w = int8 * scale. Both per-tensor and per-row int8 grids are
# exactly representable in GGUF Q8_0 (fp16 scale + 32x int8 per block; every
# block scale in row r = that row's scale), so we repack bit-faithfully
# instead of dequantizing + requantizing.
#
# ConvRot (--convrot): weights are stored PRE-ROTATED (W' = W @ H_block^T per
# input-dim group, H = regular Hadamard, power-of-4 group size). Handling:
# - decoder.* weights keep the rotation; the layer is recorded in the GGUF
# KV "acestep.convrot_map" ("name:group;...") and the engine applies the
# matching activation rotation at inference (see engine/src/dit-graph.h).
# - all other components (encoder/tokenizer/detokenizer) are dequantized and
# UNROTATED offline to BF16 — they run once per generation, so keeping
# them int8 isn't worth wiring rotation through their graph builders.
#
# Non-quantized tensors follow convert.py conventions (F32 -> BF16 truncate).
# The DiT GGUF must also carry silence_latent + the acestep.* config KVs,
# which ComfyUI files lack -- both are copied from a donor GGUF of the same
# architecture (any convert.py-produced acestep-v15-*.gguf that matches).
#
# Usage:
# python convert-comfy-int8.py <comfy.safetensors> <donor.gguf> <out.gguf> [--name NAME]
import argparse
import json
import os
import sys
import numpy as np
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)
import gguf # noqa: E402
import convert # noqa: E402 (engine/convert.py: read_sf_header, add_metadata)
Q8_0 = gguf.GGMLQuantizationType.Q8_0
BF16 = gguf.GGMLQuantizationType.BF16
QK8_0 = 32 # ggml Q8_0 block size
def log(msg):
print("[COMFY-INT8] %s" % msg, file=sys.stderr, flush=True)
def f32_to_bf16(arr_f32):
"""Truncate F32 to BF16 (as uint16), matching convert.py."""
a = np.ascontiguousarray(arr_f32, dtype=np.float32)
return (a.view(np.uint32) >> 16).astype(np.uint16)
def pack_q8_0(qs, row_scales):
"""Pack int8 weights (rows, cols) + per-row f32 scales into raw Q8_0 blocks.
Q8_0 block layout: 2-byte fp16 scale followed by 32 int8 values.
Returns a uint8 array of shape (rows, cols//32 * 34).
"""
rows, cols = qs.shape
nb = cols // QK8_0
d = np.repeat(row_scales.astype(np.float16).reshape(rows, 1, 1), nb, axis=1)
blocks = np.concatenate([d.view(np.uint8), qs.reshape(rows, nb, QK8_0).view(np.uint8)], axis=2)
return np.ascontiguousarray(blocks.reshape(rows, nb * (QK8_0 + 2)))
def build_hadamard(size):
"""Regular Hadamard (H4 Kronecker powers, normalized). Power-of-4 sizes.
Mirrors convert_to_quant utils/convrot.py build_hadamard(); symmetric.
"""
import math
if size < 4 or (size & (size - 1)) != 0 or (math.log(size, 4) % 1 != 0):
raise ValueError("unsupported Hadamard size %d (power of 4 only)" % size)
H4 = np.array([[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype=np.float64)
H = H4
while H.shape[0] < size:
H = np.kron(H, H4)
return H / math.sqrt(size)
def unrotate_weight(w, group_size):
"""Undo the offline ConvRot rotation: W = W_rot @ H per input-dim group
(H symmetric orthogonal, so H^T == H and W_rot @ H recovers W)."""
out_f, in_f = w.shape
H = build_hadamard(group_size)
wg = w.reshape(out_f, in_f // group_size, group_size)
return (wg @ H).reshape(out_f, in_f).astype(np.float32)
def read_donor(donor_path):
"""Return (config dict, donor name, silence_latent f32 array, {name: shape})."""
r = gguf.GGUFReader(donor_path)
fields = {f.name: f for f in r.fields.values()}
def get_str(key):
f = fields.get(key)
return bytes(f.parts[f.data[0]]).decode() if f else None
cfg_json = get_str("acestep.config_json")
if not cfg_json:
log("FATAL: donor %s has no acestep.config_json (not a convert.py DiT GGUF?)" % donor_path)
sys.exit(1)
silence = None
shapes = {}
for t in r.tensors:
shapes[t.name] = tuple(reversed([int(d) for d in t.shape])) # ne order -> torch order
if t.name == "silence_latent":
silence = np.asarray(t.data).flatten().reshape(15000, 64).astype(np.float32)
if silence is None:
log("FATAL: donor has no silence_latent tensor")
sys.exit(1)
return json.loads(cfg_json), get_str("general.name"), silence, shapes
def main():
ap = argparse.ArgumentParser()
ap.add_argument("comfy_st")
ap.add_argument("donor_gguf")
ap.add_argument("out_gguf")
ap.add_argument("--name", default=None, help="general.name for the output (default: derived from output filename)")
ap.add_argument("--no-runtime-rotation", action="store_true",
help="Dequantize + unrotate EVERYTHING to BF16 (no acestep.convrot_map, no engine "
"rotation needed). Numerically equivalent reference build — 2x the size; used "
"to A/B-validate the engine's runtime rotation path.")
args = ap.parse_args()
name = args.name or os.path.basename(args.out_gguf).rsplit(".", 1)[0]
cfg, donor_name, silence, donor_shapes = read_donor(args.donor_gguf)
log("donor: %s (%d tensors)" % (donor_name, len(donor_shapes)))
meta, hdr_size = convert.read_sf_header(args.comfy_st)
w = gguf.GGUFWriter(args.out_gguf, "acestep-dit", use_temp_file=True)
w.add_name(name)
convert.add_metadata(w, cfg, "dit")
n_q8, n_bf16, n_dequant, total = 0, 0, 0, 0
shape_mismatches = []
convrot_map = [] # (tensor name, group size) kept rotated in the GGUF
with open(args.comfy_st, "rb") as f:
def read_tensor(tname):
info = meta[tname]
f.seek(hdr_size + info["data_offsets"][0])
return f.read(info["data_offsets"][1] - info["data_offsets"][0]), info
def layer_quant_config(base):
key = base + ".comfy_quant"
if key not in meta:
return {}
raw, _ = read_tensor(key)
try:
return json.loads(raw.decode())
except Exception:
return {}
for tname in sorted(meta.keys()):
if tname.endswith(".weight_scale") or tname.endswith(".comfy_quant"):
continue
info = meta[tname]
dtype, shape = info["dtype"], info["shape"]
donor_shape = donor_shapes.get(tname)
if donor_shape is None:
log(" WARNING: %s not in donor GGUF -- writing anyway" % tname)
elif donor_shape != tuple(shape):
shape_mismatches.append((tname, tuple(shape), donor_shape))
raw, _ = read_tensor(tname)
if dtype == "I8":
base = tname[: -len(".weight")]
sraw, sinfo = read_tensor(base + ".weight_scale")
scales = np.frombuffer(sraw, dtype=np.float32)
qcfg = layer_quant_config(base)
rot_group = int(qcfg.get("convrot_groupsize", 0)) if qcfg.get("convrot") else 0
qs = np.frombuffer(raw, dtype=np.int8).reshape(-1, shape[-1])
rows, cols = qs.shape
# scalar scale -> broadcast per-row; per-row scale as-is
if scales.size == 1:
row_scales = np.full(rows, scales[0], dtype=np.float32)
elif scales.size == rows:
row_scales = scales.reshape(rows)
else:
log(" FATAL: %s weight_scale has %d entries for %d rows" % (tname, scales.size, rows))
sys.exit(1)
keep_rotation = rot_group > 0 and tname.startswith("decoder.") and not args.no_runtime_rotation
packable = len(shape) == 2 and cols % QK8_0 == 0 and not (rot_group > 0 and args.no_runtime_rotation)
if packable and (rot_group == 0 or keep_rotation):
packed = pack_q8_0(qs, row_scales)
w.add_tensor(tname, packed, raw_dtype=Q8_0)
if keep_rotation:
convrot_map.append((tname, rot_group))
n_q8 += 1
total += packed.nbytes
else:
# dequant fallback; undo rotation so no runtime support needed
deq = qs.astype(np.float32) * row_scales[:, None]
if rot_group > 0:
deq = unrotate_weight(deq, rot_group)
w.add_tensor(tname, f32_to_bf16(deq.reshape(shape)), raw_dtype=BF16)
n_dequant += 1
total += deq.size * 2
elif dtype == "BF16":
arr = np.frombuffer(raw, dtype=np.uint16).reshape(shape)
w.add_tensor(tname, arr, raw_dtype=BF16)
n_bf16 += 1
total += arr.nbytes
elif dtype == "F32":
arr = f32_to_bf16(np.frombuffer(raw, dtype=np.float32)).reshape(shape)
w.add_tensor(tname, arr, raw_dtype=BF16)
n_bf16 += 1
total += arr.nbytes
else:
log(" FATAL: %s has unsupported dtype %s" % (tname, dtype))
sys.exit(1)
w.add_tensor("silence_latent", silence)
total += silence.nbytes
if shape_mismatches:
for tname, got, want in shape_mismatches:
log(" FATAL: shape mismatch %s: comfy %s vs donor %s" % (tname, got, want))
sys.exit(1)
if convrot_map:
w.add_string("acestep.convrot_map", ";".join("%s:%d" % (n, g) for n, g in convrot_map))
groups = sorted(set(g for _, g in convrot_map))
log("convrot: %d rotated decoder weights kept (group sizes %s) -> acestep.convrot_map"
% (len(convrot_map), groups))
log("tensors: %d Q8_0, %d BF16, %d dequant-fallback, + silence_latent (%.2f GB)"
% (n_q8, n_bf16, n_dequant, total / (1 << 30)))
w.write_header_to_file()
w.write_kv_data_to_file()
w.write_tensors_to_file(progress=True)
w.close()
log("wrote %.0f MB -> %s" % (os.path.getsize(args.out_gguf) / (1 << 20), args.out_gguf))
if __name__ == "__main__":
main()
engine/convert-sa3.py
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
# convert-sa3.py: Stable Audio 3 checkpoints -> GGUF for the StableStep GGML backend.
#
# Produces four GGUFs into models/:
# sa3-dit-BF16.gguf arch "sa3-dit" from stable-audio-3-medium (model.* keys)
# sa3-same-enc-F16.gguf arch "sa3-same-enc" from pretransform.model.* (encoder side)
# sa3-same-dec-F16.gguf arch "sa3-same-dec" from pretransform.model.* (decoder side)
# sa3-text-enc-BF16.gguf arch "sa3-t5gemma" from the t5gemma-b-b-ul2 subfolder
#
# The SAME pair is stored F16 (not BF16): the decoder's sinusoidal FF blocks
# amplify per-weight rounding noise across its 12 layers (see write_sa3_gguf).
#
# Tensor policy: >=2D weights -> BF16; 1D tensors (norms, biases, scales) -> F32
# (precision finding from the ONNX leg: this model's norm/timestep paths are
# fp32-sensitive — measured cosine 0.966 with blanket fp16 vs 0.9995 scoped).
# Tensor names are the source names minus the strip prefix; the C++ graph
# builders consume them as-is. The full model_config.json is embedded verbatim
# under metadata key "sa3.config_json" (the C++ side parses what it needs).
#
# Runs in the StableAudio3 uv venv:
# cd d:/Ace-Step-Latest/StableAudio3
# uv run --with gguf python d:/Ace-Step-Latest/hot-step-cpp/engine/convert-sa3.py
import json
import os
import sys
import numpy as np
import gguf
sys.path.insert(0, r"d:/Ace-Step-Latest/StableAudio3")
from huggingface_hub import hf_hub_download
from safetensors import safe_open
OUTPUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models")
REPO = "stabilityai/stable-audio-3-medium"
def log(msg):
print(f"[convert-sa3] {msg}", file=sys.stderr, flush=True)
def to_np(t):
import torch
if t.dtype == torch.bfloat16:
return t.float().numpy()
return t.numpy()
def write_sa3_gguf(out_path, arch, tensors, config_json, extra_meta=None, half="bf16"):
"""tensors: list of (name, np.float32 array). >=2D stored in `half`
("bf16" or "f16"), 1D tensors (norms, biases, scales) stored F32.
half="f16" is used for the SAME autoencoder halves: the decoder's
sinusoidal FF layers amplify weight rounding noise layer over layer
(bf16 ~0.4% rel error -> parity cosine 0.9987 < 0.999; f16 ~0.05%
passes). Same file size either way; weight magnitudes are far inside
f16 range."""
w = gguf.GGUFWriter(out_path, arch)
w.add_string("sa3.config_json", config_json)
for k, v in (extra_meta or {}).items():
w.add_string(k, v)
import torch
n_half = n_f32 = 0
for name, arr in tensors:
arr = np.ascontiguousarray(arr, dtype=np.float32)
if arr.size == 0:
# e.g. bottleneck.noise_scaling_factor (1, 0, 1) when
# noise_augment_dim == 0. ggml's gguf reader hits an integer
# divide-by-zero on ne==0 tensors, and the C++ side never reads
# them — drop.
log(f" skipping zero-element tensor {name} {arr.shape}")
continue
if arr.ndim >= 2:
# raw_dtype does NOT convert — it labels. Convert to 16-bit bytes
# explicitly (uint16 view keeps the logical shape).
if half == "f16":
h = torch.from_numpy(arr).to(torch.float16).view(torch.uint16).numpy()
w.add_tensor(name, h, raw_dtype=gguf.GGMLQuantizationType.F16)
else:
h = torch.from_numpy(arr).to(torch.bfloat16).view(torch.uint16).numpy()
w.add_tensor(name, h, raw_dtype=gguf.GGMLQuantizationType.BF16)
n_half += 1
else:
w.add_tensor(name, arr) # F32
n_f32 += 1
w.write_header_to_file()
w.write_kv_data_to_file()
w.write_tensors_to_file()
w.close()
size = os.path.getsize(out_path) / 1e9
log(f"{os.path.basename(out_path)}: {n_half} {half.upper()} + {n_f32} F32 tensors, {size:.2f} GB")
def main():
# Local-checkpoint support (e.g. LoRA-merged models): --ckpt/--config
# override the HF download; --out-dir redirects output; --dit-only skips
# the SAME + T5Gemma GGUFs (unchanged when only the DiT was fine-tuned).
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--ckpt", default=None, help="Local model.safetensors (default: HF download)")
ap.add_argument("--config", default=None, help="Local model_config.json (default: HF download)")
ap.add_argument("--out-dir", default=OUTPUT_DIR)
ap.add_argument("--dit-only", action="store_true")
args = ap.parse_args()
out_dir = args.out_dir
os.makedirs(out_dir, exist_ok=True)
cfg_path = args.config or hf_hub_download(REPO, "model_config.json")
ckpt_path = args.ckpt or hf_hub_download(REPO, "model.safetensors")
with open(cfg_path) as f:
config_json = f.read()
# ── Split the combined checkpoint by prefix ─────────────────────────
dit_tensors, enc_tensors, dec_tensors = [], [], []
with safe_open(ckpt_path, framework="pt", device="cpu") as f:
for key in f.keys():
if key.startswith("model."):
dit_tensors.append((key[len("model."):], to_np(f.get_tensor(key))))
elif key.startswith("pretransform.model."):
sub = key[len("pretransform.model."):]
# AudioAutoencoder members: encoder.*, decoder.*, bottleneck.*,
# pretransform.* (patched — no weights). Bottleneck params go to BOTH
# (encoder applies scale/bias+running_std, decoder inverts).
if sub.startswith("encoder."):
enc_tensors.append((sub, to_np(f.get_tensor(key))))
elif sub.startswith("decoder."):
dec_tensors.append((sub, to_np(f.get_tensor(key))))
elif sub.startswith("bottleneck."):
t = to_np(f.get_tensor(key))
enc_tensors.append((sub, t))
dec_tensors.append((sub, t))
# conditioner.* (learned padding, seconds embedder) rides with the DiT
# gguf — small and needed by the same backend module.
elif key.startswith("conditioner."):
dit_tensors.append((key, to_np(f.get_tensor(key))))
write_sa3_gguf(os.path.join(out_dir, "sa3-dit-BF16.gguf"),
"sa3-dit", dit_tensors, config_json)
if args.dit_only:
log("Done (dit-only).")
return
write_sa3_gguf(os.path.join(out_dir, "sa3-same-enc-F16.gguf"),
"sa3-same-enc", enc_tensors, config_json, half="f16")
write_sa3_gguf(os.path.join(out_dir, "sa3-same-dec-F16.gguf"),
"sa3-same-dec", dec_tensors, config_json, half="f16")
# ── T5Gemma encoder (separate HF model in the repo subfolder) ───────
t5_cfg = hf_hub_download(REPO, "config.json", subfolder="t5gemma-b-b-ul2")
t5_ckpt = hf_hub_download(REPO, "model.safetensors", subfolder="t5gemma-b-b-ul2")
with open(t5_cfg) as f:
t5_config_json = f.read()
t5_tensors = []
with safe_open(t5_ckpt, framework="pt", device="cpu") as f:
for key in f.keys():
# Encoder-only: drop the decoder half (never used by SA3)
if key.startswith("decoder."):
continue
t5_tensors.append((key, to_np(f.get_tensor(key))))
# The SA3 conditioner's learned padding embedding is applied to the text
# encoder's output (padded positions replaced) — it belongs to this module,
# so duplicate it here (it also rides in the DiT gguf with the rest of
# conditioner.*).
with safe_open(ckpt_path, framework="pt", device="cpu") as f:
key = "conditioner.conditioners.prompt.padding_embedding"
t5_tensors.append((key, to_np(f.get_tensor(key))))
write_sa3_gguf(os.path.join(out_dir, "sa3-text-enc-BF16.gguf"),
"sa3-t5gemma", t5_tensors, t5_config_json,
extra_meta={"sa3.parent_config_json": config_json})
log("Done.")
if __name__ == "__main__":
main()
engine/convert.py
+304
View File
@@ -0,0 +1,304 @@
#!/usr/bin/env python3
# convert.py: safetensors to GGUF for ACE-Step (LM, DiT, TextEncoder, VAE)
# Reads from checkpoints/, writes GGUF to models/
# Each GGUF is self-contained: weights + config + tokenizer + silence_latent
import os
import sys
import json
import struct
import zipfile
import numpy as np
import gguf
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CHECKPOINT_DIR = os.path.join(SCRIPT_DIR, "checkpoints")
OUTPUT_DIR = os.path.join(SCRIPT_DIR, "models")
BF16 = gguf.GGMLQuantizationType.BF16
def log(tag, msg):
print("[%s] %s" % (tag, msg), file=sys.stderr, flush=True)
# Safetensors reader
def read_sf_header(path):
with open(path, "rb") as f:
n = struct.unpack("<Q", f.read(8))[0]
meta = json.loads(f.read(n))
meta.pop("__metadata__", None)
return meta, 8 + n
def find_sf_files(model_dir):
"""Return list of safetensors paths (single, sharded, or diffusers VAE)."""
single = os.path.join(model_dir, "model.safetensors")
if os.path.exists(single):
return [single]
index = os.path.join(model_dir, "model.safetensors.index.json")
if os.path.exists(index):
with open(index, "r", encoding="utf-8") as f:
idx = json.load(f)
shards = sorted(set(idx["weight_map"].values()))
return [os.path.join(model_dir, s) for s in shards]
diffusers = os.path.join(model_dir, "diffusion_pytorch_model.safetensors")
if os.path.exists(diffusers):
return [diffusers]
return []
# Model classification
ARCHS = {
"lm": "acestep-lm",
"dit": "acestep-dit",
"text-enc": "acestep-text-enc",
"vae": "acestep-vae",
}
def classify(name):
if name.startswith("acestep-5Hz-lm"):
return "lm"
if name.startswith("acestep-v15"):
return "dit"
if name.startswith("Qwen3-Embedding"):
return "text-enc"
if name == "vae":
return "vae"
return None
# GGUF metadata from config.json
def add_metadata(w, cfg, model_type):
if "num_hidden_layers" in cfg:
w.add_block_count(cfg["num_hidden_layers"])
if "hidden_size" in cfg:
w.add_embedding_length(cfg["hidden_size"])
if "intermediate_size" in cfg:
w.add_feed_forward_length(cfg["intermediate_size"])
if "num_attention_heads" in cfg:
w.add_head_count(cfg["num_attention_heads"])
if "num_key_value_heads" in cfg:
w.add_head_count_kv(cfg["num_key_value_heads"])
if "head_dim" in cfg:
w.add_key_length(cfg["head_dim"])
if "vocab_size" in cfg:
w.add_vocab_size(cfg["vocab_size"])
if "max_position_embeddings" in cfg:
w.add_context_length(cfg["max_position_embeddings"])
if "rms_norm_eps" in cfg:
w.add_layer_norm_rms_eps(cfg["rms_norm_eps"])
rope = cfg.get("rope_theta")
if rope:
w.add_rope_freq_base(float(rope))
if model_type == "lm":
if cfg.get("tie_word_embeddings"):
w.add_bool("acestep.tie_word_embeddings", True)
if model_type == "dit":
for key in [
"in_channels", "audio_acoustic_hidden_dim", "patch_size",
"sliding_window", "fsq_dim", "text_hidden_dim", "timbre_hidden_dim",
"num_lyric_encoder_hidden_layers", "num_timbre_encoder_hidden_layers",
"num_audio_decoder_hidden_layers", "num_attention_pooler_hidden_layers",
]:
if key in cfg:
w.add_uint32("acestep.%s" % key, cfg[key])
# XL models have separate encoder dimensions (2B models omit these)
for key in [
"encoder_hidden_size", "encoder_intermediate_size",
"encoder_num_attention_heads", "encoder_num_key_value_heads",
]:
if key in cfg:
w.add_uint32("acestep.%s" % key, cfg[key])
if cfg.get("is_turbo"):
w.add_bool("acestep.is_turbo", True)
levels = cfg.get("fsq_input_levels")
if levels:
w.add_array("acestep.fsq_input_levels", levels)
w.add_string("acestep.config_json", json.dumps(cfg, separators=(",", ":")))
# Tensor packing from safetensors
def add_tensors_from_sf(w, sf_path, tag, model_type):
meta, hdr_size = read_sf_header(sf_path)
names = sorted(meta.keys())
with open(sf_path, "rb") as f:
count = 0
total = 0
for name in names:
info = meta[name]
# normalize: some upstream checkpoints omit the "model." prefix
if model_type == "lm" and not name.startswith("model."):
name = "model." + name
dtype_str = info["dtype"]
shape = info["shape"]
off0, off1 = info["data_offsets"]
nbytes = off1 - off0
f.seek(hdr_size + off0)
raw = f.read(nbytes)
if dtype_str == "BF16":
arr = np.frombuffer(raw, dtype=np.uint16).reshape(shape)
w.add_tensor(name, arr, raw_dtype=BF16)
elif dtype_str == "F16":
arr = np.frombuffer(raw, dtype=np.float16).reshape(shape)
w.add_tensor(name, arr)
elif dtype_str == "F32":
# convert F32 to BF16: truncate lower 16 mantissa bits
arr = np.frombuffer(raw, dtype=np.uint32).reshape(shape)
arr = (arr >> 16).astype(np.uint16)
w.add_tensor(name, arr, raw_dtype=BF16)
nbytes = nbytes // 2 # actual stored size
else:
log(tag, " skip %s: dtype %s" % (name, dtype_str))
continue
count += 1
total += nbytes
return count, total
# silence_latent.pt reader (replaces pt2bin C++ tool)
# PyTorch .pt is a ZIP with entry "*/data/0" containing f32 [64, 15000]
# We transpose to [15000, 64] (ggml layout: 64 contiguous per frame)
def read_silence_latent(model_dir):
pt_path = os.path.join(model_dir, "silence_latent.pt")
if not os.path.exists(pt_path):
return None
with zipfile.ZipFile(pt_path) as z:
for entry in z.namelist():
if entry.endswith("/data/0"):
raw = z.read(entry)
src = np.frombuffer(raw, dtype=np.float32).reshape(64, 15000)
return src.T.copy()
return None
# BPE tokenizer embedding (vocab.json + merges.txt -> GGUF KV)
def add_bpe_tokenizer(w, model_dir, tag):
vocab_path = os.path.join(model_dir, "vocab.json")
merges_path = os.path.join(model_dir, "merges.txt")
if not os.path.exists(vocab_path) or not os.path.exists(merges_path):
return False
with open(vocab_path, "r", encoding="utf-8") as f:
vocab = json.load(f)
tokens = [""] * len(vocab)
for tok_str, tok_id in vocab.items():
if 0 <= tok_id < len(tokens):
tokens[tok_id] = tok_str
with open(merges_path, "r", encoding="utf-8") as f:
merges = []
for line in f:
line = line.rstrip("\n\r")
if not line:
continue
if line.startswith("#version:"):
continue
merges.append(line)
w.add_tokenizer_model("gpt2")
w.add_token_list(tokens)
w.add_token_merges(merges)
log(tag, " tokenizer: %d vocab, %d merges" % (len(tokens), len(merges)))
return True
# Main conversion
def convert_model(name, model_dir, output_path, model_type):
tag = "GGUF"
cfg_path = os.path.join(model_dir, "config.json")
if not os.path.exists(cfg_path):
log(tag, "skip %s: no config.json" % name)
return False
with open(cfg_path, "r", encoding="utf-8") as f:
cfg = json.load(f)
sf_files = find_sf_files(model_dir)
if not sf_files:
log(tag, "skip %s: no safetensors" % name)
return False
arch = ARCHS[model_type]
log(tag, "%s (%s, %d shard%s) -> %s" % (
name, arch, len(sf_files), "" if len(sf_files) == 1 else "s",
os.path.basename(output_path)))
w = gguf.GGUFWriter(output_path, arch, use_temp_file=True)
w.add_name(name)
add_metadata(w, cfg, model_type)
# BPE tokenizer for LM and text encoder
if model_type in ("lm", "text-enc"):
add_bpe_tokenizer(w, model_dir, tag)
# Model weights
n_tensors = 0
n_bytes = 0
for sf in sf_files:
c, b = add_tensors_from_sf(w, sf, tag, model_type)
n_tensors += c
n_bytes += b
if len(sf_files) > 1:
log(tag, " %s: %d tensors" % (os.path.basename(sf), c))
# silence_latent for DiT (read .pt, transpose, embed as f32 tensor)
if model_type == "dit":
sl = read_silence_latent(model_dir)
if sl is not None:
w.add_tensor("silence_latent", sl)
n_tensors += 1
n_bytes += sl.nbytes
log(tag, " silence_latent: [%d, %d] f32 (%.1f MB)" % (
sl.shape[0], sl.shape[1], sl.nbytes / (1 << 20)))
else:
log(tag, " WARNING: no silence_latent.pt found")
log(tag, " total: %d tensors, %.1f GB" % (n_tensors, n_bytes / (1 << 30)))
w.write_header_to_file()
w.write_kv_data_to_file()
w.write_tensors_to_file(progress=True)
w.close()
out_mb = os.path.getsize(output_path) / (1 << 20)
log(tag, " wrote %.0f MB -> %s" % (out_mb, output_path))
return True
def main():
if not os.path.isdir(CHECKPOINT_DIR):
log("GGUF", "checkpoints/ not found, run checkpoints.sh first")
sys.exit(1)
os.makedirs(OUTPUT_DIR, exist_ok=True)
entries = sorted(os.listdir(CHECKPOINT_DIR))
converted = 0
skipped = []
for name in entries:
model_dir = os.path.join(CHECKPOINT_DIR, name)
if not os.path.isdir(model_dir):
continue
model_type = classify(name)
if model_type is None:
skipped.append(name)
continue
output_path = os.path.join(OUTPUT_DIR, "%s-BF16.gguf" % name)
if os.path.exists(output_path):
log("GGUF", "skip %s: %s exists" % (name, os.path.basename(output_path)))
converted += 1
continue
if convert_model(name, model_dir, output_path, model_type):
converted += 1
if skipped:
log("GGUF", "skipped (unknown): %s" % ", ".join(skipped))
log("GGUF", "done: %d model(s) in %s" % (converted, OUTPUT_DIR))
if __name__ == "__main__":
main()
engine/docs/ARCHITECTURE.md
+1176
View File
File diff suppressed because it is too large Load Diff
engine/examples/ace-understand.sh
+37
View File
@@ -0,0 +1,37 @@
#!/bin/bash
# Roundtrip: audio -> understand -> SFT DiT -> MP3
#
# Usage: ./ace-understand.sh input.wav (or input.mp3)
#
# understand:
# input -> ace-understand.json (audio codes + metadata)
#
# ace-synth:
# ace-understand.json -> ace-understand0.mp3
set -eu
if [ $# -lt 1 ]; then
echo "Usage: $0 <input.wav|input.mp3>"
exit 1
fi
input="$1"
../build/ace-understand \
--src-audio "$input" \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf \
-o ace-understand.json
sed -i \
's/"audio_cover_strength": *[0-9.]*/"audio_cover_strength": 0.04/' \
ace-understand.json
../build/ace-synth \
--src-audio "$input" \
--request ace-understand.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/client-batch.py
+104
View File
@@ -0,0 +1,104 @@
#!/usr/bin/env python3
# client-batch.py: test batching via ace-server
#
# POST /lm (lm_batch_size=2 in JSON) -> 2 enriched requests
# POST /synth (JSON array of 2 requests) -> 2 MP3s in one GPU batch
#
# Start the server first: ./server.sh
import json
import sys
import urllib.error
import urllib.request
URL = "http://127.0.0.1:8085"
def post_json(endpoint, data):
body = json.dumps(data).encode()
req = urllib.request.Request(
URL + endpoint,
data=body,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req) as resp:
return resp.read(), resp.headers
def parse_multipart_mixed(data, content_type):
"""Parse multipart/mixed response into list of body bytes."""
boundary = None
for part in content_type.split(";"):
part = part.strip()
if part.startswith("boundary="):
boundary = part[len("boundary="):].strip().encode()
break
if not boundary:
raise ValueError("no boundary in content-type: " + content_type)
delimiter = b"--" + boundary
parts = []
for chunk in data.split(delimiter):
if not chunk or chunk.startswith(b"--"):
continue
chunk = chunk.strip(b"\r\n")
if not chunk:
continue
sep = chunk.find(b"\r\n\r\n")
if sep < 0:
continue
body = chunk[sep + 4:]
if body.endswith(b"\r\n"):
body = body[:-2]
parts.append(body)
return parts
# Phase 1: LM generates N variations
try:
with open("simple-batch.json") as f:
request_json = json.load(f)
except FileNotFoundError:
print("ERROR: simple-batch.json not found (run from the examples/ directory)")
sys.exit(1)
try:
lm_batch_size = request_json.get("lm_batch_size", 1)
print("POST /lm (lm_batch_size=%d)..." % lm_batch_size)
lm_data, _ = post_json("/lm", request_json)
except urllib.error.URLError as e:
print("ERROR: cannot connect to %s (%s)" % (URL, e.reason))
print("Start the server first: ./server.sh")
sys.exit(1)
lm_results = json.loads(lm_data)
print(" -> %d enriched requests" % len(lm_results))
# Phase 2: synth all in one GPU batch (send JSON array)
print("POST /synth (batch=%d, JSON array)..." % len(lm_results))
body = json.dumps(lm_results).encode()
req = urllib.request.Request(
URL + "/synth",
data=body,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req) as resp:
resp_data = resp.read()
content_type = resp.headers.get("Content-Type", "")
if "multipart/mixed" in content_type:
parts = parse_multipart_mixed(resp_data, content_type)
for i, mp3_data in enumerate(parts):
path = "server-batch%d.mp3" % i
with open(path, "wb") as f:
f.write(mp3_data)
print(" -> %s (%d bytes)" % (path, len(mp3_data)))
else:
path = "server-batch0.mp3"
with open(path, "wb") as f:
f.write(resp_data)
print(" -> %s (%d bytes)" % (path, len(resp_data)))
print("Done: %d MP3(s)" % len(lm_results))
engine/examples/client-understand.sh
+54
View File
@@ -0,0 +1,54 @@
#!/bin/bash
# Roundtrip via ace-server: audio -> understand -> synth -> MP3
#
# Usage: ./client-understand.sh input.wav (or input.mp3)
#
# POST /understand (async job):
# input -> server-understand.json (audio codes + metadata)
#
# POST /synth (async job):
# server-understand.json + input -> server-understand.mp3
#
# Start the server first (./server.sh).
set -eu
if [ $# -lt 1 ]; then
echo "Usage: $0 <input.wav|input.mp3>"
exit 1
fi
HOST="http://127.0.0.1:8085"
input="$1"
# poll a job until done, exit 1 on failure
wait_job() {
local id="$1"
while true; do
status=$(curl -sf "${HOST}/job?id=${id}" | jq -r '.status')
case "$status" in
done) return 0 ;;
failed|cancelled) echo "Job ${id}: ${status}"; return 1 ;;
esac
sleep 2
done
}
# understand: submit, poll, fetch result
UND_ID=$(curl -sf "${HOST}/understand" \
-F "audio=@${input}" | jq -r '.id')
echo "Understand job: ${UND_ID}"
wait_job "${UND_ID}"
curl -sf "${HOST}/job?id=${UND_ID}&result=1" -o server-understand.json
sed -i \
-e 's/"audio_cover_strength": *[0-9.]*/"audio_cover_strength": 0.04/' \
server-understand.json
# synth: submit, poll, fetch result
SYNTH_ID=$(curl -sf "${HOST}/synth" \
-F "request=@server-understand.json" \
-F "audio=@${input}" | jq -r '.id')
echo "Synth job: ${SYNTH_ID}"
wait_job "${SYNTH_ID}"
curl -sf "${HOST}/job?id=${SYNTH_ID}&result=1" -o server-understand.mp3
engine/examples/client.sh
+36
View File
@@ -0,0 +1,36 @@
#!/bin/bash
# Test ace-server: LM enriches caption, synth renders to MP3.
# Start the server first (./server.sh), then run this.
set -eu
HOST="http://127.0.0.1:8085"
# poll a job until done, exit 1 on failure
wait_job() {
local id="$1"
while true; do
status=$(curl -sf "${HOST}/job?id=${id}" | jq -r '.status')
case "$status" in
done) return 0 ;;
failed|cancelled) echo "Job ${id}: ${status}"; return 1 ;;
esac
sleep 2
done
}
# LM: submit, poll, fetch result
LM_ID=$(curl -sf "${HOST}/lm" \
-H "Content-Type: application/json" \
-d @full-sft.json | jq -r '.id')
echo "LM job: ${LM_ID}"
wait_job "${LM_ID}"
curl -sf "${HOST}/job?id=${LM_ID}&result=1" | jq '.[0]' > server-lm0.json
# synth: submit, poll, fetch result
SYNTH_ID=$(curl -sf "${HOST}/synth" \
-H "Content-Type: application/json" \
-d @server-lm0.json | jq -r '.id')
echo "Synth job: ${SYNTH_ID}"
wait_job "${SYNTH_ID}"
curl -sf "${HOST}/job?id=${SYNTH_ID}&result=1" -o server0.mp3
engine/examples/dit-only-sft.json
+12
View File
@@ -0,0 +1,12 @@
{
"caption": "Ambient electronic soundscape with warm analog pads",
"lyrics": "",
"bpm": 90,
"duration": 180,
"keyscale": "C minor",
"timesignature": "4",
"vocal_language": "en",
"inference_steps": 50,
"guidance_scale": 1.0,
"shift": 1.0
}
engine/examples/dit-only-sft.sh
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
set -eu
../build/ace-synth \
--request dit-only-sft.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/dit-only.json
+12
View File
@@ -0,0 +1,12 @@
{
"caption": "Ambient electronic soundscape with warm analog pads",
"lyrics": "",
"bpm": 90,
"duration": 180,
"keyscale": "C minor",
"timesignature": "4",
"vocal_language": "en",
"inference_steps": 8,
"guidance_scale": 1.0,
"shift": 3.0
}
engine/examples/dit-only.sh
+9
View File
@@ -0,0 +1,9 @@
#!/bin/bash
set -eu
../build/ace-synth \
--request dit-only.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/full-sft.json
+12
View File
@@ -0,0 +1,12 @@
{
"caption": "Upbeat French house with infectious disco-inspired bassline, crisp four-on-the-floor kick pattern, wah-wah filtered guitar riffs, retro synth stabs, soulful male lead vocals with gospel-style backing harmonies, smooth saxophone accents, warm vinyl crackle texture, bright summer vibe, polished modern mix with vintage analog warmth, driving yet laid-back energy perfect for rooftop parties and sunset drives",
"lyrics": "[Intro - Ligne de Basse Funk & Beat House]\n\n[Verse 1]\nSous le soleil de Paris, on danse sans fin\nLa nuit s'allume, le beat nous guide\nLes étoiles scintillent au rythme du kick\nUn sourire léger, tout est si vivant\n\n[Pre-Chorus]\nLaisse-toi porter par la musique qui chante\n\n[Chorus]\nOn danse sous le ciel étoilé\nLe monde s'arrête, on s'envole\nAvec ce groove qui nous emporte\nJusqu'au matin, on ne s'arrête pas\n\n[Verse 2]\nLa ville respire au son des cuivres légers\nLes mains en l'air, on oublie le temps\nLa basse funk nous secoue les pieds\nUn été éternel, rien ne peut nous briser\n\n[Chorus]\nOn danse sous le ciel étoilé\nLe monde s'arrête, on s'envole\nAvec ce groove qui nous emporte\nJusqu'au matin, on ne s'arrête pas\n\n[Guitar Solo - Wah-Wah Funk]\n\n[Bridge - Saxophone & Cordes]\nRespire profondément, l'univers t'appelle\n\n[Outro - Synth Fade avec Craquement Vinyle]",
"duration": 240,
"bpm": 124,
"vocal_language": "fr",
"keyscale": "F# major",
"timesignature": "4",
"inference_steps": 50,
"guidance_scale": 1.0,
"shift": 1.0
}
engine/examples/full-sft.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request full-sft.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request full-sft0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/full.json
+12
View File
@@ -0,0 +1,12 @@
{
"caption": "Upbeat French house with infectious disco-inspired bassline, crisp four-on-the-floor kick pattern, wah-wah filtered guitar riffs, retro synth stabs, soulful male lead vocals with gospel-style backing harmonies, smooth saxophone accents, warm vinyl crackle texture, bright summer vibe, polished modern mix with vintage analog warmth, driving yet laid-back energy perfect for rooftop parties and sunset drives",
"lyrics": "[Intro - Ligne de Basse Funk & Beat House]\n\n[Verse 1]\nSous le soleil de Paris, on danse sans fin\nLa nuit s'allume, le beat nous guide\nLes étoiles scintillent au rythme du kick\nUn sourire léger, tout est si vivant\n\n[Pre-Chorus]\nLaisse-toi porter par la musique qui chante\n\n[Chorus]\nOn danse sous le ciel étoilé\nLe monde s'arrête, on s'envole\nAvec ce groove qui nous emporte\nJusqu'au matin, on ne s'arrête pas\n\n[Verse 2]\nLa ville respire au son des cuivres légers\nLes mains en l'air, on oublie le temps\nLa basse funk nous secoue les pieds\nUn été éternel, rien ne peut nous briser\n\n[Chorus]\nOn danse sous le ciel étoilé\nLe monde s'arrête, on s'envole\nAvec ce groove qui nous emporte\nJusqu'au matin, on ne s'arrête pas\n\n[Guitar Solo - Wah-Wah Funk]\n\n[Bridge - Saxophone & Cordes]\nRespire profondément, l'univers t'appelle\n\n[Outro - Synth Fade avec Craquement Vinyle]",
"duration": 240,
"bpm": 124,
"vocal_language": "fr",
"keyscale": "F# major",
"timesignature": "4",
"inference_steps": 8,
"guidance_scale": 1.0,
"shift": 3.0
}
engine/examples/full.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request full.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request full0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/lego.json
+9
View File
@@ -0,0 +1,9 @@
{
"caption": "electric guitar riff, funk guitar, house music, instrumental",
"lyrics": "[Instrumental]",
"task_type": "lego",
"track": "guitar",
"inference_steps": 50,
"guidance_scale": 1.0,
"shift": 1.0
}
engine/examples/lego.sh
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
# Generate a source track, then lego a guitar stem over it
#
# Note: lego requires acestep-v15-base; turbo/sft do not support it
#
# LM + DiT phase (source track):
# simple.json -> simple0.json -> simple00.wav
#
# Lego phase (guitar stem over source):
# lego.json + simple00.wav -> lego0.wav
set -eu
# Phase 1: generate a source track with the simple prompt
../build/ace-lm \
--request simple.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request simple0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf \
--format wav16
# Phase 2: lego guitar on the generated track (base model required)
../build/ace-synth \
--src-audio simple00.wav \
--request lego.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-base-Q8_0.gguf \
--vae ../models/vae-BF16.gguf \
--format wav16
engine/examples/partial-sft.json
+9
View File
@@ -0,0 +1,9 @@
{
"caption": "Hard-hitting hip hop track with deep 808 bass, crisp trap hi-hats, heavy snare rolls, dark piano melody, and confident aggressive vocal delivery",
"lyrics": "[Intro]\nYeah... c'est comme ca...\n\n[Verse 1]\nJe marche dans la ville quand le soleil se couche\nLes lumieres s'allument j'ai les mots dans la bouche\nOn m'a dit fais attention le monde est pas facile\nJ'ai repondu tranquille j'ai grandi dans la ville\nLes murs ont des oreilles les rues ont des histoires\nChaque coin chaque angle chaque bout de trottoir\nJ'ai vu des gens tomber j'ai vu des gens se lever\nMoi je reste debout j'ai pas le temps de plier\n\n[Chorus]\nOn avance on recule pas\nLa vie nous teste a chaque pas\nOn avance on recule pas\nRegarde devant oublie tout ca\n\n[Verse 2]\nLe reveil sonne tot le cafe brule les levres\nLe metro le boulot la routine la fievre\nMais le soir dans ma chambre je reprends mon stylo\nJe pose sur le papier tout ce que j'ai sur le dos\nMes reves sont plus grands que les murs de ma chambre\nPlus chauds que juillet plus forts que decembre\nOn m'a dit sois realiste range tes illusions\nJ'ai repondu ma vie c'est pas de la fiction\n\n[Chorus]\nOn avance on recule pas\nLa vie nous teste a chaque pas\nOn avance on recule pas\nRegarde devant oublie tout ca\n\n[Outro]\nYeah... on lache rien... jamais...",
"duration": 200,
"vocal_language": "fr",
"inference_steps": 50,
"guidance_scale": 1.0,
"shift": 1.0
}
engine/examples/partial-sft.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request partial-sft.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request partial-sft0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/partial.json
+9
View File
@@ -0,0 +1,9 @@
{
"caption": "Hard-hitting hip hop track with deep 808 bass, crisp trap hi-hats, heavy snare rolls, dark piano melody, and confident aggressive vocal delivery",
"lyrics": "[Intro]\nYeah... c'est comme ca...\n\n[Verse 1]\nJe marche dans la ville quand le soleil se couche\nLes lumieres s'allument j'ai les mots dans la bouche\nOn m'a dit fais attention le monde est pas facile\nJ'ai repondu tranquille j'ai grandi dans la ville\nLes murs ont des oreilles les rues ont des histoires\nChaque coin chaque angle chaque bout de trottoir\nJ'ai vu des gens tomber j'ai vu des gens se lever\nMoi je reste debout j'ai pas le temps de plier\n\n[Chorus]\nOn avance on recule pas\nLa vie nous teste a chaque pas\nOn avance on recule pas\nRegarde devant oublie tout ca\n\n[Verse 2]\nLe reveil sonne tot le cafe brule les levres\nLe metro le boulot la routine la fievre\nMais le soir dans ma chambre je reprends mon stylo\nJe pose sur le papier tout ce que j'ai sur le dos\nMes reves sont plus grands que les murs de ma chambre\nPlus chauds que juillet plus forts que decembre\nOn m'a dit sois realiste range tes illusions\nJ'ai repondu ma vie c'est pas de la fiction\n\n[Chorus]\nOn avance on recule pas\nLa vie nous teste a chaque pas\nOn avance on recule pas\nRegarde devant oublie tout ca\n\n[Outro]\nYeah... on lache rien... jamais...",
"duration": 200,
"vocal_language": "fr",
"inference_steps": 8,
"guidance_scale": 1.0,
"shift": 3.0
}
engine/examples/partial.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request partial.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request partial0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/simple-batch.json
+8
View File
@@ -0,0 +1,8 @@
{
"caption": "Upbeat pop rock anthem with driving electric guitars, punchy drums, catchy vocal hooks, and a singalong chorus",
"vocal_language": "fr",
"lm_batch_size": 2,
"inference_steps": 8,
"guidance_scale": 1.0,
"shift": 3.0
}
engine/examples/simple-batch.sh
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Generate 2 songs: LM produces 2 enriched requests (different codes/metas),
# DiT renders them in a single GPU batch.
#
# LM phase (lm_batch_size=2 in simple-batch.json):
# simple-batch.json -> simple-batch0.json, simple-batch1.json
#
# DiT phase (both requests in one batch):
# simple-batch0.json + simple-batch1.json -> simple-batch00.mp3, simple-batch11.mp3
set -eu
# Phase 1: LM generates 2 variations (different lyrics/codes/metas)
../build/ace-lm \
--request simple-batch.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
# Phase 2: DiT+VAE renders both in one GPU batch
../build/ace-synth \
--request simple-batch0.json simple-batch1.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/simple-sft.json
+7
View File
@@ -0,0 +1,7 @@
{
"caption": "Upbeat pop rock anthem with driving electric guitars, punchy drums, catchy vocal hooks, and a singalong chorus",
"vocal_language": "fr",
"inference_steps": 50,
"guidance_scale": 1.0,
"shift": 1.0
}
engine/examples/simple-sft.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request simple-sft.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request simple-sft0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-sft-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/examples/simple.cmd
+15
View File
@@ -0,0 +1,15 @@
@echo off
set PATH=%~dp0..\build\Release;%PATH%
ace-lm.exe ^
--request simple.json ^
--lm ..\models\acestep-5Hz-lm-4B-Q6_K.gguf
ace-synth.exe ^
--request simple0.json ^
--embedding ..\models\Qwen3-Embedding-0.6B-Q8_0.gguf ^
--dit ..\models\acestep-v15-turbo-Q6_K.gguf ^
--vae ..\models\vae-BF16.gguf
pause
engine/examples/simple.json
+7
View File
@@ -0,0 +1,7 @@
{
"caption": "Upbeat pop rock anthem with driving electric guitars, punchy drums, catchy vocal hooks, and a singalong chorus",
"vocal_language": "fr",
"inference_steps": 8,
"guidance_scale": 1.0,
"shift": 3.0
}
engine/examples/simple.sh
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
set -eu
../build/ace-lm \
--request simple.json \
--lm ../models/acestep-5Hz-lm-4B-Q8_0.gguf
../build/ace-synth \
--request simple0.json \
--embedding ../models/Qwen3-Embedding-0.6B-Q8_0.gguf \
--dit ../models/acestep-v15-turbo-Q8_0.gguf \
--vae ../models/vae-BF16.gguf
engine/format.sh
+3
View File
@@ -0,0 +1,3 @@
#!/bin/bash
find . -name "*.cpp" -o -name "*.h" | grep -v -e build/ -e ggml/ -e vendor/ | xargs clang-format -i
engine/ggml/.editorconfig
+22
View File
@@ -0,0 +1,22 @@
# https://EditorConfig.org
# Top-most EditorConfig file
root = true
# Unix-style newlines with a newline ending every file, utf-8 charset
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 4
[*.md]
indent_size = 2
[Makefile]
indent_style = tab
[prompts/*.txt]
insert_final_newline = unset
engine/ggml/.github/pull_request_template.md
Vendored
+1
View File
@@ -0,0 +1 @@
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
engine/ggml/.github/workflows/ci.yml
Vendored
+272
View File
@@ -0,0 +1,272 @@
name: CI
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
cancel-in-progress: true
jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
libraries: [shared, static]
runs-on: ${{ matrix.os }}
steps:
- name: Clone
uses: actions/checkout@v6
- name: Dependencies for Ubuntu
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt-get install llvm
- name: Add msbuild to PATH
if: matrix.os == 'windows-latest'
uses: microsoft/setup-msbuild@v2
- name: Create Build Environment
run: mkdir build
- name: Configure CMake
working-directory: ./build
run: cmake ..
${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
${{ matrix.libraries == 'static' && '-DBUILD_SHARED_LIBS=OFF' || '-DBUILD_SHARED_LIBS=ON' }}
-DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed
-DGGML_METAL=OFF
- name: Build
working-directory: ./build
run: cmake --build . ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
- name: Test
working-directory: ./build
run: ctest --verbose --timeout 900 ${{ contains(matrix.os, 'windows') && '--build-config Release' || '' }}
- name: Install
working-directory: ./build
run: cmake --build . --target install ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
- name: Test CMake config
run: |
mkdir test-cmake
cmake -S examples/test-cmake -B test-cmake -DCMAKE_PREFIX_PATH=${{ github.workspace }}/installed ${{ contains(matrix.os, 'windows') && '-A x64' || '-G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++' }}
cmake --build test-cmake ${{ contains(matrix.os, 'windows') && '--config Release' || '' }}
# TODO: simplify the following workflows using a matrix
ggml-ci-x64-cpu-low-perf:
runs-on: ubuntu-22.04
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-x64-cpu-low-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-low-perf:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-low-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-x64-cpu-high-perf:
runs-on: ubuntu-22.04
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-x64-cpu-high-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-high-perf:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-high-perf
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-arm64-cpu-high-perf-sve:
runs-on: ubuntu-22.04-arm
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: ccache
uses: ggml-org/ccache-action@v1.2.16
with:
key: ggml-ci-arm64-cpu-high-perf-sve
evict-old-files: 1d
- name: Dependencies
id: depends
run: |
sudo apt-get update
sudo apt-get install build-essential libcurl4-openssl-dev
- name: Test
id: ggml-ci
run: |
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
ggml-ci-x64-nvidia-cuda:
runs-on: [self-hosted, Linux, X64, NVIDIA]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
nvidia-smi
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-x64-nvidia-vulkan-cm:
runs-on: [self-hosted, Linux, X64, NVIDIA]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-x64-nvidia-vulkan-cm2:
runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml /mnt/ggml
# TODO: provision AMX-compatible machine
#ggml-ci-x64-cpu-amx:
# runs-on: [self-hosted, Linux, X64, CPU, AMX]
# steps:
# - name: Clone
# id: checkout
# uses: actions/checkout@v6
# - name: Test
# id: ggml-ci
# run: |
# bash ./ci/run.sh ~/results/ggml /mnt/ggml
ggml-ci-mac-metal:
runs-on: [self-hosted, macOS, ARM64]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
ggml-ci-mac-vulkan:
runs-on: [self-hosted, macOS, ARM64]
steps:
- name: Clone
id: checkout
uses: actions/checkout@v6
- name: Test
id: ggml-ci
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/ggml ~/mnt/ggml
engine/ggml/.github/workflows/release.yml
Vendored
+27
View File
@@ -0,0 +1,27 @@
name: Release
on:
push:
tags:
- 'v*'
jobs:
release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Create Release
id: create_release
uses: ggml-org/action-create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref_name }}
release_name: ${{ github.ref }}
draft: false
prerelease: false
engine/ggml/.gitignore
+39
View File
@@ -0,0 +1,39 @@
build/
build-*/
out/
tmp/
models/
models-mnt
compile_commands.json
CMakeSettings.json
.vs/
.vscode/
.idea/
.clangd
.venv/
ggml_env/
.exrc
.cache
.DS_Store
.stablelm
.gpt-2
src/arm_neon.h
tests/arm_neon.h
zig-out/
zig-cache/
*.o
*.d
*.dot
*.sw?
__pycache__/
# Model files
ggml-model-f16.bin
*.bat
engine/ggml/.gitmodules
View File
engine/ggml/AUTHORS
+335
View File
@@ -0,0 +1,335 @@
# date: Tue Feb 4 13:03:51 EET 2025
# this file is auto-generated by scripts/gen-authors.sh
0cc4m <picard12@live.de>
65a <10104049+65a@users.noreply.github.com>
AT <manyoso@users.noreply.github.com>
Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com>
Adam Tazi <52357206+ad1tazi@users.noreply.github.com>
Adrien Gallouët <adrien@gallouet.fr>
Adrien Gallouët <angt@huggingface.co>
Ahmad Tameem <113388789+Tameem-10xE@users.noreply.github.com>
AidanBeltonS <87009434+AidanBeltonS@users.noreply.github.com>
AidanBeltonS <aidan.belton@codeplay.com>
Akarshan Biswas <akarshan.biswas@gmail.com>
Akarshan Biswas <akarshanbiswas@fedoraproject.org>
Albert Jin <albert.jin@gmail.com>
Alberto Cabrera Pérez <alberto.cabrera@codeplay.com>
Alberto Cabrera Pérez <alberto.cabrera@intel.com>
Alex Azarov <alex@azarov.by>
Alex O'Connell <35843486+acon96@users.noreply.github.com>
Alex von Gluck IV <kallisti5@unixzen.com>
AmbientL <107641468+AmbientL@users.noreply.github.com>
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
Ananta Bastola <anantarajbastola@gmail.com>
Andreas (Andi) Kunar <andreask@msn.com>
Andreas Kieslinger <47689530+aendk@users.noreply.github.com>
Andrei <abetlen@gmail.com>
Andrew Minh Nguyen <40281306+amqdn@users.noreply.github.com>
Andrii Ryzhkov <andriiryzhkov@users.noreply.github.com>
Arjun <ccldarjun@icloud.com>
Ashraful Islam <ashraful.meche@gmail.com>
Astariul <43774355+astariul@users.noreply.github.com>
AsukaMinato <asukaminato@nyan.eu.org>
Avi Lumelsky <avilume@gmail.com>
Bart Pelle <3662930+Velocity-@users.noreply.github.com>
Ben Ashbaugh <ben.ashbaugh@intel.com>
Bernhard M. Wiedemann <githubbmwprimary@lsmod.de>
Borislav Stanimirov <b.stanimirov@abv.bg>
Brad Ito <phlogisticfugu@users.noreply.github.com>
Brad Murray <59848399+bradmurray-dt@users.noreply.github.com>
Brian <mofosyne@gmail.com>
Bryan Lozano <b.lozano.havoc@gmail.com>
Carolinabanana <140120812+Carolinabanana@users.noreply.github.com>
CarterLi999 <664681047@qq.com>
Cebtenzzre <cebtenzzre@gmail.com>
Changyeon Kim <cyzero.kim@samsung.com>
Charles Xu <63788048+chaxu01@users.noreply.github.com>
Charles Xu <charles.xu@arm.com>
Chen Xi <xi2.chen@intel.com>
Chen Xi <xixichen08@foxmail.com>
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
Chris Elrod <elrodc@gmail.com>
Christian Kastner <ckk@kvr.at>
Clint Herron <hanclinto@gmail.com>
Conrad Kramer <conrad@conradkramer.com>
Cordeiro <1471463+ocordeiro@users.noreply.github.com>
Cristiano Calcagno <cristianoc@users.noreply.github.com>
DAN™ <dranger003@gmail.com>
Dan Forbes <dan@danforbes.dev>
Dan Johansson <164997844+eddnjjn@users.noreply.github.com>
Dan Johansson <dan.johansson@arm.com>
Daniel Bevenius <daniel.bevenius@gmail.com>
Daniel Ziegenberg <daniel@ziegenberg.at>
Daniele <57776841+daniandtheweb@users.noreply.github.com>
Daulet Zhanguzin <daulet@users.noreply.github.com>
Dave <dave-fl@users.noreply.github.com>
Dave Airlie <airlied@gmail.com>
Dave Airlie <airlied@redhat.com>
David Miller <david@patagona.ca>
DavidKorczynski <david@adalogics.com>
Davidson Francis <davidsondfgl@gmail.com>
Dibakar Gope <dibakar.gope@arm.com>
Didzis Gosko <didzis@users.noreply.github.com>
Diego Devesa <slarengh@gmail.com>
Diogo <dgcruz983@gmail.com>
Djip007 <3705339+Djip007@users.noreply.github.com>
Djip007 <djip.perois@free.fr>
Dou Xinpeng <15529241576@163.com>
Dou Xinpeng <81913537+Dou-Git@users.noreply.github.com>
Dr. Tom Murphy VII Ph.D <499244+tom7@users.noreply.github.com>
Ebey Abraham <ebey97@gmail.com>
Eldar Yusupov <eyusupov@gmail.com>
Emmanuel Durand <emmanueldurand@protonmail.com>
Engininja2 <139037756+Engininja2@users.noreply.github.com>
Eric Zhang <34133756+EZForever@users.noreply.github.com>
Erik Scholz <Green-Sky@users.noreply.github.com>
Ettore Di Giacinto <mudler@users.noreply.github.com>
Eve <139727413+netrunnereve@users.noreply.github.com>
F1L1P <78918286+F1L1Pv2@users.noreply.github.com>
Faisal Zaghloul <quic_fzaghlou@quicinc.com>
FantasyGmm <16450052+FantasyGmm@users.noreply.github.com>
Felix <stenbackfelix@gmail.com>
Finn Voorhees <finnvoorhees@gmail.com>
FirstTimeEZ <179362031+FirstTimeEZ@users.noreply.github.com>
Frankie Robertson <frankier@users.noreply.github.com>
GainLee <perfecter.gen@gmail.com>
George Hindle <george@georgehindle.com>
Georgi Gerganov <ggerganov@gmail.com>
Gilad S <7817232+giladgd@users.noreply.github.com>
Gilad S <giladgd@users.noreply.github.com>
Gilad S. <7817232+giladgd@users.noreply.github.com>
Guillaume Wenzek <gwenzek@users.noreply.github.com>
Halalaluyafail3 <55773281+Halalaluyafail3@users.noreply.github.com>
Haus1 <haus.xda@gmail.com>
Herman Semenov <GermanAizek@yandex.ru>
HimariO <dsfhe49854@gmail.com>
Hirochika Matsumoto <git@hkmatsumoto.com>
Hong Bo PENG <penghb@cn.ibm.com>
Hugo Rosenkranz-Costa <hugo.rosenkranz@gmail.com>
Hyunsung Lee <ita9naiwa@gmail.com>
IGUILIZ Salah-Eddine <76955987+salahiguiliz@users.noreply.github.com>
Ian Bull <irbull@eclipsesource.com>
Ihar Hrachyshka <ihrachys@redhat.com>
Ikko Eltociear Ashimine <eltociear@gmail.com>
Ivan <nekotekina@gmail.com>
Ivan Filipov <159561759+vanaka11@users.noreply.github.com>
Ivan Stepanov <ivanstepanovftw@gmail.com>
Ivan Zdane <accounts@ivanzdane.com>
Jack Mousseau <jmousseau@users.noreply.github.com>
Jack Vial <vialjack@gmail.com>
JacobLinCool <jacoblincool@gmail.com>
Jakob Frick <jakob.maria.frick@gmail.com>
Jan Ploski <jpl@plosquare.com>
Jared Van Bortel <jared@nomic.ai>
Jeff Bolz <jbolz@nvidia.com>
Jeffrey Quesnelle <jquesnelle@gmail.com>
Jeroen Mostert <jeroen.mostert@cm.com>
Jiahao Li <liplus17@163.com>
JidongZhang-THU <1119708529@qq.com>
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
Jo Liss <joliss42@gmail.com>
Joe Todd <joe.todd@codeplay.com>
Johannes Gäßler <johannesg@5d6.de>
John Balis <phobossystems@gmail.com>
Josh Bleecher Snyder <josharian@gmail.com>
Judd <foldl@users.noreply.github.com>
Jun Hee Yoo <contact.jhyoo@gmail.com>
Junil Kim <logyourself@gmail.com>
Justina Cho <justcho5@gmail.com>
Justine Tunney <jtunney@gmail.com>
Justine Tunney <jtunney@mozilla.com>
Karol Kontny <82021046+kkontny@users.noreply.github.com>
Kawrakow <48489457+ikawrakow@users.noreply.github.com>
Kevin Gibbons <bakkot@gmail.com>
Konstantin Zhuravlyov <konstantin.zhuravlyov@amd.com>
Kylin <56434533+KyL0N@users.noreply.github.com>
LoganDark <git@logandark.mozmail.com>
LoganDark <github@logandark.mozmail.com>
LostRuins <39025047+LostRuins@users.noreply.github.com>
Lukas Möller <mail@lukas-moeller.ch>
M Refi D.A <24388107+refinism@users.noreply.github.com>
M. Yusuf Sarıgöz <yusufsarigoz@gmail.com>
Ma Mingfei <mingfei.ma@intel.com>
Mahesh Madhav <67384846+heshpdx@users.noreply.github.com>
MaiHD <maihd.dev@gmail.com>
Mark Zhuang <zhuangqiubin@gmail.com>
Markus Tavenrath <mtavenrath@users.noreply.github.com>
Masaya, Kato <62578291+msy-kato@users.noreply.github.com>
Mathieu Baudier <mbaudier@argeo.org>
Mathijs de Bruin <mathijs@mathijsfietst.nl>
Matt Stephenson <mstephenson6@users.noreply.github.com>
Max Krasnyansky <max.krasnyansky@gmail.com>
Max Krasnyansky <quic_maxk@quicinc.com>
Mayank Kumar Pal <mynkpl1998@gmail.com>
Meng, Hengyu <hengyu.meng@intel.com>
Mengqing Cao <cmq0113@163.com>
Metal Whale <45712559+metalwhale@users.noreply.github.com>
Michael Klimenko <mklimenko29@gmail.com>
Michael Podvitskiy <podvitskiymichael@gmail.com>
Michael Verrilli <msv@pobox.com>
Molly Sophia <mollysophia379@gmail.com>
Natsu <chino@hotococoa.moe>
Neo Zhang <14088817+arthw@users.noreply.github.com>
Neo Zhang Jianyu <jianyu.zhang@intel.com>
Neuman Vong <neuman.vong@gmail.com>
Nevin <nevinpuri1901@gmail.com>
Nicholai Tukanov <nicholaitukanov@gmail.com>
Nico Bosshard <nico@bosshome.ch>
Nicolò Scipione <nicolo.scipione@codeplay.com>
Nikita Sarychev <42014488+sARY77@users.noreply.github.com>
Nouamane Tazi <nouamane98@gmail.com>
Olivier Chafik <ochafik@google.com>
Olivier Chafik <ochafik@users.noreply.github.com>
Ondřej Čertík <ondrej@certik.us>
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
PAB <pierreantoine.bannier@gmail.com>
Paul Tsochantaris <ptsochantaris@icloud.com>
Peter <peter277@users.noreply.github.com>
Philpax <me@philpax.me>
Pierre Alexandre SCHEMBRI <pa.schembri@gmail.com>
Plamen Minev <pacominev@gmail.com>
Playdev <josang1204@gmail.com>
Prashant Vithule <119530321+Vithulep@users.noreply.github.com>
Przemysław Pawełczyk <przemoc@gmail.com>
R0CKSTAR <xiaodong.ye@mthreads.com>
R0CKSTAR <yeahdongcn@gmail.com>
Radoslav Gerganov <rgerganov@gmail.com>
Radosław Gryta <radek.gryta@gmail.com>
Ravindra Marella <marella@users.noreply.github.com>
Ray Cromwell <cromwellian@gmail.com>
Reinforce-II <fate@eastal.com>
Rémy Oudompheng <oudomphe@phare.normalesup.org>
Reza Rezvan <reza@rezvan.xyz>
Rick G <26732651+TheFlipbook@users.noreply.github.com>
RiverZhou <riverzhou2000@gmail.com>
Robert Ormandi <52251610+ormandi@users.noreply.github.com>
Romain Biessy <romain.biessy@codeplay.com>
Ronsor <ronsor@ronsor.pw>
Rotem Dan <rotemdan@gmail.com>
Ryan Hitchman <hitchmanr@gmail.com>
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
SXX <sxx1136965276@gmail.com>
Salvatore Mesoraca <s.mesoraca16@gmail.com>
Sam Spilsbury <smspillaz@gmail.com>
Sanchit Gandhi <93869735+sanchit-gandhi@users.noreply.github.com>
Santtu Keskinen <santtu.keskinen@gmail.com>
Sergio López <slp@redhat.com>
Sergio López <slp@sinrega.org>
Shanshan Shen <467638484@qq.com>
Shijie <821898965@qq.com>
Shupei Fan <dymarkfan@outlook.com>
Siddharth Ramakrishnan <srr2141@columbia.edu>
Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
Skyler Celestinian-Sterling <80314197+Celestinian@users.noreply.github.com>
Slava Primenko <primenko.s@gmail.com>
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
Steward Garcia <57494570+FSSRepo@users.noreply.github.com>
Supreet Sethi <supreet.sethi@gmail.com>
Takuya Takeuchi <takuya.takeuchi.dev@gmail.com>
Tamotsu Takahashi <ttakah+github@gmail.com>
Tanmay <tnmysachan@gmail.com>
Tanmay Sachan <tnmysachan@gmail.com>
Timothy Cronin <40186632+4imothy@users.noreply.github.com>
Tom Bailey <tombailey@users.noreply.github.com>
Tom Jobbins <784313+TheBloke@users.noreply.github.com>
Tony Wasserka <4840017+neobrain@users.noreply.github.com>
Tristan Druyen <tristan@vault81.mozmail.com>
Tyé singwa <92231658+tye-singwa@users.noreply.github.com>
UEXTM.com <84163508+uextm@users.noreply.github.com>
WillCorticesAI <150854901+WillCorticesAI@users.noreply.github.com>
William Tambellini <william.tambellini@gmail.com>
William Tambellini <wtambellini@sdl.com>
XiaotaoChen <chenxiaotao1234@gmail.com>
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
Xuan Son Nguyen <thichthat@gmail.com>
Yavor Ivanov <yivanov@viewray.com>
YavorGIvanov <yivanov@viewray.com>
Yilong Guo <vfirst218@gmail.com>
Yilong Guo <yilong.guo@intel.com>
Yuri Khrustalev <ykhrustalev@users.noreply.github.com>
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
Zhiyuan Li <lizhiyuan@uniartisan.com>
Zhiyuan Li <uniartisan2017@gmail.com>
a3sh <38979186+A3shTnT@users.noreply.github.com>
ag2s20150909 <19373730+ag2s20150909@users.noreply.github.com>
agray3 <agray3@users.noreply.github.com>
amd-dwang <dong.wang@amd.com>
amritahs-ibm <amritahs@linux.vnet.ibm.com>
apcameron <37645737+apcameron@users.noreply.github.com>
appvoid <78444142+appvoid@users.noreply.github.com>
ariez-xyz <41232910+ariez-xyz@users.noreply.github.com>
automaticcat <daogiatuank54@gmail.com>
bandoti <141645996+bandoti@users.noreply.github.com>
bmwl <brian.marshall@tolko.com>
bobqianic <129547291+bobqianic@users.noreply.github.com>
bssrdf <merlintiger@hotmail.com>
chengchi <davesjoewang@gmail.com>
compilade <113953597+compilade@users.noreply.github.com>
compilade <git@compilade.net>
ddpasa <112642920+ddpasa@users.noreply.github.com>
denersc <denerstassun@gmail.com>
dscripka <dscripka@users.noreply.github.com>
fitzsim <fitzsim@fitzsim.org>
fj-y-saito <85871716+fj-y-saito@users.noreply.github.com>
fraxy-v <65565042+fraxy-v@users.noreply.github.com>
gn64 <yukikaze.jp@gmail.com>
goerch <jhr.walter@t-online.de>
goldwaving <77494627+goldwaving@users.noreply.github.com>
haopeng <657407891@qq.com>
hidenorly <hidenorly@users.noreply.github.com>
hipudding <huafengchun@gmail.com>
hydai <z54981220@gmail.com>
issixx <46835150+issixx@users.noreply.github.com>
jaeminSon <woalsdnd@gmail.com>
jdomke <28772296+jdomke@users.noreply.github.com>
jiez <373447296@qq.com>
johnson442 <56517414+johnson442@users.noreply.github.com>
junchao-loongson <68935141+junchao-loongson@users.noreply.github.com>
k.h.lai <adrian.k.h.lai@outlook.com>
katsu560 <118887472+katsu560@users.noreply.github.com>
klosax <131523366+klosax@users.noreply.github.com>
kunnis <kunnis@users.noreply.github.com>
l3utterfly <gc.pthzfoldr@gmail.com>
le.chang <cljs118@126.com>
leejet <31925346+leejet@users.noreply.github.com>
leejet <leejet714@gmail.com>
leo-pony <nengjunma@outlook.com>
lhez <quic_lih@quicinc.com>
liuwei-git <14815172+liuwei-git@users.noreply.github.com>
luoyu-intel <yu.luo@intel.com>
magicse <magicse@users.noreply.github.com>
mahorozte <41834471+mahorozte@users.noreply.github.com>
mashizora <30516315+mashizora@users.noreply.github.com>
matt23654 <matthew.webber@protonmail.com>
matteo <matteogeniaccio@yahoo.it>
ochafik <ochafik@google.com>
otaGran <ujt2h8@gmail.com>
pengxin99 <pengxin.yuan@intel.com>
pikalover6 <49179590+pikalover6@users.noreply.github.com>
postmasters <namnguyen@google.com>
sjinzh <sjinzh@gmail.com>
skirodev <57715494+skirodev@users.noreply.github.com>
slaren <slarengh@gmail.com>
snadampal <87143774+snadampal@users.noreply.github.com>
someone13574 <81528246+someone13574@users.noreply.github.com>
stduhpf <stephduh@live.fr>
taher <8665427+nullhook@users.noreply.github.com>
texmex76 <40733439+texmex76@users.noreply.github.com>
the-crypt-keeper <84680712+the-crypt-keeper@users.noreply.github.com>
thewh1teagle <61390950+thewh1teagle@users.noreply.github.com>
ucag.li <ucag@qq.com>
ulatekh <ulatekh@yahoo.com>
uvos <devnull@uvos.xyz>
uvos <philipp@uvos.xyz>
wangshuai09 <391746016@qq.com>
woachk <24752637+woachk@users.noreply.github.com>
xctan <axunlei@gmail.com>
yangyaofei <yangyaofei@gmail.com>
yuri@FreeBSD <yuri@FreeBSD>
zhentaoyu <zhentao.yu@intel.com>
zhouwg <6889919+zhouwg@users.noreply.github.com>
zhouwg <zhouwg2000@gmail.com>
谢乃闻 <sienaiwun@users.noreply.github.com>
布客飞龙 <562826179@qq.com>
旺旺碎冰冰 <38837039+Cyberhan123@users.noreply.github.com>
engine/ggml/CMakeLists.txt
+505
View File
@@ -0,0 +1,505 @@
cmake_minimum_required(VERSION 3.14...3.28) # for add_link_options and implicit target directories.
project("ggml" C CXX ASM)
### GGML Version
set(GGML_VERSION_MAJOR 0)
set(GGML_VERSION_MINOR 15)
set(GGML_VERSION_PATCH 2)
set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
if(GIT_EXE)
# Get current git commit hash
execute_process(COMMAND ${GIT_EXE} rev-parse --short HEAD
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE GGML_BUILD_COMMIT
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET
)
# Check if the working directory is dirty (i.e., has uncommitted changes)
execute_process(COMMAND ${GIT_EXE} diff-index --quiet HEAD -- .
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE GGML_GIT_DIRTY
ERROR_QUIET
)
endif()
set(GGML_VERSION "${GGML_VERSION_BASE}")
if(NOT GGML_BUILD_COMMIT)
set(GGML_BUILD_COMMIT "unknown")
endif()
# Build the commit string with optional dirty flag
if(DEFINED GGML_GIT_DIRTY AND GGML_GIT_DIRTY EQUAL 1)
set(GGML_BUILD_COMMIT "${GGML_BUILD_COMMIT}-dirty")
endif()
include(CheckIncludeFileCXX)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()
if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(GGML_STANDALONE ON)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
# configure project version
# TODO
else()
set(GGML_STANDALONE OFF)
if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
endif()
endif()
if (EMSCRIPTEN)
set(BUILD_SHARED_LIBS_DEFAULT OFF)
option(GGML_WASM_SINGLE_FILE "ggml: embed WASM inside the generated ggml.js" ON)
else()
if (MINGW)
set(BUILD_SHARED_LIBS_DEFAULT OFF)
else()
set(BUILD_SHARED_LIBS_DEFAULT ON)
endif()
endif()
# remove the lib prefix on win32 mingw
if (WIN32)
set(CMAKE_STATIC_LIBRARY_PREFIX "")
set(CMAKE_SHARED_LIBRARY_PREFIX "")
set(CMAKE_SHARED_MODULE_PREFIX "")
endif()
option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
option(GGML_BACKEND_DL "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
set(GGML_BACKEND_DIR "" CACHE PATH "ggml: directory to load dynamic backends from (requires GGML_BACKEND_DL")
#
# option list
#
# TODO: mark all options as advanced when not GGML_STANDALONE
if (APPLE)
set(GGML_METAL_DEFAULT ON)
set(GGML_BLAS_DEFAULT ON)
set(GGML_BLAS_VENDOR_DEFAULT "Apple")
else()
set(GGML_METAL_DEFAULT OFF)
set(GGML_BLAS_DEFAULT OFF)
set(GGML_BLAS_VENDOR_DEFAULT "Generic")
endif()
if (CMAKE_CROSSCOMPILING OR DEFINED ENV{SOURCE_DATE_EPOCH})
message(STATUS "Setting GGML_NATIVE_DEFAULT to OFF")
set(GGML_NATIVE_DEFAULT OFF)
else()
set(GGML_NATIVE_DEFAULT ON)
endif()
# defaults
if (NOT GGML_LLAMAFILE_DEFAULT)
set(GGML_LLAMAFILE_DEFAULT OFF)
endif()
if (NOT GGML_CUDA_GRAPHS_DEFAULT)
set(GGML_CUDA_GRAPHS_DEFAULT OFF)
endif()
# general
option(GGML_STATIC "ggml: static link libraries" OFF)
option(GGML_NATIVE "ggml: optimize the build for the current system" ${GGML_NATIVE_DEFAULT})
option(GGML_LTO "ggml: enable link time optimization" OFF)
option(GGML_CCACHE "ggml: use ccache if available" ON)
# debug
option(GGML_ALL_WARNINGS "ggml: enable all compiler warnings" ON)
option(GGML_ALL_WARNINGS_3RD_PARTY "ggml: enable all compiler warnings in 3rd party libs" OFF)
option(GGML_GPROF "ggml: enable gprof" OFF)
# build
option(GGML_FATAL_WARNINGS "ggml: enable -Werror flag" OFF)
# sanitizers
option(GGML_SANITIZE_THREAD "ggml: enable thread sanitizer" OFF)
option(GGML_SANITIZE_ADDRESS "ggml: enable address sanitizer" OFF)
option(GGML_SANITIZE_UNDEFINED "ggml: enable undefined sanitizer" OFF)
# instruction set specific
if (GGML_NATIVE OR NOT GGML_NATIVE_DEFAULT)
set(INS_ENB OFF)
else()
set(INS_ENB ON)
endif()
message(DEBUG "GGML_NATIVE : ${GGML_NATIVE}")
message(DEBUG "GGML_NATIVE_DEFAULT : ${GGML_NATIVE_DEFAULT}")
message(DEBUG "INS_ENB : ${INS_ENB}")
option(GGML_CPU_HBM "ggml: use memkind for CPU HBM" OFF)
option(GGML_CPU_REPACK "ggml: use runtime weight conversion of Q4_0 to Q4_X_X" ON)
option(GGML_CPU_KLEIDIAI "ggml: use KleidiAI optimized kernels if applicable" OFF)
option(GGML_SSE42 "ggml: enable SSE 4.2" ${INS_ENB})
option(GGML_AVX "ggml: enable AVX" ${INS_ENB})
option(GGML_AVX_VNNI "ggml: enable AVX-VNNI" OFF)
option(GGML_AVX2 "ggml: enable AVX2" ${INS_ENB})
option(GGML_BMI2 "ggml: enable BMI2" ${INS_ENB})
option(GGML_AVX512 "ggml: enable AVX512F" OFF)
option(GGML_AVX512_VBMI "ggml: enable AVX512-VBMI" OFF)
option(GGML_AVX512_VNNI "ggml: enable AVX512-VNNI" OFF)
option(GGML_AVX512_BF16 "ggml: enable AVX512-BF16" OFF)
if (NOT MSVC)
# in MSVC F16C and FMA is implied with AVX2/AVX512
option(GGML_FMA "ggml: enable FMA" ${INS_ENB})
option(GGML_F16C "ggml: enable F16C" ${INS_ENB})
# MSVC does not seem to support AMX
option(GGML_AMX_TILE "ggml: enable AMX-TILE" OFF)
option(GGML_AMX_INT8 "ggml: enable AMX-INT8" OFF)
option(GGML_AMX_BF16 "ggml: enable AMX-BF16" OFF)
endif()
option(GGML_LASX "ggml: enable lasx" ON)
option(GGML_LSX "ggml: enable lsx" ON)
option(GGML_RVV "ggml: enable rvv" ON)
option(GGML_RV_ZFH "ggml: enable riscv zfh" ON)
option(GGML_RV_ZVFH "ggml: enable riscv zvfh" ON)
option(GGML_RV_ZICBOP "ggml: enable riscv zicbop" ON)
option(GGML_RV_ZIHINTPAUSE "ggml: enable riscv zihintpause" ON)
option(GGML_RV_ZVFBFWMA "ggml: enable riscv zvfbfwma" OFF)
option(GGML_XTHEADVECTOR "ggml: enable xtheadvector" OFF)
option(GGML_VXE "ggml: enable vxe" ${GGML_NATIVE})
option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF)
set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM")
set(GGML_CPU_POWERPC_CPUTYPE "" CACHE STRING "ggml: CPU type for PowerPC")
# ggml core
set(GGML_SCHED_MAX_COPIES "4" CACHE STRING "ggml: max input copies for pipeline parallelism")
option(GGML_CPU "ggml: enable CPU backend" ON)
option(GGML_SCHED_NO_REALLOC "ggml: disallow reallocations in ggml-alloc (for debugging)" OFF)
# 3rd party libs / backends
option(GGML_ACCELERATE "ggml: enable Accelerate framework" ON)
option(GGML_BLAS "ggml: use BLAS" ${GGML_BLAS_DEFAULT})
set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING
"ggml: BLAS library vendor")
option(GGML_LLAMAFILE "ggml: use LLAMAFILE" ${GGML_LLAMAFILE_DEFAULT})
option(GGML_CUDA "ggml: use CUDA" OFF)
option(GGML_MUSA "ggml: use MUSA" OFF)
option(GGML_CUDA_FORCE_MMQ "ggml: use mmq kernels instead of cuBLAS" OFF)
option(GGML_CUDA_FORCE_CUBLAS "ggml: always use cuBLAS instead of mmq kernels" OFF)
set (GGML_CUDA_PEER_MAX_BATCH_SIZE "128" CACHE STRING
"ggml: max. batch size for using peer access")
option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copies" OFF)
option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF)
option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON)
option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF)
option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT})
option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON)
set (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING
"ggml: cuda link binary compression mode; requires cuda 12.8+")
set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size")
option(GGML_HIP "ggml: use HIP" OFF)
option(GGML_HIP_GRAPHS "ggml: use HIP graph" ON)
option(GGML_HIP_RCCL "ggml: use ROCm Collective Comm. Library" OFF)
option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON)
option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF)
option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON)
option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF)
option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF)
option(GGML_MUSA_MUDNN_COPY "ggml: enable muDNN for accelerated copy" OFF)
option(GGML_VULKAN "ggml: use Vulkan" OFF)
option(GGML_VULKAN_CHECK_RESULTS "ggml: run Vulkan op checks" OFF)
option(GGML_VULKAN_DEBUG "ggml: enable Vulkan debug output" OFF)
option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug output" OFF)
option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF)
option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF)
option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF)
option(GGML_WEBGPU "ggml: use WebGPU" OFF)
option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF)
option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF)
option(GGML_WEBGPU_GPU_PROFILE "ggml: enable WebGPU profiling (GPU)" OFF)
option(GGML_WEBGPU_JSPI "ggml: use JSPI for WebGPU" ON)
option(GGML_ZDNN "ggml: use zDNN" OFF)
option(GGML_VIRTGPU "ggml: use the VirtGPU/Virglrenderer API Remoting frontend" OFF)
option(GGML_VIRTGPU_BACKEND "ggml: build the VirtGPU/Virglrenderer API Remoting backend" OFF)
option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT})
option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF)
option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF)
option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" ${GGML_METAL})
set (GGML_METAL_MACOSX_VERSION_MIN "" CACHE STRING
"ggml: metal minimum macOS version")
set (GGML_METAL_STD "" CACHE STRING "ggml: metal standard version (-std flag)")
option(GGML_OPENMP "ggml: use OpenMP" ON)
option(GGML_RPC "ggml: use RPC" OFF)
option(GGML_SYCL "ggml: use SYCL" OFF)
option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF)
option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON)
option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON)
option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON)
option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON)
set (GGML_SYCL_TARGET "INTEL" CACHE STRING
"ggml: sycl target device")
set (GGML_SYCL_DEVICE_ARCH "" CACHE STRING
"ggml: sycl device architecture")
option(GGML_OPENVINO "ggml: use OPENVINO" OFF)
option(GGML_OPENCL "ggml: use OpenCL" OFF)
option(GGML_OPENCL_PROFILING "ggml: use OpenCL profiling (increases overhead)" OFF)
option(GGML_OPENCL_EMBED_KERNELS "ggml: embed kernels" ON)
option(GGML_OPENCL_USE_ADRENO_KERNELS "ggml: use optimized kernels for Adreno" ON)
set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
"ggml: OpenCL API version to target")
option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF)
set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)")
# toolchain for vulkan-shaders-gen
set (GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "" CACHE FILEPATH "ggml: toolchain file for vulkan-shaders-gen")
option(GGML_ZENDNN "ggml: use ZenDNN" OFF)
option(ZENDNN_ROOT "ggml: path to ZenDNN installation" "")
# extra artifacts
option(GGML_BUILD_TESTS "ggml: build tests" ${GGML_STANDALONE})
option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
#
# dependencies
#
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_STANDARD_REQUIRED true)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED true)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include(GNUInstallDirs)
#
# build the library
#
add_subdirectory(src)
#
# tests and examples
#
if (GGML_BUILD_TESTS)
enable_testing()
add_subdirectory(tests)
endif ()
if (GGML_BUILD_EXAMPLES)
add_subdirectory(examples)
endif ()
#
# install
#
include(CMakePackageConfigHelpers)
# all public headers
set(GGML_PUBLIC_HEADERS
include/ggml.h
include/ggml-cpu.h
include/ggml-alloc.h
include/ggml-backend.h
include/ggml-blas.h
include/ggml-cann.h
include/ggml-cpp.h
include/ggml-cuda.h
include/ggml-opt.h
include/ggml-metal.h
include/ggml-rpc.h
include/ggml-virtgpu.h
include/ggml-sycl.h
include/ggml-vulkan.h
include/ggml-webgpu.h
include/ggml-zendnn.h
include/ggml-openvino.h
include/gguf.h)
set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
#if (GGML_METAL)
# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
#endif()
install(TARGETS ggml LIBRARY PUBLIC_HEADER)
install(TARGETS ggml-base LIBRARY)
if (GGML_STANDALONE)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ggml.pc.in
${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
@ONLY)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml.pc
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
endif()
#
# Create CMake package
#
# Capture variables prefixed with GGML_.
set(variable_set_statements
"
####### Expanded from @GGML_VARIABLES_EXPANED@ by configure_package_config_file() #######
####### Any changes to this file will be overwritten by the next CMake run #######
")
set(GGML_SHARED_LIB ${BUILD_SHARED_LIBS})
get_cmake_property(all_variables VARIABLES)
foreach(variable_name IN LISTS all_variables)
if(variable_name MATCHES "^GGML_")
string(REPLACE ";" "\\;"
variable_value "${${variable_name}}")
set(variable_set_statements
"${variable_set_statements}set(${variable_name} \"${variable_value}\")\n")
endif()
endforeach()
set(GGML_VARIABLES_EXPANDED ${variable_set_statements})
# Create the CMake package and set install location.
set(GGML_INSTALL_VERSION ${GGML_VERSION})
set(GGML_INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR} CACHE PATH "Location of header files")
set(GGML_LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE PATH "Location of library files")
set(GGML_BIN_INSTALL_DIR ${CMAKE_INSTALL_BINDIR} CACHE PATH "Location of binary files")
configure_package_config_file(
${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml-config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml
PATH_VARS GGML_INCLUDE_INSTALL_DIR
GGML_LIB_INSTALL_DIR
GGML_BIN_INSTALL_DIR)
write_basic_package_version_file(
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
VERSION ${GGML_INSTALL_VERSION}
COMPATIBILITY SameMajorVersion)
target_compile_definitions(ggml-base PRIVATE
GGML_VERSION="${GGML_INSTALL_VERSION}"
GGML_COMMIT="${GGML_BUILD_COMMIT}"
)
message(STATUS "ggml version: ${GGML_INSTALL_VERSION}")
message(STATUS "ggml commit: ${GGML_BUILD_COMMIT}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/ggml-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/ggml-version.cmake
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ggml)
if (MSVC)
set(MSVC_WARNING_FLAGS
/wd4005 # Macro redefinition
/wd4244 # Conversion from one type to another type, possible loss of data
/wd4267 # Conversion from 'size_t' to a smaller type, possible loss of data
/wd4305 # Conversion from 'type1' to 'type2', possible loss of data
/wd4566 # Conversion from 'char' to 'wchar_t', possible loss of data
/wd4996 # Disable POSIX deprecation warnings
/wd4702 # Unreachable code warnings
)
set(MSVC_COMPILE_OPTIONS
"$<$<COMPILE_LANGUAGE:C>:/utf-8>"
"$<$<COMPILE_LANGUAGE:CXX>:/utf-8>"
)
function(configure_msvc_target target_name)
if(TARGET ${target_name})
target_compile_options(${target_name} PRIVATE ${MSVC_WARNING_FLAGS})
target_compile_options(${target_name} PRIVATE ${MSVC_COMPILE_OPTIONS})
endif()
endfunction()
configure_msvc_target(ggml-base)
configure_msvc_target(ggml)
configure_msvc_target(ggml-cpu)
configure_msvc_target(ggml-cpu-x64)
configure_msvc_target(ggml-cpu-sse42)
configure_msvc_target(ggml-cpu-sandybridge)
# __FMA__ and __F16C__ are not defined in MSVC, however they are implied with AVX2/AVX512
# skipping ggml-cpu-ivybridge
# skipping ggml-cpu-piledriver
configure_msvc_target(ggml-cpu-haswell)
configure_msvc_target(ggml-cpu-skylakex)
configure_msvc_target(ggml-cpu-cannonlake)
configure_msvc_target(ggml-cpu-cascadelake)
configure_msvc_target(ggml-cpu-icelake)
# MSVC 2022 doesn't support BF16 intrinsics without `/arch:AVX10.1` ?!
# https://learn.microsoft.com/en-us/cpp/intrinsics/x64-amd64-intrinsics-list?view=msvc-170
# https://learn.microsoft.com/en-us/cpp/build/reference/arch-x64?view=msvc-170
# skipping ggml-cpu-cooperlake
# skipping ggml-cpu-zen4
configure_msvc_target(ggml-cpu-alderlake)
# MSVC doesn't support AMX
# skipping ggml-cpu-sapphirerapids
if (GGML_BUILD_EXAMPLES)
configure_msvc_target(common-ggml)
configure_msvc_target(common)
configure_msvc_target(mnist-common)
configure_msvc_target(mnist-eval)
configure_msvc_target(mnist-train)
configure_msvc_target(gpt-2-ctx)
configure_msvc_target(gpt-2-alloc)
configure_msvc_target(gpt-2-backend)
configure_msvc_target(gpt-2-sched)
configure_msvc_target(gpt-2-quantize)
configure_msvc_target(gpt-2-batched)
configure_msvc_target(gpt-j)
configure_msvc_target(gpt-j-quantize)
configure_msvc_target(magika)
configure_msvc_target(yolov3-tiny)
configure_msvc_target(sam)
configure_msvc_target(simple-ctx)
configure_msvc_target(simple-backend)
endif()
if (GGML_BUILD_TESTS)
configure_msvc_target(test-mul-mat)
configure_msvc_target(test-arange)
configure_msvc_target(test-backend-ops)
configure_msvc_target(test-cont)
configure_msvc_target(test-conv-transpose)
configure_msvc_target(test-conv-transpose-1d)
configure_msvc_target(test-conv1d)
configure_msvc_target(test-conv2d)
configure_msvc_target(test-conv2d-dw)
configure_msvc_target(test-customop)
configure_msvc_target(test-dup)
configure_msvc_target(test-opt)
configure_msvc_target(test-pool)
endif ()
endif()
engine/ggml/CONTRIBUTING.md
+3
View File
@@ -0,0 +1,3 @@
Please use [llama.cpp's contribution guidelines](https://github.com/ggml-org/llama.cpp/blob/master/CONTRIBUTING.md) for this project.
*For changes to the core `ggml` library (including to the CMake build system), please open a PR in https://github.com/ggml-org/llama.cpp. Doing so will make your PR more visible, better tested and more likely to be reviewed.*
engine/ggml/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023-2026 The ggml authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
engine/ggml/README.md
+50
View File
@@ -0,0 +1,50 @@
# ggml
[Manifesto](https://github.com/ggerganov/llama.cpp/discussions/205)
Tensor library for machine learning
***Note that this project is under active development. \
Some of the development is currently happening in the [llama.cpp](https://github.com/ggerganov/llama.cpp) and [whisper.cpp](https://github.com/ggerganov/whisper.cpp) repos***
## Features
- Low-level cross-platform implementation
- Integer quantization support
- Broad hardware support
- Automatic differentiation
- ADAM and L-BFGS optimizers
- No third-party dependencies
- Zero memory allocations during runtime
## Build
```bash
git clone https://github.com/ggml-org/ggml
cd ggml
# install python dependencies in a virtual environment
python3.10 -m venv .venv
source .venv/bin/activate
pip install -r requirements.txt
# build the examples
mkdir build && cd build
cmake ..
cmake --build . --config Release -j 8
```
## GPT inference (example)
```bash
# run the GPT-2 small 117M model
../examples/gpt-2/download-ggml-model.sh 117M
./bin/gpt-2-backend -m models/gpt-2-117M/ggml-model.bin -p "This is an example"
```
For more information, checkout the corresponding programs in the [examples](examples) folder.
## Resources
- [Introduction to ggml](https://huggingface.co/blog/introduction-to-ggml)
- [The GGUF file format](https://github.com/ggerganov/ggml/blob/master/docs/gguf.md)
engine/ggml/ci/run.sh
+395
View File
@@ -0,0 +1,395 @@
#/bin/bash
#
# sample usage:
#
# mkdir tmp
#
# # CPU-only build
# bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
# # with CUDA support
# GG_BUILD_CUDA=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
# # With SYCL support
# GG_BUILD_SYCL=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
#
if [ -z "$2" ]; then
echo "usage: $0 <output-dir> <mnt-dir>"
exit 1
fi
mkdir -p "$1"
mkdir -p "$2"
OUT=$(realpath "$1")
MNT=$(realpath "$2")
rm -v $OUT/*.log
rm -v $OUT/*.exit
rm -v $OUT/*.md
sd=`dirname $0`
cd $sd/../
SRC=`pwd`
CMAKE_EXTRA=""
CTEST_EXTRA=""
if [ ! -z ${GG_BUILD_METAL} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=ON"
fi
if [ ! -z ${GG_BUILD_CUDA} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_CUDA=ON"
if command -v nvidia-smi >/dev/null 2>&1; then
CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d '.')
if [[ -n "$CUDA_ARCH" && "$CUDA_ARCH" =~ ^[0-9]+$ ]]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCH}"
else
echo "Warning: Using fallback CUDA architectures"
CMAKE_EXTRA="${CMAKE_EXTRA} -DCMAKE_CUDA_ARCHITECTURES=61;70;75;80;86;89"
fi
else
echo "Error: nvidia-smi not found, cannot build with CUDA"
exit 1
fi
fi
if [ ! -z ${GG_BUILD_ROCM} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_HIP=ON"
if [ -z ${GG_BUILD_AMDGPU_TARGETS} ]; then
echo "Missing GG_BUILD_AMDGPU_TARGETS, please set it to your GPU architecture (e.g. gfx90a, gfx1100, etc.)"
exit 1
fi
CMAKE_EXTRA="${CMAKE_EXTRA} -DAMDGPU_TARGETS=${GG_BUILD_AMDGPU_TARGETS}"
fi
if [ ! -z ${GG_BUILD_SYCL} ]; then
if [ -z ${ONEAPI_ROOT} ]; then
echo "Not detected ONEAPI_ROOT, please install oneAPI base toolkit and enable it by:"
echo "source /opt/intel/oneapi/setvars.sh"
exit 1
fi
# Use only main GPU
export ONEAPI_DEVICE_SELECTOR="level_zero:0"
# Enable sysman for correct memory reporting
export ZES_ENABLE_SYSMAN=1
# to circumvent precision issues on CPY operations
export SYCL_PROGRAM_COMPILE_OPTIONS="-cl-fp32-correctly-rounded-divide-sqrt"
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_SYCL=1 -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON"
fi
if [ ! -z ${GG_BUILD_VULKAN} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN=1"
# if on Mac, disable METAL
if [[ "$OSTYPE" == "darwin"* ]]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_METAL=OFF -DGGML_BLAS=OFF"
fi
fi
if [ ! -z ${GG_BUILD_WEBGPU} ]; then
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_WEBGPU=1"
fi
if [ ! -z ${GG_BUILD_MUSA} ]; then
# Use qy1 by default (MTT S80)
MUSA_ARCH=${MUSA_ARCH:-21}
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_MUSA=ON -DMUSA_ARCHITECTURES=${MUSA_ARCH}"
fi
if [ ! -z ${GG_BUILD_NO_SVE} ]; then
# arm 9 and newer enables sve by default, adjust these flags depending on the cpu used
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm"
fi
## helpers
# download a file if it does not exist or if it is outdated
function gg_wget {
local out=$1
local url=$2
local cwd=`pwd`
mkdir -p $out
cd $out
# should not re-download if file is the same
wget -nv -N $url
cd $cwd
}
function gg_printf {
printf -- "$@" >> $OUT/README.md
}
function gg_run {
ci=$1
set -o pipefail
set -x
gg_run_$ci | tee $OUT/$ci.log
cur=$?
echo "$cur" > $OUT/$ci.exit
set +x
set +o pipefail
gg_sum_$ci
ret=$((ret | cur))
}
## ci
# ctest_debug
function gg_run_ctest_debug {
cd ${SRC}
rm -rf build-ci-debug && mkdir build-ci-debug && cd build-ci-debug
set -e
(time cmake -DCMAKE_BUILD_TYPE=Debug ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
(time ctest ${CTEST_EXTRA} --output-on-failure -E "test-opt|test-backend-ops" ) 2>&1 | tee -a $OUT/${ci}-ctest.log
set +e
}
function gg_sum_ctest_debug {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs ctest in debug mode\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
gg_printf '```\n'
gg_printf '\n'
}
# ctest_release
function gg_run_ctest_release {
cd ${SRC}
rm -rf build-ci-release && mkdir build-ci-release && cd build-ci-release
set -e
(time cmake -DCMAKE_BUILD_TYPE=Release ${CMAKE_EXTRA} .. ) 2>&1 | tee -a $OUT/${ci}-cmake.log
(time make -j$(nproc) ) 2>&1 | tee -a $OUT/${ci}-make.log
if [ -z $GG_BUILD_LOW_PERF ]; then
(time ctest ${CTEST_EXTRA} --output-on-failure ) 2>&1 | tee -a $OUT/${ci}-ctest.log
else
(time ctest ${CTEST_EXTRA} --output-on-failure -E test-opt ) 2>&1 | tee -a $OUT/${ci}-ctest.log
fi
set +e
}
function gg_sum_ctest_release {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs ctest in release mode\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-ctest.log)"
gg_printf '```\n'
}
# gpt_2
function gg_run_gpt_2 {
cd ${SRC}
gg_wget models-mnt/gpt-2 https://huggingface.co/ggerganov/ggml/resolve/main/ggml-model-gpt-2-117M.bin
cd build-ci-release
set -e
model="../models-mnt/gpt-2/ggml-model-gpt-2-117M.bin"
prompts="../examples/prompts/gpt-2.txt"
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -tt ${prompts} ) 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-backend --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-sched --model ${model} -s 1234 -n 64 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
(time ./bin/gpt-2-batched --model ${model} -s 1234 -n 64 -np 8 -p "I believe the meaning of life is") 2>&1 | tee -a $OUT/${ci}-tg.log
set +e
}
function gg_sum_gpt_2 {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Runs short GPT-2 text generation\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-tg.log)"
gg_printf '```\n'
}
# TODO: update
## mnist
#
#function gg_run_mnist {
# cd ${SRC}
#
# cd build-ci-release
#
# set -e
#
# mkdir -p models/mnist
# python3 ../examples/mnist/convert-h5-to-ggml.py ../examples/mnist/models/mnist/mnist_model.state_dict
#
# model_f32="./models/mnist/ggml-model-f32.bin"
# samples="../examples/mnist/models/mnist/t10k-images.idx3-ubyte"
#
# # first command runs and exports "mnist.ggml", the second command runs the exported model
#
# (time ./bin/mnist ${model_f32} ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
# (time ./bin/mnist-cpu ./mnist.ggml ${samples} ) 2>&1 | tee -a $OUT/${ci}-mnist.log
#
# set +e
#}
#
#function gg_sum_mnist {
# gg_printf '### %s\n\n' "${ci}"
#
# gg_printf 'MNIST\n'
# gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
# gg_printf '```\n'
# gg_printf '%s\n' "$(cat $OUT/${ci}-mnist.log)"
# gg_printf '```\n'
#}
# sam
function gg_run_sam {
cd ${SRC}
gg_wget models-mnt/sam/ https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth
gg_wget models-mnt/sam/ https://raw.githubusercontent.com/YavorGIvanov/sam.cpp/ceafb7467bff7ec98e0c4f952e58a9eb8fd0238b/img.jpg
cd build-ci-release
set -e
path_models="../models-mnt/sam/"
model_f16="${path_models}/ggml-model-f16.bin"
img_0="${path_models}/img.jpg"
python3 ../examples/sam/convert-pth-to-ggml.py ${path_models}/sam_vit_b_01ec64.pth ${path_models}/ 1
# Test default parameters
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 ) 2>&1 | tee -a $OUT/${ci}-main.log
grep -q "point prompt" $OUT/${ci}-main.log
grep -q "bbox (371, 436), (144, 168)" $OUT/${ci}-main.log ||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
# Test box prompt and single mask output
(time ./bin/sam -m ${model_f16} -i ${img_0} -st 0.925 -b 368,144,441,173 -sm) 2>&1 | tee -a $OUT/${ci}-main.log
grep -q "box prompt" $OUT/${ci}-main.log
grep -q "bbox (370, 439), (144, 169)" $OUT/${ci}-main.log ||
grep -q "bbox (370, 439), (144, 168)" $OUT/${ci}-main.log
set +e
}
function gg_sum_sam {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Run SAM\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
gg_printf '```\n'
}
# yolo
function gg_run_yolo {
cd ${SRC}
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/yolov3-tiny.weights
gg_wget models-mnt/yolo/ https://huggingface.co/ggml-org/models/resolve/main/yolo/dog.jpg
cd build-ci-release
cp -r ../examples/yolo/data .
set -e
path_models="../models-mnt/yolo/"
python3 ../examples/yolo/convert-yolov3-tiny.py ${path_models}/yolov3-tiny.weights
(time ./bin/yolov3-tiny -m yolov3-tiny.gguf -i ${path_models}/dog.jpg ) 2>&1 | tee -a $OUT/${ci}-main.log
grep -qE "dog: (55|56|57|58|59)%" $OUT/${ci}-main.log
grep -qE "car: (50|51|52|53|54)%" $OUT/${ci}-main.log
grep -qE "truck: (54|55|56|57|58)%" $OUT/${ci}-main.log
grep -qE "bicycle: (57|58|59|60|61)%" $OUT/${ci}-main.log
set +e
}
function gg_sum_yolo {
gg_printf '### %s\n\n' "${ci}"
gg_printf 'Run YOLO\n'
gg_printf '- status: %s\n' "$(cat $OUT/${ci}.exit)"
gg_printf '```\n'
gg_printf '%s\n' "$(cat $OUT/${ci}-main.log)"
gg_printf '```\n'
}
## main
if true ; then
# Create symlink: ./ggml/models-mnt -> $MNT/models/models-mnt
rm -rf ${SRC}/models-mnt
mnt_models=${MNT}/models
mkdir -p ${mnt_models}
ln -sfn ${mnt_models} ${SRC}/models-mnt
# Create a fresh python3 venv and enter it
if ! python3 -m venv "$MNT/venv"; then
echo "Error: Failed to create Python virtual environment at $MNT/venv."
exit 1
fi
source "$MNT/venv/bin/activate"
pip install -r ${SRC}/requirements.txt --disable-pip-version-check
fi
ret=0
test $ret -eq 0 && gg_run ctest_debug
test $ret -eq 0 && gg_run ctest_release
test $ret -eq 0 && gg_run gpt_2
#test $ret -eq 0 && gg_run mnist
test $ret -eq 0 && gg_run sam
test $ret -eq 0 && gg_run yolo
if [ -z $GG_BUILD_LOW_PERF ]; then
# run tests meant for low-perf runners
date
fi
cat $OUT/README.md
exit $ret
engine/ggml/cmake/FindNCCL.cmake
+36
View File
@@ -0,0 +1,36 @@
# cmake/FindNCCL.cmake
# NVIDIA does not distribute CMake files with NCCl, therefore use this file to find it instead.
find_path(NCCL_INCLUDE_DIR
NAMES nccl.h
HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda
PATH_SUFFIXES include
)
find_library(NCCL_LIBRARY
NAMES nccl
HINTS ${NCCL_ROOT} $ENV{NCCL_ROOT} $ENV{CUDA_HOME} /usr/local/cuda
PATH_SUFFIXES lib lib64
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(NCCL
DEFAULT_MSG
NCCL_LIBRARY NCCL_INCLUDE_DIR
)
if(NCCL_FOUND)
set(NCCL_LIBRARIES ${NCCL_LIBRARY})
set(NCCL_INCLUDE_DIRS ${NCCL_INCLUDE_DIR})
if(NOT TARGET NCCL::NCCL)
add_library(NCCL::NCCL UNKNOWN IMPORTED)
set_target_properties(NCCL::NCCL PROPERTIES
IMPORTED_LOCATION "${NCCL_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}"
)
endif()
endif()
mark_as_advanced(NCCL_INCLUDE_DIR NCCL_LIBRARY)
engine/ggml/cmake/GitVars.cmake
+22
View File
@@ -0,0 +1,22 @@
find_package(Git)
# the commit's SHA1
execute_process(COMMAND
"${GIT_EXECUTABLE}" describe --match=NeVeRmAtCh --always --abbrev=8
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_SHA1
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
# the date of the commit
execute_process(COMMAND
"${GIT_EXECUTABLE}" log -1 --format=%ad --date=local
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_DATE
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
# the subject of the commit
execute_process(COMMAND
"${GIT_EXECUTABLE}" log -1 --format=%s
WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
OUTPUT_VARIABLE GIT_COMMIT_SUBJECT
ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
engine/ggml/cmake/common.cmake
+50
View File
@@ -0,0 +1,50 @@
function(ggml_get_flags CCID CCVER)
set(C_FLAGS "")
set(CXX_FLAGS "")
if (CCID MATCHES "Clang")
set(C_FLAGS -Wunreachable-code-break -Wunreachable-code-return)
set(CXX_FLAGS -Wunreachable-code-break -Wunreachable-code-return -Wmissing-prototypes -Wextra-semi)
if (
(CCID STREQUAL "Clang" AND CCVER VERSION_GREATER_EQUAL 3.8.0) OR
(CCID STREQUAL "AppleClang" AND CCVER VERSION_GREATER_EQUAL 7.3.0)
)
list(APPEND C_FLAGS -Wdouble-promotion)
endif()
elseif (CCID STREQUAL "GNU")
set(C_FLAGS -Wdouble-promotion)
set(CXX_FLAGS -Wno-array-bounds)
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
list(APPEND CXX_FLAGS -Wextra-semi)
endif()
endif()
set(GF_C_FLAGS ${C_FLAGS} PARENT_SCOPE)
set(GF_CXX_FLAGS ${CXX_FLAGS} PARENT_SCOPE)
endfunction()
function(ggml_get_system_arch)
if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE)
elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR
CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
(NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64|amd64)$"))
set(GGML_SYSTEM_ARCH "x86" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc|power")
set(GGML_SYSTEM_ARCH "PowerPC" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
set(GGML_SYSTEM_ARCH "loongarch64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64")
set(GGML_SYSTEM_ARCH "riscv64" PARENT_SCOPE)
elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x")
set(GGML_SYSTEM_ARCH "s390x" PARENT_SCOPE)
else()
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
endif()
endfunction()
engine/ggml/cmake/ggml-config.cmake.in
+201
View File
@@ -0,0 +1,201 @@
@PACKAGE_INIT@
@GGML_VARIABLES_EXPANDED@
# Find all dependencies before creating any target.
include(CMakeFindDependencyMacro)
find_dependency(Threads)
if (NOT GGML_SHARED_LIB)
set(GGML_BASE_INTERFACE_LINK_LIBRARIES "")
set(GGML_CPU_INTERFACE_LINK_LIBRARIES "")
set(GGML_CPU_INTERFACE_LINK_OPTIONS "")
if (APPLE AND GGML_ACCELERATE)
find_library(ACCELERATE_FRAMEWORK Accelerate)
if(NOT ACCELERATE_FRAMEWORK)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${ACCELERATE_FRAMEWORK})
endif()
if (GGML_OPENMP_ENABLED)
find_dependency(OpenMP)
set(GGML_OPENMP_INTERFACE_LINK_LIBRARIES "")
if (TARGET OpenMP::OpenMP_C)
list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_C)
endif()
if (TARGET OpenMP::OpenMP_CXX)
list(APPEND GGML_OPENMP_INTERFACE_LINK_LIBRARIES OpenMP::OpenMP_CXX)
endif()
list(APPEND GGML_BASE_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES})
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES ${GGML_OPENMP_INTERFACE_LINK_LIBRARIES})
endif()
if (GGML_CPU_HBM)
find_library(memkind memkind)
if(NOT memkind)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES memkind)
endif()
if (GGML_BLAS)
find_dependency(BLAS)
list(APPEND GGML_BLAS_INTERFACE_LINK_LIBRARIES ${BLAS_LIBRARIES})
list(APPEND GGML_BLAS_INTERFACE_LINK_OPTIONS ${BLAS_LINKER_FLAGS})
endif()
if (GGML_CUDA)
set(GGML_CUDA_INTERFACE_LINK_LIBRARIES "")
find_dependency(CUDAToolkit)
if (GGML_STATIC)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cudart_static>)
if (WIN32)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas> $<LINK_ONLY:CUDA::cublasLt>)
else()
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cublas_static> $<LINK_ONLY:CUDA::cublasLt_static>)
endif()
endif()
if (NOT GGML_CUDA_NO_VMM)
list(APPEND GGML_CUDA_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:CUDA::cuda_driver>)
endif()
endif()
if (GGML_METAL)
find_library(FOUNDATION_LIBRARY Foundation)
find_library(METAL_FRAMEWORK Metal)
find_library(METALKIT_FRAMEWORK MetalKit)
if(NOT FOUNDATION_LIBRARY OR NOT METAL_FRAMEWORK OR NOT METALKIT_FRAMEWORK)
set(${CMAKE_FIND_PACKAGE_NAME}_FOUND 0)
return()
endif()
set(GGML_METAL_INTERFACE_LINK_LIBRARIES
${FOUNDATION_LIBRARY} ${METAL_FRAMEWORK} ${METALKIT_FRAMEWORK})
endif()
if (GGML_OPENCL)
find_dependency(OpenCL)
set(GGML_OPENCL_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:OpenCL::OpenCL>)
endif()
if (GGML_VULKAN)
find_dependency(Vulkan)
set(GGML_VULKAN_INTERFACE_LINK_LIBRARIES $<LINK_ONLY:Vulkan::Vulkan>)
endif()
if (GGML_HIP)
find_dependency(hip)
find_dependency(hipblas)
find_dependency(rocblas)
set(GGML_HIP_INTERFACE_LINK_LIBRARIES hip::host roc::rocblas roc::hipblas)
endif()
if (GGML_SYCL)
set(GGML_SYCL_INTERFACE_LINK_LIBRARIES "")
find_package(DNNL)
if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL")
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES DNNL::dnnl)
endif()
if (WIN32)
find_dependency(IntelSYCL)
find_dependency(MKL)
list(APPEND GGML_SYCL_INTERFACE_LINK_LIBRARIES IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
endif()
endif()
endif()
set_and_check(GGML_INCLUDE_DIR "@PACKAGE_GGML_INCLUDE_INSTALL_DIR@")
set_and_check(GGML_LIB_DIR "@PACKAGE_GGML_LIB_INSTALL_DIR@")
#set_and_check(GGML_BIN_DIR "@PACKAGE_GGML_BIN_INSTALL_DIR@")
if(NOT TARGET ggml::ggml)
find_package(Threads REQUIRED)
find_library(GGML_LIBRARY ggml
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
add_library(ggml::ggml UNKNOWN IMPORTED)
set_target_properties(ggml::ggml
PROPERTIES
IMPORTED_LOCATION "${GGML_LIBRARY}")
find_library(GGML_BASE_LIBRARY ggml-base
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
add_library(ggml::ggml-base UNKNOWN IMPORTED)
set_target_properties(ggml::ggml-base
PROPERTIES
IMPORTED_LOCATION "${GGML_BASE_LIBRARY}"
INTERFACE_LINK_LIBRARIES "${GGML_BASE_INTERFACE_LINK_LIBRARIES}")
set(_ggml_all_targets "")
if (NOT GGML_BACKEND_DL)
foreach(_ggml_backend ${GGML_AVAILABLE_BACKENDS})
string(REPLACE "-" "_" _ggml_backend_pfx "${_ggml_backend}")
string(TOUPPER "${_ggml_backend_pfx}" _ggml_backend_pfx)
find_library(${_ggml_backend_pfx}_LIBRARY ${_ggml_backend}
REQUIRED
HINTS ${GGML_LIB_DIR}
NO_CMAKE_FIND_ROOT_PATH)
message(STATUS "Found ${${_ggml_backend_pfx}_LIBRARY}")
add_library(ggml::${_ggml_backend} UNKNOWN IMPORTED)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${GGML_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${${_ggml_backend_pfx}_LIBRARY}"
INTERFACE_COMPILE_FEATURES c_std_90
POSITION_INDEPENDENT_CODE ON)
string(REGEX MATCH "^ggml-cpu" is_cpu_variant "${_ggml_backend}")
if(is_cpu_variant)
list(APPEND GGML_CPU_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_LIBRARIES "${GGML_CPU_INTERFACE_LINK_LIBRARIES}")
if(GGML_CPU_INTERFACE_LINK_OPTIONS)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_OPTIONS "${GGML_CPU_INTERFACE_LINK_OPTIONS}")
endif()
else()
list(APPEND ${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES "ggml::ggml-base")
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_LIBRARIES "${${_ggml_backend_pfx}_INTERFACE_LINK_LIBRARIES}")
if(${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS)
set_target_properties(ggml::${_ggml_backend}
PROPERTIES
INTERFACE_LINK_OPTIONS "${${_ggml_backend_pfx}_INTERFACE_LINK_OPTIONS}")
endif()
endif()
list(APPEND _ggml_all_targets ggml::${_ggml_backend})
endforeach()
endif()
list(APPEND GGML_INTERFACE_LINK_LIBRARIES ggml::ggml-base "${_ggml_all_targets}")
set_target_properties(ggml::ggml
PROPERTIES
INTERFACE_LINK_LIBRARIES "${GGML_INTERFACE_LINK_LIBRARIES}")
add_library(ggml::all INTERFACE IMPORTED)
set_target_properties(ggml::all
PROPERTIES
INTERFACE_LINK_LIBRARIES "${_ggml_all_targets}")
endif()
check_required_components(ggml)
engine/ggml/docs/gguf.md
+828
View File
@@ -0,0 +1,828 @@
# GGUF
GGUF is a file format for storing models for inference with GGML and executors based on GGML. GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML.
It is a successor file format to GGML, GGMF and GGJT, and is designed to be unambiguous by containing all the information needed to load a model. It is also designed to be extensible, so that new information can be added to models without breaking compatibility.
For more information about the motivation behind GGUF, see [Historical State of Affairs](#historical-state-of-affairs).
## Specification
GGUF is a format based on the existing GGJT, but makes a few changes to the format to make it more extensible and easier to use. The following features are desired:
- Single-file deployment: they can be easily distributed and loaded, and do not require any external files for additional information.
- Extensible: new features can be added to GGML-based executors/new information can be added to GGUF models without breaking compatibility with existing models.
- `mmap` compatibility: models can be loaded using `mmap` for fast loading and saving.
- Easy to use: models can be easily loaded and saved using a small amount of code, with no need for external libraries, regardless of the language used.
- Full information: all information needed to load a model is contained in the model file, and no additional information needs to be provided by the user.
The key difference between GGJT and GGUF is the use of a key-value structure for the hyperparameters (now referred to as metadata), rather than a list of untyped values. This allows for new metadata to be added without breaking compatibility with existing models, and to annotate the model with additional information that may be useful for inference or for identifying the model.
### GGUF Naming Convention
GGUF follow a naming convention of `[<Sidecar>]<BaseName><SizeLabel><FineTune><Version><Encoding><Type><Shard>.gguf` where each component is delimitated by a `-` if present. Ultimately this is intended to make it easier for humans to at a glance get the most important details of a model. It is not intended to be perfectly parsable in the field due to the diversity of existing gguf filenames.
The components are:
1. **Sidecar**: (Optional) Prefix marking the file as an auxiliary module loaded alongside a base model, rather than a standalone model. When present, sits at the very front of the filename followed by `-`. Lowercase by convention.
- `mmproj` : Multimodal projector (vision/audio encoder and projection layer for use with a base LLM)
- `mtp` : Multi-Token Prediction heads (speculative-decoding draft module, intended to be loaded alongside a base model of matching architecture and version). Note that oftentimes the MTP weights can be distributed inside the base model, in which case there is no separate `mtp-` sidecar file.
1. **BaseName**: A descriptive name for the model base type or architecture.
- This can be derived from gguf metadata `general.basename` substituting spaces for dashes.
1. **SizeLabel**: Parameter weight class (useful for leader boards) represented as `<expertCount>x<count><scale-prefix>`
- This can be derived from gguf metadata `general.size_label` if available or calculated if missing.
- Rounded decimal point is supported in count with a single letter scale prefix to assist in floating point exponent shown below
- `Q`: Quadrillion parameters.
- `T`: Trillion parameters.
- `B`: Billion parameters.
- `M`: Million parameters.
- `K`: Thousand parameters.
- Additional `-<attributes><count><scale-prefix>` can be appended as needed to indicate other attributes of interest
1. **FineTune**: A descriptive name for the model fine tuning goal (e.g. Chat, Instruct, etc...)
- This can be derived from gguf metadata `general.finetune` substituting spaces for dashes.
1. **Version**: (Optional) Denotes the model version number, formatted as `v<Major>.<Minor>`
- If model is missing a version number then assume `v1.0` (First Public Release)
- This can be derived from gguf metadata `general.version`
1. **Encoding**: Indicates the weights encoding scheme that was applied to the model. Content, type mixture and arrangement however are determined by user code and can vary depending on project needs.
1. **Type**: Indicates the kind of gguf file and the intended purpose for it
- If missing, then file is by default a typical gguf tensor model file
- `LoRA` : GGUF file is a LoRA adapter
- `vocab` : GGUF file with only vocab data and metadata
1. **Shard**: (Optional) Indicates and denotes that the model has been split into multiple shards, formatted as `<ShardNum>-of-<ShardTotal>`.
- *ShardNum* : Shard position in this model. Must be 5 digits padded by zeros.
- Shard number always starts from `00001` onwards (e.g. First shard always starts at `00001-of-XXXXX` rather than `00000-of-XXXXX`).
- *ShardTotal* : Total number of shards in this model. Must be 5 digits padded by zeros.
#### Validating Above Naming Convention
At a minimum all model files should have at least BaseName, SizeLabel, Version, in order to be easily validated as a file that is keeping with the GGUF Naming Convention. An example of this issue is that it is easy for Encoding to be mistaken as a FineTune if Version is omitted.
To validate you can use this regular expression `^(?:(?<Sidecar>mmproj|mtp)-)?(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$` which will check that you got the minimum BaseName, SizeLabel and Version present in the correct order.
For example:
* `Mixtral-8x7B-v0.1-KQ2.gguf`:
- Model Name: Mixtral
- Expert Count: 8
- Parameter Count: 7B
- Version Number: v0.1
- Weight Encoding Scheme: KQ2
* `Hermes-2-Pro-Llama-3-8B-F16.gguf`:
- Model Name: Hermes 2 Pro Llama 3
- Expert Count: 0
- Parameter Count: 8B
- Version Number: v1.0
- Weight Encoding Scheme: F16
- Shard: N/A
* `Grok-100B-v1.0-Q4_0-00003-of-00009.gguf`
- Model Name: Grok
- Expert Count: 0
- Parameter Count: 100B
- Version Number: v1.0
- Weight Encoding Scheme: Q4_0
- Shard: 3 out of 9 total shards
* `mtp-Qwen3-27B-v1.0-Q4_K_M.gguf`
- Sidecar: mtp (Multi-Token Prediction draft module)
- Model Name: Qwen3
- Expert Count: 0
- Parameter Count: 27B (of the main model — sidecar tensors are smaller)
- Version Number: v1.0
- Weight Encoding Scheme: Q4_K_M
* `mmproj-Qwen2-VL-7B-v1.0-F16.gguf`
- Sidecar: mmproj (multimodal projector)
- Model Name: Qwen2-VL
- Expert Count: 0
- Parameter Count: 7B (of the main model — sidecar tensors are smaller)
- Version Number: v1.0
- Weight Encoding Scheme: F16
<details><summary>Example Node.js Regex Function</summary>
```js
#!/usr/bin/env node
const ggufRegex = /^(?:(?<Sidecar>mmproj|mtp)-)?(?<BaseName>[A-Za-z0-9\s]*(?:(?:-(?:(?:[A-Za-z\s][A-Za-z0-9\s]*)|(?:[0-9\s]*)))*))-(?:(?<SizeLabel>(?:\d+x)?(?:\d+\.)?\d+[A-Za-z](?:-[A-Za-z]+(\d+\.)?\d+[A-Za-z]+)?)(?:-(?<FineTune>[A-Za-z0-9\s-]+))?)?-(?:(?<Version>v\d+(?:\.\d+)*))(?:-(?<Encoding>(?!LoRA|vocab)[\w_]+))?(?:-(?<Type>LoRA|vocab))?(?:-(?<Shard>\d{5}-of-\d{5}))?\.gguf$/;
function parseGGUFFilename(filename) {
const match = ggufRegex.exec(filename);
if (!match)
return null;
const {Sidecar = null, BaseName = null, SizeLabel = null, FineTune = null, Version = "v1.0", Encoding = null, Type = null, Shard = null} = match.groups;
return {Sidecar: Sidecar, BaseName: BaseName, SizeLabel: SizeLabel, FineTune: FineTune, Version: Version, Encoding: Encoding, Type: Type, Shard: Shard};
}
const testCases = [
{filename: 'Mixtral-8x7B-v0.1-KQ2.gguf', expected: { Sidecar: null, BaseName: 'Mixtral', SizeLabel: '8x7B', FineTune: null, Version: 'v0.1', Encoding: 'KQ2', Type: null, Shard: null}},
{filename: 'Grok-100B-v1.0-Q4_0-00003-of-00009.gguf', expected: { Sidecar: null, BaseName: 'Grok', SizeLabel: '100B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_0', Type: null, Shard: "00003-of-00009"}},
{filename: 'Hermes-2-Pro-Llama-3-8B-v1.0-F16.gguf', expected: { Sidecar: null, BaseName: 'Hermes-2-Pro-Llama-3', SizeLabel: '8B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},
{filename: 'Phi-3-mini-3.8B-ContextLength4k-instruct-v1.0.gguf', expected: { Sidecar: null, BaseName: 'Phi-3-mini', SizeLabel: '3.8B-ContextLength4k', FineTune: 'instruct', Version: 'v1.0', Encoding: null, Type: null, Shard: null}},
{filename: 'mtp-Qwen3-27B-v1.0-Q4_K_M.gguf', expected: { Sidecar: 'mtp', BaseName: 'Qwen3', SizeLabel: '27B', FineTune: null, Version: 'v1.0', Encoding: 'Q4_K_M', Type: null, Shard: null}},
{filename: 'mmproj-Qwen2-VL-7B-v1.0-F16.gguf', expected: { Sidecar: 'mmproj', BaseName: 'Qwen2-VL', SizeLabel: '7B', FineTune: null, Version: 'v1.0', Encoding: 'F16', Type: null, Shard: null}},
{filename: 'not-a-known-arrangement.gguf', expected: null},
];
testCases.forEach(({ filename, expected }) => {
const result = parseGGUFFilename(filename);
const passed = JSON.stringify(result) === JSON.stringify(expected);
console.log(`${filename}: ${passed ? "PASS" : "FAIL"}`);
if (!passed) {
console.log(result);
console.log(expected);
}
});
```
</details>
### File Structure
![image](https://github.com/ggerganov/ggml/assets/1991296/c3623641-3a1d-408e-bfaf-1b7c4e16aa63)
*diagram by [@mishig25](https://github.com/mishig25) (GGUF v3)*
GGUF files are structured as follows. They use a global alignment specified in the `general.alignment` metadata field, referred to as `ALIGNMENT` below. Where required, the file is padded with `0x00` bytes to the next multiple of `general.alignment`.
Fields, including arrays, are written sequentially without alignment unless otherwise specified.
Models are little-endian by default. They can also come in big-endian for use with big-endian computers; in this case, all values (including metadata values and tensors) will also be big-endian. At the time of writing, there is no way to determine if a model is big-endian; this may be rectified in future versions. If no additional information is provided, assume the model is little-endian.
```c
enum ggml_type: uint32_t {
GGML_TYPE_F32 = 0,
GGML_TYPE_F16 = 1,
GGML_TYPE_Q4_0 = 2,
GGML_TYPE_Q4_1 = 3,
// GGML_TYPE_Q4_2 = 4, support has been removed
// GGML_TYPE_Q4_3 = 5, support has been removed
GGML_TYPE_Q5_0 = 6,
GGML_TYPE_Q5_1 = 7,
GGML_TYPE_Q8_0 = 8,
GGML_TYPE_Q8_1 = 9,
GGML_TYPE_Q2_K = 10,
GGML_TYPE_Q3_K = 11,
GGML_TYPE_Q4_K = 12,
GGML_TYPE_Q5_K = 13,
GGML_TYPE_Q6_K = 14,
GGML_TYPE_Q8_K = 15,
GGML_TYPE_IQ2_XXS = 16,
GGML_TYPE_IQ2_XS = 17,
GGML_TYPE_IQ3_XXS = 18,
GGML_TYPE_IQ1_S = 19,
GGML_TYPE_IQ4_NL = 20,
GGML_TYPE_IQ3_S = 21,
GGML_TYPE_IQ2_S = 22,
GGML_TYPE_IQ4_XS = 23,
GGML_TYPE_I8 = 24,
GGML_TYPE_I16 = 25,
GGML_TYPE_I32 = 26,
GGML_TYPE_I64 = 27,
GGML_TYPE_F64 = 28,
GGML_TYPE_IQ1_M = 29,
GGML_TYPE_BF16 = 30,
// GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
// GGML_TYPE_Q4_0_4_8 = 32,
// GGML_TYPE_Q4_0_8_8 = 33,
GGML_TYPE_TQ1_0 = 34,
GGML_TYPE_TQ2_0 = 35,
// GGML_TYPE_IQ4_NL_4_4 = 36,
// GGML_TYPE_IQ4_NL_4_8 = 37,
// GGML_TYPE_IQ4_NL_8_8 = 38,
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
GGML_TYPE_COUNT = 40,
};
enum gguf_metadata_value_type: uint32_t {
// The value is a 8-bit unsigned integer.
GGUF_METADATA_VALUE_TYPE_UINT8 = 0,
// The value is a 8-bit signed integer.
GGUF_METADATA_VALUE_TYPE_INT8 = 1,
// The value is a 16-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT16 = 2,
// The value is a 16-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT16 = 3,
// The value is a 32-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT32 = 4,
// The value is a 32-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT32 = 5,
// The value is a 32-bit IEEE754 floating point number.
GGUF_METADATA_VALUE_TYPE_FLOAT32 = 6,
// The value is a boolean.
// 1-byte value where 0 is false and 1 is true.
// Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy.
GGUF_METADATA_VALUE_TYPE_BOOL = 7,
// The value is a UTF-8 non-null-terminated string, with length prepended.
GGUF_METADATA_VALUE_TYPE_STRING = 8,
// The value is an array of other values, with the length and type prepended.
///
// Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes.
GGUF_METADATA_VALUE_TYPE_ARRAY = 9,
// The value is a 64-bit unsigned little-endian integer.
GGUF_METADATA_VALUE_TYPE_UINT64 = 10,
// The value is a 64-bit signed little-endian integer.
GGUF_METADATA_VALUE_TYPE_INT64 = 11,
// The value is a 64-bit IEEE754 floating point number.
GGUF_METADATA_VALUE_TYPE_FLOAT64 = 12,
};
// A string in GGUF.
struct gguf_string_t {
// The length of the string, in bytes.
uint64_t len;
// The string as a UTF-8 non-null-terminated string.
char string[len];
};
union gguf_metadata_value_t {
uint8_t uint8;
int8_t int8;
uint16_t uint16;
int16_t int16;
uint32_t uint32;
int32_t int32;
float float32;
uint64_t uint64;
int64_t int64;
double float64;
bool bool_;
gguf_string_t string;
struct {
// Any value type is valid, including arrays.
gguf_metadata_value_type type;
// Number of elements, not bytes
uint64_t len;
// The array of values.
gguf_metadata_value_t array[len];
} array;
};
struct gguf_metadata_kv_t {
// The key of the metadata. It is a standard GGUF string, with the following caveats:
// - It must be a valid ASCII string.
// - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`.
// - It must be at most 2^16-1/65535 bytes long.
// Any keys that do not follow these rules are invalid.
gguf_string_t key;
// The type of the value.
// Must be one of the `gguf_metadata_value_type` values.
gguf_metadata_value_type value_type;
// The value.
gguf_metadata_value_t value;
};
struct gguf_header_t {
// Magic number to announce that this is a GGUF file.
// Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`.
// Your executor might do little-endian byte order, so it might be
// check for 0x46554747 and letting the endianness cancel out.
// Consider being *very* explicit about the byte order here.
uint32_t magic;
// The version of the format implemented.
// Must be `3` for version described in this spec, which introduces big-endian support.
//
// This version should only be increased for structural changes to the format.
// Changes that do not affect the structure of the file should instead update the metadata
// to signify the change.
uint32_t version;
// The number of tensors in the file.
// This is explicit, instead of being included in the metadata, to ensure it is always present
// for loading the tensors.
uint64_t tensor_count;
// The number of metadata key-value pairs.
uint64_t metadata_kv_count;
// The metadata key-value pairs.
gguf_metadata_kv_t metadata_kv[metadata_kv_count];
};
uint64_t align_offset(uint64_t offset) {
return offset + (ALIGNMENT - (offset % ALIGNMENT)) % ALIGNMENT;
}
struct gguf_tensor_info_t {
// The name of the tensor. It is a standard GGUF string, with the caveat that
// it must be at most 64 bytes long.
gguf_string_t name;
// The number of dimensions in the tensor.
// Currently at most 4, but this may change in the future.
uint32_t n_dimensions;
// The dimensions of the tensor.
uint64_t dimensions[n_dimensions];
// The type of the tensor.
ggml_type type;
// The offset of the tensor's data in this file in bytes.
//
// This offset is relative to `tensor_data`, not to the start
// of the file, to make it easier for writers to write the file.
// Readers should consider exposing this offset relative to the
// file to make it easier to read the data.
//
// Must be a multiple of `ALIGNMENT`. That is, `align_offset(offset) == offset`.
uint64_t offset;
};
struct gguf_file_t {
// The header of the file.
gguf_header_t header;
// Tensor infos, which can be used to locate the tensor data.
gguf_tensor_info_t tensor_infos[header.tensor_count];
// Padding to the nearest multiple of `ALIGNMENT`.
//
// That is, if `sizeof(header) + sizeof(tensor_infos)` is not a multiple of `ALIGNMENT`,
// this padding is added to make it so.
//
// This can be calculated as `align_offset(position) - position`, where `position` is
// the position of the end of `tensor_infos` (i.e. `sizeof(header) + sizeof(tensor_infos)`).
uint8_t _padding[];
// Tensor data.
//
// This is arbitrary binary data corresponding to the weights of the model. This data should be close
// or identical to the data in the original model file, but may be different due to quantization or
// other optimizations for inference. Any such deviations should be recorded in the metadata or as
// part of the architecture definition.
//
// Each tensor's data must be stored within this array, and located through its `tensor_infos` entry.
// The offset of each tensor's data must be a multiple of `ALIGNMENT`, and the space between tensors
// should be padded to `ALIGNMENT` bytes.
uint8_t tensor_data[];
};
```
## Standardized key-value pairs
The following key-value pairs are standardized. This list may grow in the future as more use cases are discovered. Where possible, names are shared with the original model definitions to make it easier to map between the two.
Not all of these are required, but they are all recommended. Keys that are required are bolded. For omitted pairs, the reader should assume that the value is unknown and either default or error as appropriate.
The community can develop their own key-value pairs to carry additional data. However, these should be namespaced with the relevant community name to avoid collisions. For example, the `rustformers` community might use `rustformers.` as a prefix for all of their keys.
If a particular community key is widely used, it may be promoted to a standardized key.
By convention, most counts/lengths/etc are `uint64` unless otherwise specified. This is to allow for larger models to be supported in the future. Some models may use `uint32` for their values; it is recommended that readers support both.
### General
#### Required
- **`general.architecture: string`**: describes what architecture this model implements. All lowercase ASCII, with only `[a-z0-9]+` characters allowed. Known values include:
- `llama`
- `mpt`
- `gptneox`
- `gptj`
- `gpt2`
- `bloom`
- `falcon`
- `mamba`
- `rwkv`
- **`general.quantization_version: uint32`**: The version of the quantization format. Not required if the model is not quantized (i.e. no tensors are quantized). If any tensors are quantized, this _must_ be present. This is separate to the quantization scheme of the tensors itself; the quantization version may change without changing the scheme's name (e.g. the quantization scheme is Q5_K, and the quantization version is 4).
- **`general.alignment: uint32`**: the global alignment to use, as described above. This can vary to allow for different alignment schemes, but it must be a multiple of 8. Some writers may not write the alignment. If the alignment is **not** specified, assume it is `32`.
#### General metadata
- `general.name: string`: The name of the model. This should be a human-readable name that can be used to identify the model. It should be unique within the community that the model is defined in.
- `general.author: string`: The author of the model.
- `general.version: string`: The version of the model.
- `general.organization: string`: The organization of the model.
- `general.basename: string`: The base model name / architecture of the model
- `general.finetune: string`: What has the base model been optimized toward.
- `general.description: string`: free-form description of the model including anything that isn't covered by the other fields
- `general.quantized_by: string`: The name of the individual who quantized the model
- `general.size_label: string`: Size class of the model, such as number of weights and experts. (Useful for leader boards)
- `general.license: string`: License of the model, expressed as a [SPDX license expression](https://spdx.github.io/spdx-spec/v2-draft/SPDX-license-expressions/) (e.g. `"MIT OR Apache-2.0`). Do not include any other information, such as the license text or the URL to the license.
- `general.license.name: string`: Human friendly license name
- `general.license.link: string`: URL to the license.
- `general.url: string`: URL to the model's homepage. This can be a GitHub repo, a paper, etc.
- `general.doi: string`: Digital Object Identifier (DOI) https://www.doi.org/
- `general.uuid: string`: [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.repo_url: string`: URL to the model's repository such as a GitHub repo or HuggingFace repo
- `general.tags: string[]`: List of tags that can be used as search terms for a search engine or social media
- `general.languages: string[]`: What languages can the model speak. Encoded as [ISO 639](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes) two letter codes
- `general.datasets: string[]`: Links or references to datasets that the model was trained upon
- `general.file_type: uint32`: An enumerated value describing the type of the majority of the tensors in the file. Optional; can be inferred from the tensor types.
- `ALL_F32 = 0`
- `MOSTLY_F16 = 1`
- `MOSTLY_Q4_0 = 2`
- `MOSTLY_Q4_1 = 3`
- `MOSTLY_Q4_1_SOME_F16 = 4`
- `MOSTLY_Q4_2 = 5` (support removed)
- `MOSTLY_Q4_3 = 6` (support removed)
- `MOSTLY_Q8_0 = 7`
- `MOSTLY_Q5_0 = 8`
- `MOSTLY_Q5_1 = 9`
- `MOSTLY_Q2_K = 10`
- `MOSTLY_Q3_K_S = 11`
- `MOSTLY_Q3_K_M = 12`
- `MOSTLY_Q3_K_L = 13`
- `MOSTLY_Q4_K_S = 14`
- `MOSTLY_Q4_K_M = 15`
- `MOSTLY_Q5_K_S = 16`
- `MOSTLY_Q5_K_M = 17`
- `MOSTLY_Q6_K = 18`
#### Source metadata
Information about where this model came from. This is useful for tracking the provenance of the model, and for finding the original source if the model is modified. For a model that was converted from GGML, for example, these keys would point to the model that was converted from.
- `general.source.url: string`: URL to the source of the model's homepage. This can be a GitHub repo, a paper, etc.
- `general.source.doi: string`: Source Digital Object Identifier (DOI) https://www.doi.org/
- `general.source.uuid: string`: Source [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.source.repo_url: string`: URL to the source of the model's repository such as a GitHub repo or HuggingFace repo
- `general.base_model.count: uint32`: Number of parent models
- `general.base_model.{id}.name: string`: The name of the parent model.
- `general.base_model.{id}.author: string`: The author of the parent model.
- `general.base_model.{id}.version: string`: The version of the parent model.
- `general.base_model.{id}.organization: string`: The organization of the parent model.
- `general.base_model.{id}.url: string`: URL to the source of the parent model's homepage. This can be a GitHub repo, a paper, etc.
- `general.base_model.{id}.doi: string`: Parent Digital Object Identifier (DOI) https://www.doi.org/
- `general.base_model.{id}.uuid: string`: Parent [Universally unique identifier](https://en.wikipedia.org/wiki/Universally_unique_identifier)
- `general.base_model.{id}.repo_url: string`: URL to the source of the parent model's repository such as a GitHub repo or HuggingFace repo
### LLM
In the following, `[llm]` is used to fill in for the name of a specific LLM architecture. For example, `llama` for LLaMA, `mpt` for MPT, etc. If mentioned in an architecture's section, it is required for that architecture, but not all keys are required for all architectures. Consult the relevant section for more information.
- `[llm].context_length: uint64`: Also known as `n_ctx`. length of the context (in tokens) that the model was trained on. For most architectures, this is the hard limit on the length of the input. Architectures, like RWKV, that are not reliant on transformer-style attention may be able to handle larger inputs, but this is not guaranteed.
- `[llm].embedding_length: uint64`: Also known as `n_embd`. Embedding layer size.
- `[llm].block_count: uint64`: The number of blocks of attention+feed-forward layers (i.e. the bulk of the LLM). Does not include the input or embedding layers.
- `[llm].feed_forward_length: uint64`: Also known as `n_ff`. The length of the feed-forward layer.
- `[llm].use_parallel_residual: bool`: Whether or not the parallel residual logic should be used.
- `[llm].tensor_data_layout: string`: When a model is converted to GGUF, tensors may be rearranged to improve performance. This key describes the layout of the tensor data. This is not required; if not present, it is assumed to be `reference`.
- `reference`: tensors are laid out in the same order as the original model
- further options can be found for each architecture in their respective sections
- `[llm].expert_count: uint32`: Number of experts in MoE models (optional for non-MoE arches).
- `[llm].expert_used_count: uint32`: Number of experts used during each token token evaluation (optional for non-MoE arches).
#### Attention
- `[llm].attention.head_count: uint64`: Also known as `n_head`. Number of attention heads.
- `[llm].attention.head_count_kv: uint64`: The number of heads per group used in Grouped-Query-Attention. If not present or if present and equal to `[llm].attention.head_count`, the model does not use GQA.
- `[llm].attention.max_alibi_bias: float32`: The maximum bias to use for ALiBI.
- `[llm].attention.clamp_kqv: float32`: Value (`C`) to clamp the values of the `Q`, `K`, and `V` tensors between (`[-C, C]`).
- `[llm].attention.layer_norm_epsilon: float32`: Layer normalization epsilon.
- `[llm].attention.layer_norm_rms_epsilon: float32`: Layer RMS normalization epsilon.
- `[llm].attention.key_length: uint32`: The optional size of a key head, $d_k$. If not specified, it will be `n_embd / n_head`.
- `[llm].attention.value_length: uint32`: The optional size of a value head, $d_v$. If not specified, it will be `n_embd / n_head`.
#### RoPE
- `[llm].rope.dimension_count: uint64`: The number of rotary dimensions for RoPE.
- `[llm].rope.freq_base: float32`: The base frequency for RoPE.
##### Scaling
The following keys describe RoPE scaling parameters:
- `[llm].rope.scaling.type: string`: Can be `none`, `linear`, or `yarn`.
- `[llm].rope.scaling.factor: float32`: A scale factor for RoPE to adjust the context length.
- `[llm].rope.scaling.original_context_length: uint32_t`: The original context length of the base model.
- `[llm].rope.scaling.finetuned: bool`: True if model has been finetuned with RoPE scaling.
Note that older models may not have these keys, and may instead use the following key:
- `[llm].rope.scale_linear: float32`: A linear scale factor for RoPE to adjust the context length.
It is recommended that models use the newer keys if possible, as they are more flexible and allow for more complex scaling schemes. Executors will need to support both indefinitely.
#### SSM
- `[llm].ssm.conv_kernel: uint32`: The size of the rolling/shift state.
- `[llm].ssm.inner_size: uint32`: The embedding size of the states.
- `[llm].ssm.state_size: uint32`: The size of the recurrent state.
- `[llm].ssm.time_step_rank: uint32`: The rank of time steps.
#### Models
The following sections describe the metadata for each model architecture. Each key specified _must_ be present.
##### LLaMA
- `llama.context_length`
- `llama.embedding_length`
- `llama.block_count`
- `llama.feed_forward_length`
- `llama.rope.dimension_count`
- `llama.attention.head_count`
- `llama.attention.layer_norm_rms_epsilon`
###### Optional
- `llama.rope.scale`
- `llama.attention.head_count_kv`
- `llama.tensor_data_layout`:
- `Meta AI original pth`:
```python
def permute(weights: NDArray, n_head: int) -> NDArray:
return (weights.reshape(n_head, 2, weights.shape[0] // n_head // 2, *weights.shape[1:])
.swapaxes(1, 2)
.reshape(weights.shape))
```
- `llama.expert_count`
- `llama.expert_used_count`
##### MPT
- `mpt.context_length`
- `mpt.embedding_length`
- `mpt.block_count`
- `mpt.attention.head_count`
- `mpt.attention.alibi_bias_max`
- `mpt.attention.clip_kqv`
- `mpt.attention.layer_norm_epsilon`
##### GPT-NeoX
- `gptneox.context_length`
- `gptneox.embedding_length`
- `gptneox.block_count`
- `gptneox.use_parallel_residual`
- `gptneox.rope.dimension_count`
- `gptneox.attention.head_count`
- `gptneox.attention.layer_norm_epsilon`
###### Optional
- `gptneox.rope.scale`
##### GPT-J
- `gptj.context_length`
- `gptj.embedding_length`
- `gptj.block_count`
- `gptj.rope.dimension_count`
- `gptj.attention.head_count`
- `gptj.attention.layer_norm_epsilon`
###### Optional
- `gptj.rope.scale`
##### GPT-2
- `gpt2.context_length`
- `gpt2.embedding_length`
- `gpt2.block_count`
- `gpt2.attention.head_count`
- `gpt2.attention.layer_norm_epsilon`
##### BLOOM
- `bloom.context_length`
- `bloom.embedding_length`
- `bloom.block_count`
- `bloom.feed_forward_length`
- `bloom.attention.head_count`
- `bloom.attention.layer_norm_epsilon`
##### Falcon
- `falcon.context_length`
- `falcon.embedding_length`
- `falcon.block_count`
- `falcon.attention.head_count`
- `falcon.attention.head_count_kv`
- `falcon.attention.use_norm`
- `falcon.attention.layer_norm_epsilon`
###### Optional
- `falcon.tensor_data_layout`:
- `jploski` (author of the original GGML implementation of Falcon):
```python
# The original query_key_value tensor contains n_head_kv "kv groups",
# each consisting of n_head/n_head_kv query weights followed by one key
# and one value weight (shared by all query heads in the kv group).
# This layout makes it a big pain to work with in GGML.
# So we rearrange them here,, so that we have n_head query weights
# followed by n_head_kv key weights followed by n_head_kv value weights,
# in contiguous fashion.
if "query_key_value" in src:
qkv = model[src].view(
n_head_kv, n_head // n_head_kv + 2, head_dim, head_dim * n_head)
q = qkv[:, :-2 ].reshape(n_head * head_dim, head_dim * n_head)
k = qkv[:, [-2]].reshape(n_head_kv * head_dim, head_dim * n_head)
v = qkv[:, [-1]].reshape(n_head_kv * head_dim, head_dim * n_head)
model[src] = torch.cat((q,k,v)).reshape_as(model[src])
```
##### Mamba
- `mamba.context_length`
- `mamba.embedding_length`
- `mamba.block_count`
- `mamba.ssm.conv_kernel`
- `mamba.ssm.inner_size`
- `mamba.ssm.state_size`
- `mamba.ssm.time_step_rank`
- `mamba.attention.layer_norm_rms_epsilon`
##### RWKV
The vocabulary size is the same as the number of rows in the `head` matrix.
- `rwkv.architecture_version: uint32`: The only allowed value currently is 4. Version 5 is expected to appear some time in the future.
- `rwkv.context_length: uint64`: Length of the context used during training or fine-tuning. RWKV is able to handle larger context than this limit, but the output quality may suffer.
- `rwkv.block_count: uint64`
- `rwkv.embedding_length: uint64`
- `rwkv.feed_forward_length: uint64`
##### Whisper
Keys that do not have types defined should be assumed to share definitions with `llm.` keys.
(For example, `whisper.context_length` is equivalent to `llm.context_length`.)
This is because they are both transformer models.
- `whisper.encoder.context_length`
- `whisper.encoder.embedding_length`
- `whisper.encoder.block_count`
- `whisper.encoder.mels_count: uint64`
- `whisper.encoder.attention.head_count`
- `whisper.decoder.context_length`
- `whisper.decoder.embedding_length`
- `whisper.decoder.block_count`
- `whisper.decoder.attention.head_count`
#### Prompting
**TODO**: Include prompt format, and/or metadata about how it should be used (instruction, conversation, autocomplete, etc).
### LoRA
**TODO**: Figure out what metadata is needed for LoRA. Probably desired features:
- match an existing model exactly, so that it can't be misapplied
- be marked as a LoRA so executors won't try to run it by itself
Should this be an architecture, or should it share the details of the original model with additional fields to mark it as a LoRA?
### Tokenizer
The following keys are used to describe the tokenizer of the model. It is recommended that model authors support as many of these as possible, as it will allow for better tokenization quality with supported executors.
#### GGML
GGML supports an embedded vocabulary that enables inference of the model, but implementations of tokenization using this vocabulary (i.e. `llama.cpp`'s tokenizer) may have lower accuracy than the original tokenizer used for the model. When a more accurate tokenizer is available and supported, it should be used instead.
It is not guaranteed to be standardized across models, and may change in the future. It is recommended that model authors use a more standardized tokenizer if possible.
- `tokenizer.ggml.model: string`: The name of the tokenizer model.
- `llama`: Llama style SentencePiece (tokens and scores extracted from HF `tokenizer.model`)
- `replit`: Replit style SentencePiece (tokens and scores extracted from HF `spiece.model`)
- `gpt2`: GPT-2 / GPT-NeoX style BPE (tokens extracted from HF `tokenizer.json`)
- `rwkv`: RWKV tokenizer
- `tokenizer.ggml.tokens: array[string]`: A list of tokens indexed by the token ID used by the model.
- `tokenizer.ggml.scores: array[float32]`: If present, the score/probability of each token. If not present, all tokens are assumed to have equal probability. If present, it must have the same length and index as `tokens`.
- `tokenizer.ggml.token_type: array[int32]`: The token type (1=normal, 2=unknown, 3=control, 4=user defined, 5=unused, 6=byte). If present, it must have the same length and index as `tokens`.
- `tokenizer.ggml.merges: array[string]`: If present, the merges of the tokenizer. If not present, the tokens are assumed to be atomic.
- `tokenizer.ggml.added_tokens: array[string]`: If present, tokens that were added after training.
##### Special tokens
- `tokenizer.ggml.bos_token_id: uint32`: Beginning of sequence marker
- `tokenizer.ggml.eos_token_id: uint32`: End of sequence marker
- `tokenizer.ggml.unknown_token_id: uint32`: Unknown token
- `tokenizer.ggml.separator_token_id: uint32`: Separator token
- `tokenizer.ggml.padding_token_id: uint32`: Padding token
#### Hugging Face
Hugging Face maintains their own `tokenizers` library that supports a wide variety of tokenizers. If your executor uses this library, it may be able to use the model's tokenizer directly.
- `tokenizer.huggingface.json: string`: the entirety of the HF `tokenizer.json` for a given model (e.g. <https://huggingface.co/mosaicml/mpt-7b-instruct/blob/main/tokenizer.json>). Included for compatibility with executors that support HF tokenizers directly.
#### Other
Other tokenizers may be used, but are not necessarily standardized. They may be executor-specific. They will be documented here as they are discovered/further developed.
- `tokenizer.rwkv.world: string`: a RWKV World tokenizer, like [this](https://github.com/BlinkDL/ChatRWKV/blob/main/tokenizer/rwkv_vocab_v20230424.txt). This text file should be included verbatim.
- `tokenizer.chat_template : string`: a Jinja template that specifies the input format expected by the model. For more details see: <https://huggingface.co/docs/transformers/main/en/chat_templating>
### Computation graph
This is a future extension and still needs to be discussed, and may necessitate a new GGUF version. At the time of writing, the primary blocker is the stabilization of the computation graph format.
A sample computation graph of GGML nodes could be included in the model itself, allowing an executor to run the model without providing its own implementation of the architecture. This would allow for a more consistent experience across executors, and would allow for more complex architectures to be supported without requiring the executor to implement them.
## Standardized tensor names
To minimize complexity and maximize compatibility, it is recommended that models using the transformer architecture use the following naming convention for their tensors:
### Base layers
`AA.weight` `AA.bias`
where `AA` can be:
- `token_embd`: Token embedding layer
- `pos_embd`: Position embedding layer
- `output_norm`: Output normalization layer
- `output`: Output layer
### Attention and feed-forward layer blocks
`blk.N.BB.weight` `blk.N.BB.bias`
where N signifies the block number a layer belongs to, and where `BB` could be:
- `attn_norm`: Attention normalization layer
- `attn_norm_2`: Attention normalization layer
- `attn_qkv`: Attention query-key-value layer
- `attn_q`: Attention query layer
- `attn_k`: Attention key layer
- `attn_v`: Attention value layer
- `attn_output`: Attention output layer
- `ffn_norm`: Feed-forward network normalization layer
- `ffn_up`: Feed-forward network "up" layer
- `ffn_gate`: Feed-forward network "gate" layer
- `ffn_down`: Feed-forward network "down" layer
- `ffn_gate_inp`: Expert-routing layer for the Feed-forward network in MoE models
- `ffn_gate_exp`: Feed-forward network "gate" layer per expert in MoE models
- `ffn_down_exp`: Feed-forward network "down" layer per expert in MoE models
- `ffn_up_exp`: Feed-forward network "up" layer per expert in MoE models
- `ssm_in`: State space model input projections layer
- `ssm_conv1d`: State space model rolling/shift layer
- `ssm_x`: State space model selective parametrization layer
- `ssm_a`: State space model state compression layer
- `ssm_d`: State space model skip connection layer
- `ssm_dt`: State space model time step layer
- `ssm_out`: State space model output projection layer
## Version History
This document is actively updated to describe the current state of the metadata, and these changes are not tracked outside of the commits.
However, the format _itself_ has changed. The following sections describe the changes to the format itself.
### v3
Adds big-endian support.
### v2
Most countable values (lengths, etc) were changed from `uint32` to `uint64` to allow for larger models to be supported in the future.
### v1
Initial version.
## Historical State of Affairs
The following information is provided for context, but is not necessary to understand the rest of this document.
### Overview
At present, there are three GGML file formats floating around for LLMs:
- **GGML** (unversioned): baseline format, with no versioning or alignment.
- **GGMF** (versioned): the same as GGML, but with versioning. Only one version exists.
- **GGJT**: Aligns the tensors to allow for use with `mmap`, which requires alignment. v1, v2 and v3 are identical, but the latter versions use a different quantization scheme that is incompatible with previous versions.
GGML is primarily used by the examples in `ggml`, while GGJT is used by `llama.cpp` models. Other executors may use any of the three formats, but this is not 'officially' supported.
These formats share the same fundamental structure:
- a magic number with an optional version number
- model-specific hyperparameters, including
- metadata about the model, such as the number of layers, the number of heads, etc.
- a `ftype` that describes the type of the majority of the tensors,
- for GGML files, the quantization version is encoded in the `ftype` divided by 1000
- an embedded vocabulary, which is a list of strings with length prepended. The GGMF/GGJT formats embed a float32 score next to the strings.
- finally, a list of tensors with their length-prepended name, type, and (aligned, in the case of GGJT) tensor data
Notably, this structure does not identify what model architecture the model belongs to, nor does it offer any flexibility for changing the structure of the hyperparameters. This means that the only way to add new hyperparameters is to add them to the end of the list, which is a breaking change for existing models.
### Drawbacks
Unfortunately, over the last few months, there are a few issues that have become apparent with the existing models:
- There's no way to identify which model architecture a given model is for, because that information isn't present
- Similarly, existing programs cannot intelligently fail upon encountering new architectures
- Adding or removing any new hyperparameters is a breaking change, which is impossible for a reader to detect without using heuristics
- Each model architecture requires its own conversion script to their architecture's variant of GGML
- Maintaining backwards compatibility without breaking the structure of the format requires clever tricks, like packing the quantization version into the ftype, which are not guaranteed to be picked up by readers/writers, and are not consistent between the two formats
### Why not other formats?
There are a few other formats that could be used, but issues include:
- requiring additional dependencies to load or save the model, which is complicated in a C environment
- limited or no support for 4-bit quantization
- existing cultural expectations (e.g. whether or not the model is a directory or a file)
- lack of support for embedded vocabularies
- lack of control over direction of future development
Ultimately, it is likely that GGUF will remain necessary for the foreseeable future, and it is better to have a single format that is well-documented and supported by all executors than to contort an existing format to fit the needs of GGML.
engine/ggml/examples/CMakeLists.txt
+34
View File
@@ -0,0 +1,34 @@
if (GGML_ALL_WARNINGS)
if (NOT MSVC)
set(cxx_flags
# TODO(marella): Add other warnings.
-Wpedantic
-Wunused-variable
-Wno-unused-function
-Wno-multichar
)
add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${cxx_flags}>")
endif()
endif()
add_library(common STATIC common.cpp)
target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_library(common-ggml STATIC common-ggml.cpp)
target_link_libraries(common-ggml PRIVATE ggml)
target_include_directories(common-ggml PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
add_subdirectory(yolo)
if (NOT GGML_BACKEND_DL)
add_subdirectory(gpt-2)
add_subdirectory(gpt-j)
add_subdirectory(mnist)
add_subdirectory(sam)
add_subdirectory(simple)
add_subdirectory(magika)
endif()
if (GGML_METAL)
add_subdirectory(perf-metal)
endif()
engine/ggml/examples/common-ggml.cpp
+244
View File
@@ -0,0 +1,244 @@
#include "common-ggml.h"
#include <regex>
#include <map>
static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {
{"q4_0", GGML_FTYPE_MOSTLY_Q4_0},
{"q4_1", GGML_FTYPE_MOSTLY_Q4_1},
{"q5_0", GGML_FTYPE_MOSTLY_Q5_0},
{"q5_1", GGML_FTYPE_MOSTLY_Q5_1},
{"q8_0", GGML_FTYPE_MOSTLY_Q8_0},
{"q2_k", GGML_FTYPE_MOSTLY_Q2_K},
{"q3_k", GGML_FTYPE_MOSTLY_Q3_K},
{"q4_k", GGML_FTYPE_MOSTLY_Q4_K},
{"q5_k", GGML_FTYPE_MOSTLY_Q5_K},
{"q6_k", GGML_FTYPE_MOSTLY_Q6_K},
};
void ggml_print_ftypes(FILE * fp) {
for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {
fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second);
}
}
enum ggml_ftype ggml_parse_ftype(const char * str) {
enum ggml_ftype ftype;
if (str[0] == 'q') {
const auto it = GGML_FTYPE_MAP.find(str);
if (it == GGML_FTYPE_MAP.end()) {
fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);
return GGML_FTYPE_UNKNOWN;
}
ftype = it->second;
} else {
ftype = (enum ggml_ftype) atoi(str);
}
return ftype;
}
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip) {
ggml_type qtype = GGML_TYPE_F32;
switch (ftype) {
case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;
case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;
case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;
case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;
case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;
case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;
case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;
case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;
case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;
case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;
case GGML_FTYPE_UNKNOWN:
case GGML_FTYPE_ALL_F32:
case GGML_FTYPE_MOSTLY_F16:
case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:
case GGML_FTYPE_MOSTLY_IQ2_XXS:
case GGML_FTYPE_MOSTLY_IQ2_XS:
case GGML_FTYPE_MOSTLY_IQ2_S:
case GGML_FTYPE_MOSTLY_IQ3_XXS:
case GGML_FTYPE_MOSTLY_IQ3_S:
case GGML_FTYPE_MOSTLY_IQ1_S:
case GGML_FTYPE_MOSTLY_IQ4_NL:
case GGML_FTYPE_MOSTLY_IQ4_XS:
case GGML_FTYPE_MOSTLY_IQ1_M:
case GGML_FTYPE_MOSTLY_BF16:
case GGML_FTYPE_MOSTLY_MXFP4:
case GGML_FTYPE_MOSTLY_NVFP4:
case GGML_FTYPE_MOSTLY_Q1_0:
{
fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);
return false;
}
};
if (!ggml_is_quantized(qtype)) {
fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));
return false;
}
size_t total_size_org = 0;
size_t total_size_new = 0;
std::vector<float> work;
std::vector<uint8_t> data_u8;
std::vector<ggml_fp16_t> data_f16;
std::vector<float> data_f32;
while (true) {
int32_t n_dims;
int32_t length;
int32_t ttype;
finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
finp.read(reinterpret_cast<char *>(&length), sizeof(length));
finp.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));
if (finp.eof()) {
break;
}
int32_t nelements = 1;
int32_t ne[4] = { 1, 1, 1, 1 };
for (int i = 0; i < n_dims; ++i) {
finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}
std::string name(length, 0);
finp.read (&name[0], length);
printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));
bool quantize = false;
// check if we should quantize this tensor
for (const auto & s : to_quant) {
if (std::regex_match(name, std::regex(s))) {
quantize = true;
break;
}
}
// check if we should skip this tensor
for (const auto & s : to_skip) {
if (std::regex_match(name, std::regex(s))) {
quantize = false;
break;
}
}
// quantize only 2D tensors
quantize &= (n_dims == 2);
if (quantize) {
if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {
fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
if (ttype == GGML_TYPE_F16) {
data_f16.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));
data_f32.resize(nelements);
for (int i = 0; i < nelements; ++i) {
data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);
}
} else {
data_f32.resize(nelements);
finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));
}
ttype = qtype;
} else {
const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);
data_u8.resize(nelements*bpe);
finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);
}
fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fout.write(reinterpret_cast<char *>(&length), sizeof(length));
fout.write(reinterpret_cast<char *>(&ttype), sizeof(ttype));
for (int i = 0; i < n_dims; ++i) {
fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
}
fout.write(&name[0], length);
if (quantize) {
work.resize(nelements); // for quantization
size_t cur_size = 0;
switch ((ggml_type) ttype) {
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
{
cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);
} break;
case GGML_TYPE_F32:
case GGML_TYPE_F16:
case GGML_TYPE_I8:
case GGML_TYPE_I16:
case GGML_TYPE_I32:
case GGML_TYPE_I64:
case GGML_TYPE_F64:
case GGML_TYPE_Q8_1:
case GGML_TYPE_Q8_K:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
case GGML_TYPE_IQ2_S:
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ3_S:
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ1_M:
case GGML_TYPE_BF16:
case GGML_TYPE_TQ1_0:
case GGML_TYPE_TQ2_0:
case GGML_TYPE_MXFP4:
case GGML_TYPE_NVFP4:
case GGML_TYPE_Q1_0:
case GGML_TYPE_COUNT:
{
fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));
return false;
}
}
fout.write(reinterpret_cast<char *>(work.data()), cur_size);
total_size_new += cur_size;
printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);
} else {
printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);
fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());
total_size_new += data_u8.size();
}
total_size_org += nelements * sizeof(float);
}
printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);
printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));
return true;
}
engine/ggml/examples/common-ggml.h
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "ggml.h"
#include <fstream>
#include <vector>
#include <string>
enum ggml_ftype ggml_parse_ftype(const char * str);
void ggml_print_ftypes(FILE * fp = stderr);
bool ggml_common_quantize_0(
std::ifstream & finp,
std::ofstream & fout,
const ggml_ftype ftype,
const std::vector<std::string> & to_quant,
const std::vector<std::string> & to_skip);
engine/ggml/examples/common.cpp
+675
View File
@@ -0,0 +1,675 @@
#define _USE_MATH_DEFINES // for M_PI
#include "common.h"
#include <cmath>
#include <codecvt>
#include <cstring>
#include <fstream>
#include <locale>
#include <regex>
#include <sstream>
// Function to check if the next argument exists
static std::string get_next_arg(int& i, int argc, char** argv, const std::string& flag, gpt_params& params) {
if (i + 1 < argc && argv[i + 1][0] != '-') {
return argv[++i];
} else {
fprintf(stderr, "error: %s requires one argument.\n", flag.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-s" || arg == "--seed") {
params.seed = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-t" || arg == "--threads") {
params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-p" || arg == "--prompt") {
params.prompt = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-n" || arg == "--n_predict") {
params.n_predict = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-np" || arg == "--n_parallel") {
params.n_parallel = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_k") {
params.top_k = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--top_p") {
params.top_p = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--temp") {
params.temp = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-last-n") {
params.repeat_last_n = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--repeat-penalty") {
params.repeat_penalty = std::stof(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-b" || arg == "--batch_size") {
params.n_batch= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-c" || arg == "--context") {
params.n_ctx= std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-ngl" || arg == "--gpu-layers" || arg == "--n-gpu-layers") {
params.n_gpu_layers = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "--ignore-eos") {
params.ignore_eos = true;
} else if (arg == "-m" || arg == "--model") {
params.model = get_next_arg(i, argc, argv, arg, params);
} else if (arg == "-i" || arg == "--interactive") {
params.interactive = true;
} else if (arg == "-ip" || arg == "--interactive-port") {
params.interactive = true;
params.interactive_port = std::stoi(get_next_arg(i, argc, argv, arg, params));
} else if (arg == "-h" || arg == "--help") {
gpt_print_usage(argc, argv, params);
exit(0);
} else if (arg == "-f" || arg == "--file") {
get_next_arg(i, argc, argv, arg, params);
std::ifstream file(argv[i]);
if (!file) {
fprintf(stderr, "error: failed to open file '%s'\n", argv[i]);
break;
}
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
if (params.prompt.back() == '\n') {
params.prompt.pop_back();
}
} else if (arg == "-tt" || arg == "--token_test") {
params.token_test = get_next_arg(i, argc, argv, arg, params);
}
else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
gpt_print_usage(argc, argv, params);
exit(0);
}
}
return true;
}
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
fprintf(stderr, " prompt to start generation with (default: random)\n");
fprintf(stderr, " -f FNAME, --file FNAME\n");
fprintf(stderr, " load prompt from a file\n");
fprintf(stderr, " -tt TOKEN_TEST, --token_test TOKEN_TEST\n");
fprintf(stderr, " test tokenization\n");
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d)\n", params.n_predict);
fprintf(stderr, " --top_k N top-k sampling (default: %d)\n", params.top_k);
fprintf(stderr, " --top_p N top-p sampling (default: %.1f)\n", params.top_p);
fprintf(stderr, " --temp N temperature (default: %.1f)\n", params.temp);
fprintf(stderr, " --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled)\n", params.repeat_last_n);
fprintf(stderr, " --repeat-penalty N penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)\n", (double)params.repeat_penalty);
fprintf(stderr, " -b N, --batch_size N batch size for prompt processing (default: %d)\n", params.n_batch);
fprintf(stderr, " -c N, --context N context / KV cache size (default: %d)\n", params.n_ctx);
fprintf(stderr, " --ignore-eos ignore EOS token during generation\n");
fprintf(stderr, " -ngl N, --gpu-layers N number of layers to offload to GPU on supported models (default: %d)\n", params.n_gpu_layers);
fprintf(stderr, " -m FNAME, --model FNAME\n");
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
fprintf(stderr, "\n");
}
std::string gpt_random_prompt(std::mt19937 & rng) {
const int r = rng() % 10;
switch (r) {
case 0: return "So";
case 1: return "Once upon a time";
case 2: return "When";
case 3: return "The";
case 4: return "After";
case 5: return "If";
case 6: return "import";
case 7: return "He";
case 8: return "She";
case 9: return "They";
}
return "The";
}
std::string trim(const std::string & s) {
std::regex e("^\\s+|\\s+$");
return std::regex_replace(s, e, "");
}
std::string replace(const std::string & s, const std::string & from, const std::string & to) {
std::string result = s;
size_t pos = 0;
while ((pos = result.find(from, pos)) != std::string::npos) {
result.replace(pos, from.length(), to);
pos += to.length();
}
return result;
}
void gpt_vocab::add_special_token(const std::string & token) {
special_tokens.push_back(token);
}
std::map<std::string, int32_t> json_parse(const std::string & fname) {
std::map<std::string, int32_t> result;
// read file into string
std::string json;
{
std::ifstream ifs(fname);
if (!ifs) {
fprintf(stderr, "Failed to open %s\n", fname.c_str());
exit(1);
}
json = std::string((std::istreambuf_iterator<char>(ifs)),
(std::istreambuf_iterator<char>()));
}
if (json[0] != '{') {
return result;
}
// parse json
{
bool has_key = false;
bool in_token = false;
std::string str_key = "";
std::string str_val = "";
int n = json.size();
for (int i = 1; i < n; ++i) {
if (!in_token) {
if (json[i] == ' ') continue;
if (json[i] == '"') {
in_token = true;
continue;
}
} else {
if (json[i] == '\\' && i+1 < n) {
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
++i;
} else if (json[i] == '"') {
if (has_key == false) {
has_key = true;
++i;
while (json[i] == ' ') ++i;
++i; // :
while (json[i] == ' ') ++i;
if (json[i] != '\"') {
while (json[i] != ',' && json[i] != '}') {
str_val += json[i++];
}
has_key = false;
} else {
in_token = true;
continue;
}
} else {
has_key = false;
}
str_key = ::replace(str_key, "\\u0120", " " ); // \u0120 -> space
str_key = ::replace(str_key, "\\u010a", "\n"); // \u010a -> new line
str_key = ::replace(str_key, "\\\"", "\""); // \\\" -> "
try {
result[str_key] = std::stoi(str_val);
} catch (...) {
//fprintf(stderr, "%s: ignoring key '%s' with value '%s'\n", fname.c_str(), str_key.c_str(), str_val.c_str());
}
str_key = "";
str_val = "";
in_token = false;
continue;
}
if (has_key == false) {
str_key += json[i];
} else {
str_val += json[i];
}
}
}
}
return result;
}
void gpt_split_words(std::string str, std::vector<std::string>& words) {
const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)";
const std::regex re(pattern);
std::smatch m;
while (std::regex_search(str, m, re)) {
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
}
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) {
std::vector<std::string> words;
// first split the text into words
{
std::string str = text;
// Generate the subpattern from the special_tokens vector if it's not empty
if (!vocab.special_tokens.empty()) {
const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])");
std::string special_tokens_subpattern;
for (const auto & token : vocab.special_tokens) {
if (!special_tokens_subpattern.empty()) {
special_tokens_subpattern += "|";
}
special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)");
}
std::regex re(special_tokens_subpattern);
std::smatch m;
// Split the text by special tokens.
while (std::regex_search(str, m, re)) {
// Split the substrings in-between special tokens into words.
gpt_split_words(m.prefix(), words);
// Add matched special tokens as words.
for (auto x : m) {
words.push_back(x);
}
str = m.suffix();
}
// Remaining text without special tokens will be handled below.
}
gpt_split_words(str, words);
}
// find the longest token that forms each word in words:
std::vector<gpt_vocab::id> tokens;
for (const auto & word : words) {
for (int i = 0; i < (int) word.size(); ){
for (int j = word.size() - 1; j >= i; j--){
auto cand = word.substr(i, j-i+1);
auto it = vocab.token_to_id.find(cand);
if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab
tokens.push_back(it->second);
i = j + 1;
break;
}
else if (j == i){ // word.substr(i, 1) has no matching
fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data());
i++;
}
}
}
}
return tokens;
}
static std::vector<gpt_vocab::id> parse_tokens_from_string(const std::string& input, char delimiter) {
std::vector<gpt_vocab::id> output;
std::stringstream ss(input);
std::string token;
while (std::getline(ss, token, delimiter)) {
output.push_back(std::stoi(token));
}
return output;
}
static std::map<std::string, std::vector<gpt_vocab::id>> extract_tests_from_file(const std::string & fpath_test){
if (fpath_test.empty()){
fprintf(stderr, "%s : No test file found.\n", __func__);
return std::map<std::string, std::vector<gpt_vocab::id>>();
}
std::map<std::string, std::vector<gpt_vocab::id>> tests;
auto fin = std::ifstream(fpath_test, std::ios_base::in);
const char * delimeter = " => ";
const char del_tok = ',';
std::string line;
while (std::getline(fin, line)) {
size_t delimiterPos = line.find(delimeter);
if (delimiterPos != std::string::npos) {
std::string text = line.substr(0, delimiterPos);
std::string s_tokens = line.substr(delimiterPos + std::strlen(delimeter));
tests[text] = parse_tokens_from_string(s_tokens, del_tok);
}
}
return tests;
}
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test){
std::map<std::string, std::vector<gpt_vocab::id>> tests = extract_tests_from_file(fpath_test);
size_t n_fails = 0;
for (const auto & test : tests) {
std::vector<gpt_vocab::id> tokens = gpt_tokenize(vocab, test.first);
if (tokens != test.second){
n_fails++;
// print out failure cases
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test.first.c_str());
fprintf(stderr, "%s : tokens in hf: ", __func__);
for (const auto & t : test.second) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : tokens in ggml: ", __func__);
for (const auto & t : tokens) {
fprintf(stderr, "%s(%d), ", vocab.id_to_token[t].c_str(), t);
}
fprintf(stderr, "\n");
}
}
fprintf(stderr, "%s : %zu tests failed out of %zu tests.\n", __func__, n_fails, tests.size());
}
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab) {
printf("%s: loading vocab from '%s'\n", __func__, fname.c_str());
vocab.token_to_id = ::json_parse(fname);
for (const auto & kv : vocab.token_to_id) {
vocab.id_to_token[kv.second] = kv.first;
}
printf("%s: vocab size = %d\n", __func__, (int) vocab.token_to_id.size());
// print the vocabulary
//for (auto kv : vocab.token_to_id) {
// printf("'%s' -> %d\n", kv.first.data(), kv.second);
//}
return true;
}
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const double scale = 1.0/temp;
for (int i = 0; i < n_logits; ++i) {
logits_id.push_back(std::make_pair(logits[i]*scale, i));
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
//printf("\n");
//for (int i = 0; i < (int) probs.size(); i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
//}
//exit(0);
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng) {
int n_logits = vocab.id_to_token.size();
const auto * plogits = logits;
const auto last_n_tokens = std::vector<int32_t>(last_n_tokens_data, last_n_tokens_data + last_n_tokens_data_size);
if (temp <= 0) {
// select the token with the highest logit directly
float max_logit = plogits[0];
gpt_vocab::id max_id = 0;
for (int i = 1; i < n_logits; ++i) {
if (plogits[i] > max_logit) {
max_logit = plogits[i];
max_id = i;
}
}
return max_id;
}
std::vector<std::pair<double, gpt_vocab::id>> logits_id;
logits_id.reserve(n_logits);
{
const float scale = 1.0f/temp;
for (int i = 0; i < n_logits; ++i) {
// repetition penalty from ctrl paper (https://arxiv.org/abs/1909.05858)
// credit https://github.com/facebookresearch/llama/compare/main...shawwn:llama:main
if (repeat_last_n > 0 && std::find(last_n_tokens.end()-repeat_last_n, last_n_tokens.end(), i) != last_n_tokens.end()) {
// if score < 0 then repetition penalty has to multiplied to reduce the previous token probability
if (plogits[i] < 0.0f) {
logits_id.push_back(std::make_pair(plogits[i]*scale*repeat_penalty, i));
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale/repeat_penalty, i));
}
} else {
logits_id.push_back(std::make_pair(plogits[i]*scale, i));
}
}
}
// find the top K tokens
std::partial_sort(
logits_id.begin(),
logits_id.begin() + top_k, logits_id.end(),
[](const std::pair<double, gpt_vocab::id> & a, const std::pair<double, gpt_vocab::id> & b) {
return a.first > b.first;
});
logits_id.resize(top_k);
double maxl = -INFINITY;
for (const auto & kv : logits_id) {
maxl = std::max(maxl, kv.first);
}
// compute probs for the top K tokens
std::vector<double> probs;
probs.reserve(logits_id.size());
double sum = 0.0;
for (const auto & kv : logits_id) {
double p = exp(kv.first - maxl);
probs.push_back(p);
sum += p;
}
// normalize the probs
for (auto & p : probs) {
p /= sum;
}
if (top_p < 1.0f) {
double cumsum = 0.0f;
for (int i = 0; i < top_k; i++) {
cumsum += probs[i];
if (cumsum >= top_p) {
top_k = i + 1;
probs.resize(top_k);
logits_id.resize(top_k);
break;
}
}
cumsum = 1.0/cumsum;
for (int i = 0; i < (int) probs.size(); i++) {
probs[i] *= cumsum;
}
}
// printf("\n");
// for (int i = 0; i < (int) probs.size(); i++) {
// for (int i = 0; i < 10; i++) {
// printf("%d: '%s' %f\n", i, vocab.id_to_token.at(logits_id[i].second).c_str(), probs[i]);
// }
std::discrete_distribution<> dist(probs.begin(), probs.end());
int idx = dist(rng);
return logits_id[idx].second;
}
void high_pass_filter(std::vector<float> & data, float cutoff, float sample_rate) {
const float rc = 1.0f / (2.0f * M_PI * cutoff);
const float dt = 1.0f / sample_rate;
const float alpha = dt / (rc + dt);
float y = data[0];
for (size_t i = 1; i < data.size(); i++) {
y = alpha * (y + data[i] - data[i - 1]);
data[i] = y;
}
}
bool vad_simple(std::vector<float> & pcmf32, int sample_rate, int last_ms, float vad_thold, float freq_thold, bool verbose) {
const int n_samples = pcmf32.size();
const int n_samples_last = (sample_rate * last_ms) / 1000;
if (n_samples_last >= n_samples) {
// not enough samples - assume no speech
return false;
}
if (freq_thold > 0.0f) {
high_pass_filter(pcmf32, freq_thold, sample_rate);
}
float energy_all = 0.0f;
float energy_last = 0.0f;
for (int i = 0; i < n_samples; i++) {
energy_all += fabsf(pcmf32[i]);
if (i >= n_samples - n_samples_last) {
energy_last += fabsf(pcmf32[i]);
}
}
energy_all /= n_samples;
energy_last /= n_samples_last;
if (verbose) {
fprintf(stderr, "%s: energy_all: %f, energy_last: %f, vad_thold: %f, freq_thold: %f\n", __func__, energy_all, energy_last, vad_thold, freq_thold);
}
if (energy_last > vad_thold*energy_all) {
return false;
}
return true;
}
float similarity(const std::string & s0, const std::string & s1) {
const size_t len0 = s0.size() + 1;
const size_t len1 = s1.size() + 1;
std::vector<int> col(len1, 0);
std::vector<int> prevCol(len1, 0);
for (size_t i = 0; i < len1; i++) {
prevCol[i] = i;
}
for (size_t i = 0; i < len0; i++) {
col[0] = i;
for (size_t j = 1; j < len1; j++) {
col[j] = std::min(std::min(1 + col[j - 1], 1 + prevCol[j]), prevCol[j - 1] + (i > 0 && s0[i - 1] == s1[j - 1] ? 0 : 1));
}
col.swap(prevCol);
}
const float dist = prevCol[len1 - 1];
return 1.0f - (dist / std::max(s0.size(), s1.size()));
}
bool is_file_exist(const char * filename) {
std::ifstream infile(filename);
return infile.good();
}
engine/ggml/examples/common.h
+322
View File
@@ -0,0 +1,322 @@
// Various helper functions and utilities
#pragma once
#include <string>
#include <map>
#include <vector>
#include <random>
#include <thread>
#include <ctime>
#include <fstream>
#include <sstream>
//
// GPT CLI argument parsing
//
struct gpt_params {
int32_t seed = -1; // RNG seed
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_predict = 200; // new tokens to predict
int32_t n_parallel = 1; // number of parallel streams
int32_t n_batch = 32; // batch size for prompt processing
int32_t n_ctx = 2048; // context size (this is the KV cache max size)
int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU
bool ignore_eos = false; // ignore EOS token when generating text
// sampling parameters
int32_t top_k = 40;
float top_p = 0.9f;
float temp = 0.9f;
int32_t repeat_last_n = 64;
float repeat_penalty = 1.00f;
std::string model = "models/gpt-2-117M/ggml-model.bin"; // model path
std::string prompt = "";
std::string token_test = "";
bool interactive = false;
int32_t interactive_port = -1;
};
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng);
//
// Vocab utils
//
std::string trim(const std::string & s);
std::string replace(
const std::string & s,
const std::string & from,
const std::string & to);
struct gpt_vocab {
using id = int32_t;
using token = std::string;
std::map<token, id> token_to_id;
std::map<id, token> id_to_token;
std::vector<std::string> special_tokens;
void add_special_token(const std::string & token);
};
// poor-man's JSON parsing
std::map<std::string, int32_t> json_parse(const std::string & fname);
std::string convert_to_utf8(const std::wstring & input);
std::wstring convert_to_wstring(const std::string & input);
void gpt_split_words(std::string str, std::vector<std::string>& words);
// split text into tokens
//
// ref: https://github.com/openai/gpt-2/blob/a74da5d99abaaba920de8131d64da2862a8f213b/src/encoder.py#L53
//
// Regex (Python):
// r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
//
// Regex (C++):
// R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"
//
std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text);
// test outputs of gpt_tokenize
//
// - compare with tokens generated by the huggingface tokenizer
// - test cases are chosen based on the model's main language (under 'prompt' directory)
// - if all sentences are tokenized identically, print 'All tests passed.'
// - otherwise, print sentence, huggingface tokens, ggml tokens
//
void test_gpt_tokenizer(gpt_vocab & vocab, const std::string & fpath_test);
// load the tokens from encoder.json
bool gpt_vocab_init(const std::string & fname, gpt_vocab & vocab);
// sample next token given probabilities for each embedding
//
// - consider only the top K tokens
// - from them, consider only the top tokens with cumulative probability > P
//
// TODO: not sure if this implementation is correct
// TODO: temperature is not implemented
//
gpt_vocab::id gpt_sample_top_k_top_p(
const gpt_vocab & vocab,
const float * logits,
int top_k,
double top_p,
double temp,
std::mt19937 & rng);
gpt_vocab::id gpt_sample_top_k_top_p_repeat(
const gpt_vocab & vocab,
const float * logits,
const int32_t * last_n_tokens_data,
size_t last_n_tokens_data_size,
int top_k,
double top_p,
double temp,
int repeat_last_n,
float repeat_penalty,
std::mt19937 & rng);
//
// Audio utils
//
// Write PCM data into WAV audio file
class wav_writer {
private:
std::ofstream file;
uint32_t dataSize = 0;
std::string wav_filename;
bool write_header(const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
file.write("RIFF", 4);
file.write("\0\0\0\0", 4); // Placeholder for file size
file.write("WAVE", 4);
file.write("fmt ", 4);
const uint32_t sub_chunk_size = 16;
const uint16_t audio_format = 1; // PCM format
const uint32_t byte_rate = sample_rate * channels * bits_per_sample / 8;
const uint16_t block_align = channels * bits_per_sample / 8;
file.write(reinterpret_cast<const char *>(&sub_chunk_size), 4);
file.write(reinterpret_cast<const char *>(&audio_format), 2);
file.write(reinterpret_cast<const char *>(&channels), 2);
file.write(reinterpret_cast<const char *>(&sample_rate), 4);
file.write(reinterpret_cast<const char *>(&byte_rate), 4);
file.write(reinterpret_cast<const char *>(&block_align), 2);
file.write(reinterpret_cast<const char *>(&bits_per_sample), 2);
file.write("data", 4);
file.write("\0\0\0\0", 4); // Placeholder for data size
return true;
}
// It is assumed that PCM data is normalized to a range from -1 to 1
bool write_audio(const float * data, size_t length) {
for (size_t i = 0; i < length; ++i) {
const int16_t intSample = int16_t(data[i] * 32767);
file.write(reinterpret_cast<const char *>(&intSample), sizeof(int16_t));
dataSize += sizeof(int16_t);
}
if (file.is_open()) {
file.seekp(4, std::ios::beg);
uint32_t fileSize = 36 + dataSize;
file.write(reinterpret_cast<char *>(&fileSize), 4);
file.seekp(40, std::ios::beg);
file.write(reinterpret_cast<char *>(&dataSize), 4);
file.seekp(0, std::ios::end);
}
return true;
}
bool open_wav(const std::string & filename) {
if (filename != wav_filename) {
if (file.is_open()) {
file.close();
}
}
if (!file.is_open()) {
file.open(filename, std::ios::binary);
wav_filename = filename;
dataSize = 0;
}
return file.is_open();
}
public:
bool open(const std::string & filename,
const uint32_t sample_rate,
const uint16_t bits_per_sample,
const uint16_t channels) {
if (open_wav(filename)) {
write_header(sample_rate, bits_per_sample, channels);
} else {
return false;
}
return true;
}
bool close() {
file.close();
return true;
}
bool write(const float * data, size_t length) {
return write_audio(data, length);
}
~wav_writer() {
if (file.is_open()) {
file.close();
}
}
};
// Apply a high-pass frequency filter to PCM audio
// Suppresses frequencies below cutoff Hz
void high_pass_filter(
std::vector<float> & data,
float cutoff,
float sample_rate);
// Basic voice activity detection (VAD) using audio energy adaptive threshold
bool vad_simple(
std::vector<float> & pcmf32,
int sample_rate,
int last_ms,
float vad_thold,
float freq_thold,
bool verbose);
// compute similarity between two strings using Levenshtein distance
float similarity(const std::string & s0, const std::string & s1);
//
// Terminal utils
//
#define SQR(X) ((X) * (X))
#define UNCUBE(x) x < 48 ? 0 : x < 115 ? 1 : (x - 35) / 40
/**
* Quantizes 24-bit RGB to xterm256 code range [16,256).
*/
static int rgb2xterm256(int r, int g, int b) {
unsigned char cube[] = {0, 0137, 0207, 0257, 0327, 0377};
int av, ir, ig, ib, il, qr, qg, qb, ql;
av = r * .299 + g * .587 + b * .114 + .5;
ql = (il = av > 238 ? 23 : (av - 3) / 10) * 10 + 8;
qr = cube[(ir = UNCUBE(r))];
qg = cube[(ig = UNCUBE(g))];
qb = cube[(ib = UNCUBE(b))];
if (SQR(qr - r) + SQR(qg - g) + SQR(qb - b) <=
SQR(ql - r) + SQR(ql - g) + SQR(ql - b))
return ir * 36 + ig * 6 + ib + 020;
return il + 0350;
}
static std::string set_xterm256_foreground(int r, int g, int b) {
int x = rgb2xterm256(r, g, b);
std::ostringstream oss;
oss << "\033[38;5;" << x << "m";
return oss.str();
}
// Lowest is red, middle is yellow, highest is green. Color scheme from
// Paul Tol; it is colorblind friendly https://sronpersonalpages.nl/~pault
const std::vector<std::string> k_colors = {
set_xterm256_foreground(220, 5, 12),
set_xterm256_foreground(232, 96, 28),
set_xterm256_foreground(241, 147, 45),
set_xterm256_foreground(246, 193, 65),
set_xterm256_foreground(247, 240, 86),
set_xterm256_foreground(144, 201, 135),
set_xterm256_foreground( 78, 178, 101),
};
// ANSI formatting codes
static std::string set_inverse() {
return "\033[7m";
}
static std::string set_underline() {
return "\033[4m";
}
static std::string set_dim() {
return "\033[2m";
}
// Style scheme for different confidence levels
const std::vector<std::string> k_styles = {
set_inverse(), // Low confidence - inverse (highlighted)
set_underline(), // Medium confidence - underlined
set_dim(), // High confidence - dim
};
//
// Other utils
//
// check if file exists using ifstream
bool is_file_exist(const char * filename);
engine/ggml/examples/gpt-2/CMakeLists.txt
+32
View File
@@ -0,0 +1,32 @@
#
# gpt-2
set(TEST_TARGET gpt-2-ctx)
add_executable(${TEST_TARGET} main-ctx.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-alloc)
add_executable(${TEST_TARGET} main-alloc.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-backend)
add_executable(${TEST_TARGET} main-backend.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
set(TEST_TARGET gpt-2-sched)
add_executable(${TEST_TARGET} main-sched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-quantize
set(TEST_TARGET gpt-2-quantize)
add_executable(${TEST_TARGET} quantize.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)
#
# gpt-2-batched
set(TEST_TARGET gpt-2-batched)
add_executable(${TEST_TARGET} main-batched.cpp)
target_link_libraries(${TEST_TARGET} PRIVATE ggml common common-ggml)

Some files were not shown because too many files have changed in this diff Show More