Initial release
This commit is contained in:
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Convert every stage-1 geometry checkpoint to GGUF (f16 for the demo, f32 for
|
||||
# validation). Run inside the reference container:
|
||||
# docker run --rm -v "$PWD":/work -w /work trellis2-ref bash scripts/convert_all.sh
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
mkdir -p ggufs
|
||||
|
||||
T2=models/TRELLIS.2-4B/ckpts
|
||||
T1=models/TRELLIS-image-large/ckpts
|
||||
PJ=models/TRELLIS.2-4B/pipeline.json
|
||||
TPJ=models/TRELLIS.2-4B/texturing_pipeline.json
|
||||
|
||||
for ft in 0 1; do
|
||||
suf=$([ "$ft" = 0 ] && echo f32 || echo f16)
|
||||
python convert_dino_to_gguf.py --output ggufs/dino_$suf.gguf --ftype "$ft"
|
||||
python convert_ss_flow_to_gguf.py --model $T2/ss_flow_img_dit_1_3B_64_bf16.safetensors --output ggufs/ss_flow_$suf.gguf --ftype "$ft"
|
||||
python convert_ss_dec_to_gguf.py --model $T1/ss_dec_conv3d_16l8_fp16.safetensors --output ggufs/ss_dec_$suf.gguf --ftype "$ft"
|
||||
python convert_slat_flow_to_gguf.py --model $T2/slat_flow_img2shape_dit_1_3B_512_bf16.safetensors --pipeline-json $PJ --output ggufs/slat_flow_$suf.gguf --ftype "$ft"
|
||||
python convert_slat_flow_to_gguf.py --model $T2/slat_flow_img2shape_dit_1_3B_1024_bf16.safetensors --pipeline-json $PJ --output ggufs/slat_flow_1024_$suf.gguf --ftype "$ft"
|
||||
python convert_shape_dec_to_gguf.py --model $T2/shape_dec_next_dc_f16c32_fp16.safetensors --output ggufs/shape_dec_$suf.gguf --ftype "$ft"
|
||||
# PBR texturing stack (shape encoder retained for standalone parity tooling)
|
||||
python convert_tex_dec_to_gguf.py --model $T2/tex_dec_next_dc_f16c32_fp16.safetensors --output ggufs/tex_dec_$suf.gguf --ftype "$ft"
|
||||
python convert_shape_enc_to_gguf.py --model $T2/shape_enc_next_dc_f16c32_fp16.safetensors --output ggufs/shape_enc_$suf.gguf --ftype "$ft"
|
||||
python convert_tex_flow_to_gguf.py --model $T2/slat_flow_imgshape2tex_dit_1_3B_512_bf16.safetensors --texturing-json $TPJ --output ggufs/tex_slat_flow_512_$suf.gguf --ftype "$ft"
|
||||
python convert_tex_flow_to_gguf.py --model $T2/slat_flow_imgshape2tex_dit_1_3B_1024_bf16.safetensors --texturing-json $TPJ --output ggufs/tex_slat_flow_1024_$suf.gguf --ftype "$ft"
|
||||
done
|
||||
|
||||
echo "all GGUFs written:"
|
||||
ls -la ggufs/
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and launch the demo server (CUDA lib + Go server) in the demo container.
|
||||
# The container carries a matching glibc/libstdc++/Go so both the CUDA
|
||||
# libtrellis2.so and the Go binary that dlopens it share one runtime — a NixOS
|
||||
# host binary won't run inside the CUDA image (different dynamic loader).
|
||||
#
|
||||
# scripts/demo.sh # fine path (needs the shape-SLAT GGUFs)
|
||||
# scripts/demo.sh -coarse # 64^3 marching-cubes preview only
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
PORT="${PORT:-8742}"
|
||||
|
||||
docker build -f docker/Dockerfile.demo -t trellis2-demo docker
|
||||
|
||||
docker run --rm -v "$ROOT":/work -w /work -e GOCACHE=/tmp/gocache trellis2-demo bash -c '
|
||||
cmake -B build-cuda-shared -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON \
|
||||
-DCMAKE_CUDA_ARCHITECTURES=120 -DBUILD_SHARED_LIBS=ON \
|
||||
-DTRELLIS2_FETCH_PRINT_REMESH_DEPS=ON \
|
||||
-DTRELLIS2_PRINT_REMESH_DEPS_DIR=/work/.deps/print-remesh >/dev/null 2>&1 &&
|
||||
cmake --build build-cuda-shared -j"$(nproc)" &&
|
||||
cd server && CGO_ENABLED=0 go build -o trellis2-server-linux .'
|
||||
|
||||
# Fetch prebuilt f16 GGUFs from the public LocalAI-io repos. Files already present
|
||||
# are skipped, so this is a no-op once ggufs/ is populated; it lets fresh demo
|
||||
# users skip the separate download_models.sh + convert_all.sh steps.
|
||||
scripts/download_ggufs.sh
|
||||
|
||||
docker rm -f trellis2-demo-run 2>/dev/null || true
|
||||
exec docker run --rm --name trellis2-demo-run --device nvidia.com/gpu=all \
|
||||
-v "$ROOT":/work -w /work/server -p "$PORT":8742 trellis2-demo \
|
||||
./trellis2-server-linux -lib /work/build-cuda-shared/libtrellis2.so \
|
||||
-ggufs /work/ggufs -store /work/generations -unload-idle -addr :8742 "$@"
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the prebuilt f16 GGUFs for the demo from the public LocalAI-io repos.
|
||||
#
|
||||
# This is the fast path for running the demo: it replaces the two-step
|
||||
# download_models.sh (safetensors) + convert_all.sh (GGUF conversion) flow with a
|
||||
# direct pull of the ready-made f16 GGUFs. Developers who need the f32 validation
|
||||
# variants or want to regenerate GGUFs should still use those two scripts.
|
||||
#
|
||||
# Files land in $GGUFS (default: repo-root ggufs/). Already-present files are
|
||||
# skipped, so re-runs are cheap and downloads resume (-C -). No auth needed
|
||||
# (public repos); HF_TOKEN is used if set.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
GGUFS="${GGUFS:-$ROOT/ggufs}"
|
||||
ORG="${GGUF_ORG:-LocalAI-io}"
|
||||
TOKEN="${HF_TOKEN:-}"
|
||||
AUTH=()
|
||||
[ -n "$TOKEN" ] && AUTH=(-H "Authorization: Bearer $TOKEN")
|
||||
|
||||
mkdir -p "$GGUFS"
|
||||
|
||||
fetch() { # fetch <repo> <file>
|
||||
local url="https://huggingface.co/$ORG/$1/resolve/main/$2"
|
||||
local dest="$GGUFS/$2"
|
||||
if [ -s "$dest" ]; then echo "have $dest"; return 0; fi
|
||||
echo "fetch $ORG/$1/$2"
|
||||
curl -sSL --fail -C - "${AUTH[@]}" -o "$dest.part" "$url"
|
||||
mv "$dest.part" "$dest"
|
||||
}
|
||||
|
||||
T2=TRELLIS.2-4B-GGUF
|
||||
T1=TRELLIS-image-large-GGUF
|
||||
DINO=dinov3-vitl16-pretrain-lvd1689m-GGUF
|
||||
|
||||
fetch "$DINO" dino_f16.gguf
|
||||
fetch "$T1" ss_dec_f16.gguf
|
||||
fetch "$T2" ss_flow_f16.gguf
|
||||
fetch "$T2" slat_flow_f16.gguf
|
||||
fetch "$T2" slat_flow_1024_f16.gguf
|
||||
fetch "$T2" shape_dec_f16.gguf
|
||||
fetch "$T2" shape_enc_f16.gguf
|
||||
fetch "$T2" tex_dec_f16.gguf
|
||||
fetch "$T2" tex_slat_flow_512_f16.gguf
|
||||
fetch "$T2" tex_slat_flow_1024_f16.gguf
|
||||
|
||||
echo "all GGUFs present in $GGUFS:"
|
||||
du -sh "$GGUFS"
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download the checkpoints needed for the stage-1 geometry pipeline ("512" path).
|
||||
#
|
||||
# NOTE on DINOv3: the official facebook/dinov3-vitl16-pretrain-lvd1689m repo is
|
||||
# gated behind a license-acceptance click. Until access is granted on your HF
|
||||
# account, this script pulls the widely-used ungated mirror
|
||||
# camenduru/dinov3-vitl16-pretrain-lvd1689m (byte-identical HF-format export,
|
||||
# ships Meta's LICENSE.md). Re-point DINO_REPO at the official repo once you
|
||||
# have access.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
MODELS="$ROOT/models"
|
||||
TOKEN="${HF_TOKEN:-$(cat ~/.cache/huggingface/token 2>/dev/null || true)}"
|
||||
AUTH=()
|
||||
[ -n "$TOKEN" ] && AUTH=(-H "Authorization: Bearer $TOKEN")
|
||||
|
||||
fetch() { # fetch <repo> <rfile> <dest-subdir>
|
||||
local url="https://huggingface.co/$1/resolve/main/$2"
|
||||
local dest="$MODELS/$3/$2"
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
if [ -s "$dest" ]; then echo "have $dest"; return 0; fi
|
||||
echo "fetch $1/$2"
|
||||
curl -sSL --fail -C - "${AUTH[@]}" -o "$dest.part" "$url"
|
||||
mv "$dest.part" "$dest"
|
||||
}
|
||||
|
||||
T2=microsoft/TRELLIS.2-4B
|
||||
T1=microsoft/TRELLIS-image-large
|
||||
DINO_REPO="${DINO_REPO:-camenduru/dinov3-vitl16-pretrain-lvd1689m}"
|
||||
|
||||
fetch $T2 pipeline.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/ss_flow_img_dit_1_3B_64_bf16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/ss_flow_img_dit_1_3B_64_bf16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_img2shape_dit_1_3B_512_bf16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_img2shape_dit_1_3B_512_bf16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_img2shape_dit_1_3B_1024_bf16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_img2shape_dit_1_3B_1024_bf16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/shape_dec_next_dc_f16c32_fp16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/shape_dec_next_dc_f16c32_fp16.safetensors TRELLIS.2-4B
|
||||
# PBR texturing stack: the validated generation path needs the shape encoder,
|
||||
# texture decoder, and both texture-SLAT flows.
|
||||
fetch $T2 texturing_pipeline.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/shape_enc_next_dc_f16c32_fp16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/shape_enc_next_dc_f16c32_fp16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/tex_dec_next_dc_f16c32_fp16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/tex_dec_next_dc_f16c32_fp16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_imgshape2tex_dit_1_3B_512_bf16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_imgshape2tex_dit_1_3B_512_bf16.safetensors TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_imgshape2tex_dit_1_3B_1024_bf16.json TRELLIS.2-4B
|
||||
fetch $T2 ckpts/slat_flow_imgshape2tex_dit_1_3B_1024_bf16.safetensors TRELLIS.2-4B
|
||||
fetch $T1 ckpts/ss_dec_conv3d_16l8_fp16.json TRELLIS-image-large
|
||||
fetch $T1 ckpts/ss_dec_conv3d_16l8_fp16.safetensors TRELLIS-image-large
|
||||
fetch "$DINO_REPO" config.json dinov3-vitl16
|
||||
fetch "$DINO_REPO" preprocessor_config.json dinov3-vitl16
|
||||
fetch "$DINO_REPO" model.safetensors dinov3-vitl16
|
||||
|
||||
echo "all downloads complete:"
|
||||
du -sh "$MODELS"/*
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference dumps for the 1024_cascade high-resolution geometry stage.
|
||||
|
||||
Produces dumps/reference_cascade.gguf with the full HR chain so the C++ port can
|
||||
be validated stage by stage:
|
||||
|
||||
cond_512, cond_1024 the two DINOv3 conds (embedded → self-contained test)
|
||||
coords32 [L,4] 32^3 scaffold (from the SS reference latent)
|
||||
lr_noise [L,32] LR sampling noise (seed 4321)
|
||||
lr_slat [L,32] 512-model sampler output, denormalized
|
||||
up_coords [Nup,4] decoder.upsample(lr_slat, upsample_times=4) → 512^3
|
||||
hr_coords [Lhr,4] quantized+unique → 64^3 (the HR flow scaffold)
|
||||
hr_noise [Lhr,32] HR sampling noise (seed 5678)
|
||||
hr_flow_t500_out [Lhr,32] 1024-model flow forward at t=500 on hr_coords
|
||||
hr_slat [Lhr,32] 1024-model sampler output, denormalized
|
||||
lvl{i}.*, out7, out_coords per-level decode taps at resolution 1024 (→ 1024^3)
|
||||
|
||||
The reference is generated with TF32 disabled (ref_common.setup) so the golden
|
||||
values are true fp32. Run inside the container (see scripts/refgen.sh).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import ref_common # noqa: E402
|
||||
|
||||
ref_common.setup() # TF32 off + sdpa sparse attention + pure-torch sparse conv
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import torch # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
|
||||
|
||||
def read_dinodata(path):
|
||||
with open(path, "rb") as f:
|
||||
assert f.read(8) == b"DINOCOND", "bad magic"
|
||||
_, _, ndim = struct.unpack("<III", f.read(12))
|
||||
shape = struct.unpack("<%dI" % ndim, f.read(4 * ndim))
|
||||
arr = np.frombuffer(f.read(), dtype="<f4").reshape(shape)
|
||||
return arr
|
||||
|
||||
|
||||
def load_ss_sample_latent(path):
|
||||
with open(path, "rb") as f:
|
||||
assert f.read(8) == b"SSSAMP01", "bad magic"
|
||||
R, cin, lkv, cctx, steps = struct.unpack("<5i", f.read(20))
|
||||
n = cin * R * R * R
|
||||
f.seek(-(n * 4), os.SEEK_END)
|
||||
z = np.frombuffer(f.read(n * 4), dtype="<f4").reshape(1, cin, R, R, R)
|
||||
return torch.from_numpy(z.copy())
|
||||
|
||||
|
||||
def load_flow(stem, dev):
|
||||
from safetensors.torch import load_file
|
||||
from trellis2.models.structured_latent_flow import SLatFlowModel
|
||||
with open(stem + ".json") as f:
|
||||
cfg = json.load(f)["args"]
|
||||
cfg.pop("initialization", None)
|
||||
cfg.pop("dtype", None)
|
||||
m = SLatFlowModel(**cfg, dtype="float32")
|
||||
sd = {k: v.float() for k, v in load_file(stem + ".safetensors").items()}
|
||||
missing, unexpected = m.load_state_dict(sd, strict=False)
|
||||
assert not unexpected, unexpected
|
||||
m.convert_to(torch.float32)
|
||||
m.eval().to(dev)
|
||||
return m, cfg
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--models", default=os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "ckpts"))
|
||||
ap.add_argument("--ss-dec", default=os.path.join(ref_common.MODELS, "TRELLIS-image-large",
|
||||
"ckpts", "ss_dec_conv3d_16l8_fp16"))
|
||||
ap.add_argument("--cond-512", default=os.path.join(ref_common.DUMPS, "fixture.dinodata"))
|
||||
ap.add_argument("--cond-1024", default=os.path.join(ref_common.DUMPS, "fixture_1024.dinodata"))
|
||||
ap.add_argument("--ss-latent", default=os.path.join(ref_common.REPO, "tests", "ss_sample_ref.bin"))
|
||||
ap.add_argument("--pipeline-json", default=os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "pipeline.json"))
|
||||
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
ap.add_argument("--t", type=float, default=500.0)
|
||||
ap.add_argument("--lr-seed", type=int, default=4321)
|
||||
ap.add_argument("--hr-seed", type=int, default=5678)
|
||||
ap.add_argument("--lr-resolution", type=int, default=512)
|
||||
ap.add_argument("--resolution", type=int, default=1024)
|
||||
ap.add_argument("--out", default=os.path.join(ref_common.DUMPS, "reference_cascade.gguf"))
|
||||
args = ap.parse_args()
|
||||
|
||||
from safetensors.torch import load_file
|
||||
from trellis2.models.sparse_structure_vae import SparseStructureDecoder
|
||||
from trellis2.models.sc_vaes.fdg_vae import FlexiDualGridVaeDecoder
|
||||
from trellis2.pipelines.samplers import FlowEulerGuidanceIntervalSampler
|
||||
from trellis2.modules import sparse as sp
|
||||
|
||||
dev = torch.device(args.device)
|
||||
caps = {}
|
||||
|
||||
cond_512 = torch.from_numpy(read_dinodata(args.cond_512).copy()).float().to(dev)
|
||||
cond_1024 = torch.from_numpy(read_dinodata(args.cond_1024).copy()).float().to(dev)
|
||||
caps["cond_512"] = cond_512[0]
|
||||
caps["cond_1024"] = cond_1024[0]
|
||||
|
||||
with open(args.pipeline_json) as f:
|
||||
pj = json.load(f)["args"]
|
||||
norm = pj["shape_slat_normalization"]
|
||||
sampler_params = pj["shape_slat_sampler"]["params"]
|
||||
mean = torch.tensor(norm["mean"], device=dev)[None]
|
||||
std = torch.tensor(norm["std"], device=dev)[None]
|
||||
sampler = FlowEulerGuidanceIntervalSampler(sigma_min=1e-5)
|
||||
|
||||
# ── 32^3 scaffold from the SS reference latent ───────────────────────────
|
||||
with open(args.ss_dec + ".json") as f:
|
||||
ss_cfg = json.load(f)["args"]
|
||||
ss_cfg["use_fp16"] = False
|
||||
ss_dec = SparseStructureDecoder(**ss_cfg)
|
||||
ss_dec.load_state_dict({k: v.float() for k, v in load_file(args.ss_dec + ".safetensors").items()})
|
||||
ss_dec.dtype = torch.float32
|
||||
ss_dec.eval().float().to(dev)
|
||||
z_s = load_ss_sample_latent(args.ss_latent).to(dev)
|
||||
with torch.no_grad():
|
||||
occ = ss_dec(z_s) > 0
|
||||
occ = (F.max_pool3d(occ.float(), 2, 2, 0) > 0.5)
|
||||
coords = torch.argwhere(occ)[:, [0, 2, 3, 4]].int().contiguous()
|
||||
L = coords.shape[0]
|
||||
caps["coords32"] = coords.float()
|
||||
print(f"scaffold: {L} voxels at 32^3")
|
||||
del ss_dec
|
||||
|
||||
# ── LR flow: sample with the 512 model + cond_512, denormalize ───────────
|
||||
flow_lr, cfg_lr = load_flow(os.path.join(args.models, "slat_flow_img2shape_dit_1_3B_512_bf16"), dev)
|
||||
g = torch.Generator().manual_seed(args.lr_seed)
|
||||
lr_noise = torch.randn(L, cfg_lr["in_channels"], generator=g).to(dev)
|
||||
caps["lr_noise"] = lr_noise
|
||||
x0 = sp.SparseTensor(feats=lr_noise.clone(), coords=coords.to(dev))
|
||||
with torch.no_grad():
|
||||
lr_slat = sampler.sample(flow_lr, x0, cond=cond_512, neg_cond=torch.zeros_like(cond_512),
|
||||
**sampler_params, verbose=True).samples
|
||||
lr_slat = lr_slat * std + mean
|
||||
caps["lr_slat"] = lr_slat.feats
|
||||
print(f"lr_slat: {lr_slat.feats.shape} mean={lr_slat.feats.mean().item():.5f}")
|
||||
del flow_lr
|
||||
if dev.type == "cuda":
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ── shape decoder: upsample(×4) → 512^3 candidate coords ─────────────────
|
||||
dstem = os.path.join(args.models, "shape_dec_next_dc_f16c32_fp16")
|
||||
with open(dstem + ".json") as f:
|
||||
dcfg = json.load(f)["args"]
|
||||
dcfg.pop("use_fp16", None)
|
||||
dcfg.pop("resolution", None)
|
||||
dec = FlexiDualGridVaeDecoder(resolution=args.resolution, use_fp16=False, **dcfg)
|
||||
dec.load_state_dict({k: v.float() for k, v in load_file(dstem + ".safetensors").items()})
|
||||
# The decoder runs on CPU: the 1024^3 expansion materializes millions of
|
||||
# voxels through the pure-torch sparse conv and would OOM the 16 GB GPU (and
|
||||
# it is exactly what the C++ port runs on CPU). Flows stay on GPU.
|
||||
dec.eval().float().cpu()
|
||||
|
||||
with torch.no_grad():
|
||||
up_coords = dec.upsample(lr_slat.cpu(), upsample_times=4) # [Nup, 4] at 512^3
|
||||
caps["up_coords"] = up_coords.float()
|
||||
print(f"upsample: {up_coords.shape[0]} candidate coords at {args.lr_resolution}^3")
|
||||
|
||||
# ── quantize + unique → 64^3 HR scaffold (verbatim pipeline formula) ─────
|
||||
hr_res = args.resolution
|
||||
quant_coords = torch.cat([
|
||||
up_coords[:, :1],
|
||||
((up_coords[:, 1:] + 0.5) / args.lr_resolution * (hr_res // 16)).int(),
|
||||
], dim=1)
|
||||
hr_coords = quant_coords.unique(dim=0)
|
||||
Lhr = hr_coords.shape[0]
|
||||
caps["hr_coords"] = hr_coords.float()
|
||||
print(f"hr scaffold: {Lhr} voxels at {hr_res // 16}^3")
|
||||
|
||||
# ── HR flow: forward @ t=500 + full sampler with 1024 model + cond_1024 ──
|
||||
flow_hr, cfg_hr = load_flow(os.path.join(args.models, "slat_flow_img2shape_dit_1_3B_1024_bf16"), dev)
|
||||
g = torch.Generator().manual_seed(args.hr_seed)
|
||||
hr_noise = torch.randn(Lhr, cfg_hr["in_channels"], generator=g).to(dev)
|
||||
caps["hr_noise"] = hr_noise
|
||||
xh = sp.SparseTensor(feats=hr_noise.clone(), coords=hr_coords.to(dev))
|
||||
with torch.no_grad():
|
||||
out = flow_hr(xh, torch.tensor([args.t], device=dev), cond_1024)
|
||||
caps["hr_flow_t500_out"] = out.feats
|
||||
print(f"hr flow t={args.t}: l2={out.feats.norm().item():.4f}")
|
||||
|
||||
xh0 = sp.SparseTensor(feats=hr_noise.clone(), coords=hr_coords.to(dev))
|
||||
with torch.no_grad():
|
||||
hr_slat = sampler.sample(flow_hr, xh0, cond=cond_1024, neg_cond=torch.zeros_like(cond_1024),
|
||||
**sampler_params, verbose=True).samples
|
||||
hr_slat = hr_slat * std + mean
|
||||
caps["hr_slat"] = hr_slat.feats
|
||||
print(f"hr_slat: {hr_slat.feats.shape} mean={hr_slat.feats.mean().item():.5f}")
|
||||
del flow_hr
|
||||
if dev.type == "cuda":
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ── final decode of the HR slat at resolution 1024, level by level ───────
|
||||
# Only the final 7-channel output is kept: the per-level intermediates are
|
||||
# multiple GB at 1024^3 (and the decoder's level logic is already validated
|
||||
# exactly at the 512 tier). We run the base SparseUnetVaeDecoder forward
|
||||
# manually to get the raw 7 channels — dec(...) would run the FDG mesh
|
||||
# conversion (stubbed o_voxel). Running the decode on CPU is host-RAM heavy.
|
||||
dec.set_resolution(args.resolution)
|
||||
hr_slat_cpu = hr_slat.cpu()
|
||||
with torch.no_grad():
|
||||
h = dec.from_latent(hr_slat_cpu.float())
|
||||
for i, res in enumerate(dec.blocks):
|
||||
for j, block in enumerate(res):
|
||||
if i < len(dec.blocks) - 1 and j == len(res) - 1:
|
||||
h, _sub = block(h)
|
||||
else:
|
||||
h = block(h)
|
||||
print(f"decode level {i}: {h.feats.shape[0]} voxels x {h.feats.shape[1]} ch")
|
||||
hn = h.replace(F.layer_norm(h.feats, h.feats.shape[-1:]))
|
||||
out7 = dec.output_layer(hn)
|
||||
caps["out7"] = out7.feats
|
||||
caps["out_coords"] = out7.coords.float()
|
||||
print(f"out7: {out7.feats.shape}")
|
||||
|
||||
import gguf
|
||||
writer = gguf.GGUFWriter(args.out, "reference")
|
||||
manifest = {"shapes": {}, "atol": 2e-3, "rtol": 2e-3,
|
||||
"lr_resolution": args.lr_resolution, "resolution": args.resolution}
|
||||
for name, t in caps.items():
|
||||
a = t.detach().cpu().float().numpy()
|
||||
manifest["shapes"][name] = list(a.shape)
|
||||
writer.add_tensor(name, np.ascontiguousarray(a.reshape(-1), dtype=np.float32))
|
||||
writer.write_header_to_file()
|
||||
writer.write_kv_data_to_file()
|
||||
writer.write_tensors_to_file()
|
||||
writer.close()
|
||||
with open(os.path.join(ref_common.DUMPS, "manifest_cascade.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(f"wrote {args.out} ({os.path.getsize(args.out):,} bytes), {len(caps)} tensors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dump DINOv3 ViT-L/16 reference activations for the C++ port.
|
||||
|
||||
Replicates DinoV3FeatureExtractor exactly (manual embeddings -> rope -> layer
|
||||
loop -> affine-free layer_norm; the model's own final layernorm is NOT applied)
|
||||
on the preprocessed fixture image, and writes:
|
||||
|
||||
dumps/reference_dino.gguf input pixels + per-layer taps + final cond
|
||||
dumps/manifest_dino.json shapes + tolerances
|
||||
dumps/fixture.dinodata the conditioning tensor for the SS-flow tests
|
||||
dumps/fixture_pre.png preprocessed (cropped, premultiplied) image
|
||||
dumps/fixture_512.png the exact 512x512 LANCZOS-resized uint8 image
|
||||
|
||||
Run inside the reference container (see scripts/refgen.sh):
|
||||
python scripts/dump_dino_reference.py --image <rgba image> [--device cuda]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import ref_common # noqa: E402 (sets sys.path for trellis2, stubs cumesh)
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import torch # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--image", required=True, help="RGBA input image")
|
||||
ap.add_argument("--model", default=os.path.join(ref_common.MODELS, "dinov3-vitl16"))
|
||||
ap.add_argument("--resolution", type=int, default=512)
|
||||
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
ap.add_argument("--out", default=os.path.join(ref_common.DUMPS, "reference_dino.gguf"))
|
||||
args = ap.parse_args()
|
||||
|
||||
os.makedirs(ref_common.DUMPS, exist_ok=True)
|
||||
|
||||
from transformers import DINOv3ViTModel
|
||||
|
||||
model = DINOv3ViTModel.from_pretrained(args.model)
|
||||
model.eval().float().to(args.device)
|
||||
|
||||
img = Image.open(args.image).convert("RGBA")
|
||||
img.save(os.path.join(ref_common.DUMPS, "fixture_rgba.png"))
|
||||
pre = ref_common.preprocess_rgba(img)
|
||||
pre.save(os.path.join(ref_common.DUMPS, "fixture_pre.png"))
|
||||
|
||||
resized = pre.resize((args.resolution, args.resolution), Image.Resampling.LANCZOS)
|
||||
resized.save(os.path.join(ref_common.DUMPS, "fixture_512.png"))
|
||||
|
||||
x = np.array(resized).astype(np.float32) / 255.0 # HWC
|
||||
x = torch.from_numpy(x).permute(2, 0, 1).unsqueeze(0) # 1CHW
|
||||
mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)
|
||||
std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)
|
||||
pixel_values = ((x - mean) / std).to(args.device)
|
||||
|
||||
caps = {}
|
||||
caps["img_512_u8"] = torch.from_numpy(np.array(resized).astype(np.float32))
|
||||
caps["pixel_values"] = pixel_values
|
||||
|
||||
# Detail taps inside the first and last layer via forward hooks.
|
||||
detail_layers = {0, len(model.layer) - 1}
|
||||
hooks = []
|
||||
|
||||
def tap(name):
|
||||
def fn(_m, _inp, out):
|
||||
o = out[0] if isinstance(out, tuple) else out
|
||||
caps[name] = o.detach()
|
||||
return fn
|
||||
|
||||
for i in sorted(detail_layers):
|
||||
layer = model.layer[i]
|
||||
for sub in ("norm1", "attention", "layer_scale1", "norm2", "mlp", "layer_scale2"):
|
||||
m = getattr(layer, sub, None)
|
||||
if m is not None:
|
||||
hooks.append(m.register_forward_hook(tap(f"l{i}.{sub}")))
|
||||
|
||||
with torch.no_grad():
|
||||
hidden = model.embeddings(pixel_values, bool_masked_pos=None)
|
||||
caps["embd"] = hidden
|
||||
rope = model.rope_embeddings(pixel_values)
|
||||
if isinstance(rope, (tuple, list)):
|
||||
for j, r in enumerate(rope):
|
||||
caps[f"rope_{j}"] = r
|
||||
else:
|
||||
caps["rope_0"] = rope
|
||||
for i, layer_module in enumerate(model.layer):
|
||||
hidden = layer_module(hidden, position_embeddings=rope)
|
||||
if isinstance(hidden, tuple):
|
||||
hidden = hidden[0]
|
||||
caps[f"l{i}.out"] = hidden
|
||||
cond = F.layer_norm(hidden, hidden.shape[-1:])
|
||||
caps["cond"] = cond
|
||||
|
||||
for h in hooks:
|
||||
h.remove()
|
||||
|
||||
cond_np = cond.cpu().numpy().astype(np.float32)
|
||||
ref_common.write_dinodata(os.path.join(ref_common.DUMPS, "fixture.dinodata"), cond_np)
|
||||
print(f"cond: shape={tuple(cond_np.shape)} mean={cond_np.mean():.6f} "
|
||||
f"min={cond_np.min():.4f} max={cond_np.max():.4f} l2={np.linalg.norm(cond_np):.4f}")
|
||||
|
||||
# Also emit the 1024-resolution conditioning (4101 tokens) that the HR stage
|
||||
# of the 1024 cascade consumes. Same encode path, image_size 1024.
|
||||
resized_hr = pre.resize((1024, 1024), Image.Resampling.LANCZOS)
|
||||
xhr = np.array(resized_hr).astype(np.float32) / 255.0
|
||||
xhr = torch.from_numpy(xhr).permute(2, 0, 1).unsqueeze(0)
|
||||
pv_hr = ((xhr - mean) / std).to(args.device)
|
||||
with torch.no_grad():
|
||||
h = model.embeddings(pv_hr, bool_masked_pos=None)
|
||||
rope_hr = model.rope_embeddings(pv_hr)
|
||||
for layer_module in model.layer:
|
||||
h = layer_module(h, position_embeddings=rope_hr)
|
||||
if isinstance(h, tuple):
|
||||
h = h[0]
|
||||
cond_hr = F.layer_norm(h, h.shape[-1:]).cpu().numpy().astype(np.float32)
|
||||
ref_common.write_dinodata(os.path.join(ref_common.DUMPS, "fixture_1024.dinodata"), cond_hr)
|
||||
print(f"cond_1024: shape={tuple(cond_hr.shape)} l2={np.linalg.norm(cond_hr):.4f}")
|
||||
|
||||
import gguf
|
||||
writer = gguf.GGUFWriter(args.out, "reference")
|
||||
manifest = {"resolution": args.resolution, "atol": 2e-3, "rtol": 2e-3, "shapes": {}}
|
||||
for name, t in caps.items():
|
||||
a = t.detach().cpu().float().numpy()
|
||||
manifest["shapes"][name] = list(a.shape)
|
||||
writer.add_tensor(name, np.ascontiguousarray(a.reshape(-1), dtype=np.float32))
|
||||
writer.write_header_to_file()
|
||||
writer.write_kv_data_to_file()
|
||||
writer.write_tensors_to_file()
|
||||
writer.close()
|
||||
|
||||
with open(os.path.join(ref_common.DUMPS, "manifest_dino.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(f"wrote {args.out} ({os.path.getsize(args.out):,} bytes), "
|
||||
f"{len(caps)} tensors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference dumps for the shape-SLAT stage ("512" pipeline type, geometry only).
|
||||
|
||||
Produces dumps/reference_slat.gguf with:
|
||||
|
||||
coords [L, 4] f32 active voxels at 32^3 (from the SS stage)
|
||||
slat_noise [L, 32] fixed sampling noise (seed 4321)
|
||||
flow_t500_out [L, 32] SLAT flow forward at t=500 (f32)
|
||||
slat [L, 32] full 12-step sampler output, denormalized
|
||||
slat_mean/std [32] shape_slat_normalization
|
||||
lvl{i}.in_coords [L_i, 4] decoder level i active voxels
|
||||
lvl{i}.pre_up [L_i, C_i] features after level i's ConvNeXt blocks
|
||||
lvl{i}.subdiv [L_i, 8] subdivision logits of level i's up-block
|
||||
out7 [L_4, 7] decoder output (pre split/sigmoid)
|
||||
out_coords [L_4, 4]
|
||||
|
||||
The coords come from decoding tests/ss_sample_ref.bin's reference latent, so
|
||||
the same scaffold is reproducible from the validated C++ SS stages.
|
||||
|
||||
Run inside the container: see scripts/refgen.sh.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import ref_common # noqa: E402
|
||||
|
||||
ref_common.setup() # sdpa sparse attention + pure-torch sparse conv
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import torch # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
|
||||
|
||||
def load_ss_sample_latent(path):
|
||||
"""Read the z_s reference produced by tests/ref_ss_sample.py (SSSAMP01)."""
|
||||
with open(path, "rb") as f:
|
||||
assert f.read(8) == b"SSSAMP01", "bad magic"
|
||||
R, cin, lkv, cctx, steps = struct.unpack("<5i", f.read(20))
|
||||
f.read(4 * 3) # gs, rescale, rescale_t
|
||||
f.read(8) # seed
|
||||
n = cin * R * R * R
|
||||
f.seek(-(n * 4), os.SEEK_END)
|
||||
z = np.frombuffer(f.read(n * 4), dtype="<f4").reshape(1, cin, R, R, R)
|
||||
return torch.from_numpy(z.copy())
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--models", default=os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "ckpts"))
|
||||
ap.add_argument("--ss-dec", default=os.path.join(ref_common.MODELS, "TRELLIS-image-large",
|
||||
"ckpts", "ss_dec_conv3d_16l8_fp16"))
|
||||
ap.add_argument("--dinodata", default=os.path.join(ref_common.DUMPS, "fixture.dinodata"))
|
||||
ap.add_argument("--ss-latent", default=os.path.join(ref_common.REPO, "tests", "ss_sample_ref.bin"))
|
||||
ap.add_argument("--pipeline-json", default=os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "pipeline.json"))
|
||||
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
ap.add_argument("--t", type=float, default=500.0)
|
||||
ap.add_argument("--seed", type=int, default=4321)
|
||||
ap.add_argument("--resolution", type=int, default=512)
|
||||
ap.add_argument("--out", default=os.path.join(ref_common.DUMPS, "reference_slat.gguf"))
|
||||
args = ap.parse_args()
|
||||
|
||||
from safetensors.torch import load_file
|
||||
from trellis2.models.sparse_structure_vae import SparseStructureDecoder
|
||||
from trellis2.models.structured_latent_flow import SLatFlowModel
|
||||
from trellis2.models.sc_vaes.fdg_vae import FlexiDualGridVaeDecoder
|
||||
from trellis2.pipelines.samplers import FlowEulerGuidanceIntervalSampler
|
||||
from trellis2.modules import sparse as sp
|
||||
|
||||
dev = torch.device(args.device)
|
||||
caps = {}
|
||||
|
||||
# ── coords from the SS stage reference latent ────────────────────────────
|
||||
with open(args.ss_dec + ".json") as f:
|
||||
ss_cfg = json.load(f)["args"]
|
||||
ss_cfg["use_fp16"] = False
|
||||
ss_dec = SparseStructureDecoder(**ss_cfg)
|
||||
ss_dec.load_state_dict({k: v.float() for k, v in load_file(args.ss_dec + ".safetensors").items()})
|
||||
ss_dec.dtype = torch.float32
|
||||
ss_dec.eval().float().to(dev)
|
||||
|
||||
z_s = load_ss_sample_latent(args.ss_latent).to(dev)
|
||||
with torch.no_grad():
|
||||
occ = ss_dec(z_s) > 0 # [1,1,64,64,64]
|
||||
occ = (F.max_pool3d(occ.float(), 2, 2, 0) > 0.5) # ss_res 32
|
||||
coords = torch.argwhere(occ)[:, [0, 2, 3, 4]].int().contiguous()
|
||||
L = coords.shape[0]
|
||||
print(f"coords: {L} active voxels at 32^3 "
|
||||
f"({100.0 * L / 32**3:.2f}% occupancy)")
|
||||
caps["coords"] = coords.float()
|
||||
del ss_dec
|
||||
|
||||
# ── conditioning ─────────────────────────────────────────────────────────
|
||||
with open(args.dinodata, "rb") as f:
|
||||
assert f.read(8) == b"DINOCOND"
|
||||
_, _, ndim = struct.unpack("<III", f.read(12))
|
||||
shape = struct.unpack("<%dI" % ndim, f.read(4 * ndim))
|
||||
cond_np = np.frombuffer(f.read(), dtype="<f4").reshape(shape)
|
||||
cond = torch.from_numpy(cond_np.copy()).float().to(dev)
|
||||
neg_cond = torch.zeros_like(cond)
|
||||
|
||||
# ── SLAT flow: forward parity point + full sampler ───────────────────────
|
||||
stem = os.path.join(args.models, "slat_flow_img2shape_dit_1_3B_512_bf16")
|
||||
with open(stem + ".json") as f:
|
||||
cfg = json.load(f)["args"]
|
||||
cfg.pop("initialization", None)
|
||||
cfg.pop("dtype", None)
|
||||
flow = SLatFlowModel(**cfg, dtype="float32")
|
||||
sd = {k: v.float() for k, v in load_file(stem + ".safetensors").items()}
|
||||
missing, unexpected = flow.load_state_dict(sd, strict=False)
|
||||
assert not unexpected, unexpected
|
||||
flow.convert_to(torch.float32)
|
||||
flow.eval().to(dev)
|
||||
|
||||
g = torch.Generator().manual_seed(args.seed)
|
||||
noise = torch.randn(L, cfg["in_channels"], generator=g).to(dev)
|
||||
caps["slat_noise"] = noise
|
||||
|
||||
x = sp.SparseTensor(feats=noise.clone(), coords=coords.to(dev))
|
||||
with torch.no_grad():
|
||||
out = flow(x, torch.tensor([args.t], device=dev), cond)
|
||||
caps["flow_t500_out"] = out.feats
|
||||
print(f"flow t={args.t}: out mean={out.feats.mean().item():.6f} "
|
||||
f"l2={out.feats.norm().item():.4f}")
|
||||
|
||||
with open(args.pipeline_json) as f:
|
||||
pj = json.load(f)["args"]
|
||||
norm = pj["shape_slat_normalization"]
|
||||
sampler_params = pj["shape_slat_sampler"]["params"]
|
||||
print("sampler params:", sampler_params)
|
||||
|
||||
sampler = FlowEulerGuidanceIntervalSampler(sigma_min=1e-5)
|
||||
x0 = sp.SparseTensor(feats=noise.clone(), coords=coords.to(dev))
|
||||
with torch.no_grad():
|
||||
slat = sampler.sample(flow, x0, cond=cond, neg_cond=neg_cond,
|
||||
**sampler_params, verbose=True).samples
|
||||
mean = torch.tensor(norm["mean"], device=dev)[None]
|
||||
std = torch.tensor(norm["std"], device=dev)[None]
|
||||
slat = slat * std + mean
|
||||
caps["slat"] = slat.feats
|
||||
caps["slat_mean"] = mean[0]
|
||||
caps["slat_std"] = std[0]
|
||||
print(f"slat: mean={slat.feats.mean().item():.5f} std={slat.feats.std().item():.5f}")
|
||||
del flow
|
||||
if dev.type == "cuda":
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# ── FDG decoder, level by level (mirrors SparseUnetVaeDecoder.forward) ──
|
||||
dstem = os.path.join(args.models, "shape_dec_next_dc_f16c32_fp16")
|
||||
with open(dstem + ".json") as f:
|
||||
dcfg = json.load(f)["args"]
|
||||
dcfg.pop("use_fp16", None)
|
||||
dcfg.pop("resolution", None)
|
||||
dec = FlexiDualGridVaeDecoder(resolution=args.resolution, use_fp16=False, **dcfg)
|
||||
dec.load_state_dict({k: v.float() for k, v in load_file(dstem + ".safetensors").items()})
|
||||
dec.eval().float().to(dev)
|
||||
|
||||
with torch.no_grad():
|
||||
h = dec.from_latent(slat.float())
|
||||
for i, res in enumerate(dec.blocks):
|
||||
caps[f"lvl{i}.in_coords"] = h.coords.float()
|
||||
for j, block in enumerate(res):
|
||||
if i < len(dec.blocks) - 1 and j == len(res) - 1:
|
||||
caps[f"lvl{i}.pre_up"] = h.feats
|
||||
h, sub = block(h)
|
||||
caps[f"lvl{i}.subdiv"] = sub.feats
|
||||
else:
|
||||
h = block(h)
|
||||
print(f"level {i}: {h.feats.shape[0]} voxels x {h.feats.shape[1]} ch")
|
||||
hn = h.replace(F.layer_norm(h.feats, h.feats.shape[-1:]))
|
||||
out7 = dec.output_layer(hn)
|
||||
caps["out7"] = out7.feats
|
||||
caps["out_coords"] = out7.coords.float()
|
||||
print(f"out7: {out7.feats.shape}, offsets mean={torch.sigmoid(out7.feats[:, 0:3]).mean().item():.4f}, "
|
||||
f"intersected frac={(out7.feats[:, 3:6] > 0).float().mean().item():.4f}")
|
||||
|
||||
import gguf
|
||||
writer = gguf.GGUFWriter(args.out, "reference")
|
||||
manifest = {"shapes": {}, "atol": 2e-3, "rtol": 2e-3}
|
||||
for name, t in caps.items():
|
||||
a = t.detach().cpu().float().numpy()
|
||||
manifest["shapes"][name] = list(a.shape)
|
||||
writer.add_tensor(name, np.ascontiguousarray(a.reshape(-1), dtype=np.float32))
|
||||
writer.write_header_to_file()
|
||||
writer.write_kv_data_to_file()
|
||||
writer.write_tensors_to_file()
|
||||
writer.close()
|
||||
with open(os.path.join(ref_common.DUMPS, "manifest_slat.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(f"wrote {args.out} ({os.path.getsize(args.out):,} bytes), {len(caps)} tensors")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reference dumps for the PBR-texture stages, for validating the C++/ggml port.
|
||||
|
||||
Produces dumps/reference_texture.gguf with per-stage golden tensors so each C++
|
||||
stage validates in isolation (feed identical inputs, compare outputs):
|
||||
|
||||
cond [T, Cc] DINOv3 conditioning at texture resolution R
|
||||
enc_vert [N, 3] shape-encoder input: dual-vertex offsets (QEF)
|
||||
enc_inter [N, 3] shape-encoder input: intersection flags (QEF)
|
||||
enc_coords [N, 4] active voxels at R (batch-idx 0 + xyz)
|
||||
shape_slat [Nl, 32] shape encoder output (mean), the concat_cond
|
||||
shape_coords [Nl, 4] latent voxels at R/16
|
||||
tex_noise [Nl, 32] fixed sampling noise (seed)
|
||||
tex_flow_t500 [Nl, 32] tex-flow forward at t=500 (concat_cond, f32)
|
||||
tex_slat [Nl, 32] full sampler output, denormalized
|
||||
pbr [M, 6] decoded PBR voxels (base_color,metal,rough,alpha), *0.5+0.5
|
||||
pbr_coords [M, 4] decoded voxels at R (should equal enc_coords set)
|
||||
shape_slat_mean/std [32] shape_slat_normalization (concat_cond in-norm)
|
||||
tex_slat_mean/std [32] tex_slat_normalization (output de-norm)
|
||||
|
||||
This dump validates the standalone arbitrary-mesh texturing path, whose encoder
|
||||
is fed a reproducible QEF dual grid. Integrated image-to-3D generation instead
|
||||
retains the generated shape SLat and replays the shape decoder's subdivisions;
|
||||
that wiring is covered by test_slat plus the sparse PBR sampling regression.
|
||||
|
||||
Run inside the reference container (real o-voxel), e.g.:
|
||||
docker exec t2tex bash -lc \
|
||||
'cd /work && TRELLIS2_PY=/trellis2 python scripts/dump_texture_reference.py \
|
||||
--mesh /s/mesh_shipped.bin --image dumps/fixture_rgba.png --resolution 512'
|
||||
"""
|
||||
import argparse, json, os, struct, sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import o_voxel # noqa: E402 real o-voxel BEFORE ref_common (skips its stub)
|
||||
import o_voxel.convert # noqa: E402
|
||||
import ref_common # noqa: E402
|
||||
ref_common.setup() # sdpa attention + pure-torch sparse conv
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import torch # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
import trimesh # noqa: E402
|
||||
from safetensors.torch import load_file # noqa: E402
|
||||
|
||||
from trellis2.models.sc_vaes.fdg_vae import FlexiDualGridVaeEncoder # noqa: E402
|
||||
from trellis2.models.sc_vaes.sparse_unet_vae import SparseUnetVaeDecoder # noqa: E402
|
||||
from trellis2.models.structured_latent_flow import SLatFlowModel # noqa: E402
|
||||
from trellis2.pipelines.samplers import FlowEulerGuidanceIntervalSampler # noqa: E402
|
||||
from trellis2.pipelines import Trellis2TexturingPipeline # noqa: E402
|
||||
from trellis2.modules.image_feature_extractor import DinoV3FeatureExtractor # noqa: E402
|
||||
from trellis2.modules import sparse as sp # noqa: E402
|
||||
|
||||
|
||||
def load_t2mesh(path):
|
||||
b = open(path, "rb").read()
|
||||
assert b[:8] == b"T2MESH01", b[:8]
|
||||
nv, nt = struct.unpack("<II", b[8:16]); o = 16
|
||||
V = np.frombuffer(b, "<f4", 3 * nv, o).reshape(-1, 3).copy(); o += 12 * nv
|
||||
o += 12 * nv # skip normals
|
||||
F_ = np.frombuffer(b, "<i4", 3 * nt, o).reshape(-1, 3).copy()
|
||||
return V, F_
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--mesh", required=True)
|
||||
ap.add_argument("--image", required=True)
|
||||
ap.add_argument("--resolution", type=int, default=512, choices=[512, 1024])
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--t", type=float, default=500.0)
|
||||
ap.add_argument("--out", default=os.path.join(ref_common.DUMPS, "reference_texture.gguf"))
|
||||
args = ap.parse_args()
|
||||
dev = "cuda"
|
||||
R = args.resolution
|
||||
CK = os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "ckpts")
|
||||
tpa = json.load(open(os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "texturing_pipeline.json")))["args"]
|
||||
|
||||
def load_model(cls, stem, fp32_flags=False, **extra):
|
||||
cfg = json.load(open(os.path.join(CK, stem + ".json")))["args"]
|
||||
if fp32_flags:
|
||||
cfg.pop("use_fp16", None); cfg["use_fp16"] = False
|
||||
m = cls(**{**cfg, **extra})
|
||||
sd = {k: v.float() for k, v in load_file(os.path.join(CK, stem + ".safetensors")).items()}
|
||||
m.load_state_dict(sd)
|
||||
return m.eval().to(dev)
|
||||
|
||||
print(f"loading models (res {R}) ...", flush=True)
|
||||
shape_enc = load_model(FlexiDualGridVaeEncoder, "shape_enc_next_dc_f16c32_fp16", fp32_flags=True)
|
||||
tex_dec = load_model(SparseUnetVaeDecoder, "tex_dec_next_dc_f16c32_fp16", fp32_flags=True)
|
||||
tex_flow = load_model(SLatFlowModel, f"slat_flow_imgshape2tex_dit_1_3B_{R}_bf16", dtype="float32")
|
||||
|
||||
dino = DinoV3FeatureExtractor(os.path.join(ref_common.MODELS, "dinov3-vitl16"), image_size=R)
|
||||
dino.model = dino.model.to(dev).eval()
|
||||
|
||||
sampler = FlowEulerGuidanceIntervalSampler(**tpa["tex_slat_sampler"]["args"])
|
||||
pipe = Trellis2TexturingPipeline(
|
||||
models={"shape_slat_encoder": shape_enc, "tex_slat_decoder": tex_dec,
|
||||
f"tex_slat_flow_model_{R}": tex_flow},
|
||||
tex_slat_sampler=sampler,
|
||||
tex_slat_sampler_params=tpa["tex_slat_sampler"]["params"],
|
||||
shape_slat_normalization=tpa["shape_slat_normalization"],
|
||||
tex_slat_normalization=tpa["tex_slat_normalization"],
|
||||
image_cond_model=dino, rembg_model=None, low_vram=False,
|
||||
)
|
||||
pipe._device = dev
|
||||
|
||||
V, Ftri = load_t2mesh(args.mesh)
|
||||
mesh = trimesh.Trimesh(vertices=V, faces=Ftri, process=False)
|
||||
img = Image.open(args.image).convert("RGBA")
|
||||
print(f"mesh {len(V)} verts / {len(Ftri)} tris; image {img.size}", flush=True)
|
||||
|
||||
caps = {}
|
||||
torch.manual_seed(args.seed)
|
||||
with torch.no_grad():
|
||||
image = pipe.preprocess_image(img)
|
||||
mesh = pipe.preprocess_mesh(mesh)
|
||||
cond = pipe.get_cond([image], R)
|
||||
caps["cond"] = cond["cond"][0] # [T, Cc]
|
||||
|
||||
# ── shape encoder (QEF dual grid input) ──────────────────────────────
|
||||
vertices = torch.from_numpy(mesh.vertices).float()
|
||||
faces = torch.from_numpy(mesh.faces).long()
|
||||
voxel_indices, dual_vertices, intersected = o_voxel.convert.mesh_to_flexible_dual_grid(
|
||||
vertices.cpu(), faces.cpu(), grid_size=R,
|
||||
aabb=[[-0.5, -0.5, -0.5], [0.5, 0.5, 0.5]],
|
||||
face_weight=1.0, boundary_weight=0.2, regularization_weight=1e-2, timing=True)
|
||||
enc_vert_feats = dual_vertices * R - voxel_indices # offset in voxel
|
||||
vtx = sp.SparseTensor(
|
||||
feats=enc_vert_feats,
|
||||
coords=torch.cat([torch.zeros_like(voxel_indices[:, 0:1]), voxel_indices], dim=-1)).to(dev)
|
||||
inter = vtx.replace(intersected).to(dev)
|
||||
caps["enc_vert"] = vtx.feats
|
||||
caps["enc_inter"] = inter.feats.float()
|
||||
caps["enc_coords"] = vtx.coords.float()
|
||||
print(f"enc input: {vtx.feats.shape[0]} voxels at {R}^3", flush=True)
|
||||
|
||||
shape_slat = shape_enc(vtx, inter)
|
||||
caps["shape_slat"] = shape_slat.feats
|
||||
caps["shape_coords"] = shape_slat.coords.float()
|
||||
print(f"shape_slat: {shape_slat.feats.shape} coords {shape_slat.coords.shape}", flush=True)
|
||||
|
||||
# ── tex flow: forward parity point + full sampler ────────────────────
|
||||
s_std = torch.tensor(tpa["shape_slat_normalization"]["std"], device=dev)[None]
|
||||
s_mean = torch.tensor(tpa["shape_slat_normalization"]["mean"], device=dev)[None]
|
||||
shape_slat_n = shape_slat.replace((shape_slat.feats - s_mean) / s_std)
|
||||
caps["shape_slat_mean"] = s_mean[0]
|
||||
caps["shape_slat_std"] = s_std[0]
|
||||
|
||||
Nl = shape_slat.coords.shape[0]
|
||||
g = torch.Generator(device=dev).manual_seed(args.seed)
|
||||
noise_feats = torch.randn(Nl, tex_flow.out_channels, generator=g, device=dev)
|
||||
caps["tex_noise"] = noise_feats
|
||||
x0 = shape_slat.replace(noise_feats.clone())
|
||||
|
||||
out = tex_flow(x0, torch.tensor([args.t], device=dev), cond["cond"], concat_cond=shape_slat_n)
|
||||
caps["tex_flow_t500"] = out.feats
|
||||
print(f"tex_flow t={args.t}: mean={out.feats.mean().item():.6f} l2={out.feats.norm().item():.4f}", flush=True)
|
||||
|
||||
slat = sampler.sample(
|
||||
tex_flow, x0, concat_cond=shape_slat_n,
|
||||
cond=cond["cond"], neg_cond=cond["neg_cond"],
|
||||
**tpa["tex_slat_sampler"]["params"], verbose=True).samples
|
||||
t_std = torch.tensor(tpa["tex_slat_normalization"]["std"], device=dev)[None]
|
||||
t_mean = torch.tensor(tpa["tex_slat_normalization"]["mean"], device=dev)[None]
|
||||
slat = slat.replace(slat.feats * t_std + t_mean)
|
||||
caps["tex_slat"] = slat.feats
|
||||
caps["tex_slat_mean"] = t_mean[0]
|
||||
caps["tex_slat_std"] = t_std[0]
|
||||
print(f"tex_slat: mean={slat.feats.mean().item():.5f} std={slat.feats.std().item():.5f}", flush=True)
|
||||
|
||||
# ── tex decoder → PBR voxels ─────────────────────────────────────────
|
||||
pbr = tex_dec(slat) * 0.5 + 0.5
|
||||
caps["pbr"] = pbr.feats
|
||||
caps["pbr_coords"] = pbr.coords.float()
|
||||
print(f"pbr: {pbr.feats.shape} range [{pbr.feats.min():.3f},{pbr.feats.max():.3f}]", flush=True)
|
||||
|
||||
# sanity: the tex decoder should reconstruct exactly the encoder's voxel set
|
||||
enc_set = set(map(tuple, vtx.coords[:, 1:].cpu().numpy().tolist()))
|
||||
pbr_set = set(map(tuple, pbr.coords[:, 1:].cpu().numpy().tolist()))
|
||||
print(f"voxel-set match: pbr=={len(pbr_set)} enc=={len(enc_set)} "
|
||||
f"symdiff={len(enc_set ^ pbr_set)} ({100.0*len(enc_set & pbr_set)/max(1,len(enc_set)):.2f}% common)",
|
||||
flush=True)
|
||||
|
||||
import gguf
|
||||
writer = gguf.GGUFWriter(args.out, "reference")
|
||||
manifest = {"shapes": {}, "atol": 2e-3, "rtol": 2e-3, "resolution": R}
|
||||
for name, t in caps.items():
|
||||
a = t.detach().cpu().float().numpy()
|
||||
manifest["shapes"][name] = list(a.shape)
|
||||
writer.add_tensor(name, np.ascontiguousarray(a.reshape(-1), dtype=np.float32))
|
||||
writer.write_header_to_file(); writer.write_kv_data_to_file(); writer.write_tensors_to_file(); writer.close()
|
||||
with open(os.path.join(ref_common.DUMPS, "manifest_texture.json"), "w") as f:
|
||||
json.dump(manifest, f, indent=1)
|
||||
print(f"wrote {args.out} ({os.path.getsize(args.out):,} bytes), {len(caps)} tensors", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+146
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fetch the exact header-only dependency set used for CGAL print remeshing.
|
||||
# Keep the four pins below together: the scheduled dependency workflow updates
|
||||
# versions and upstream-published SHA-256 digests atomically.
|
||||
set -euo pipefail
|
||||
|
||||
CGAL_VERSION=6.1.1
|
||||
CGAL_SHA256=6c5d68be1d28cbee3c3e05003746ec4791d0018c770b4276b9e6d69c3a0a355a
|
||||
BOOST_VERSION=1.88.0
|
||||
BOOST_SHA256=3621533e820dcab1e8012afd583c0c73cf0f77694952b81352bf38c1488f9cb4
|
||||
FETCH_FORMAT=2
|
||||
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
DEST=${1:-"$ROOT/.deps/print-remesh"}
|
||||
MARKER="$DEST/.trellis2-print-remesh-deps"
|
||||
EXPECTED=$(printf 'CGAL_VERSION=%s\nCGAL_SHA256=%s\nBOOST_VERSION=%s\nBOOST_SHA256=%s\nFETCH_FORMAT=%s' \
|
||||
"$CGAL_VERSION" "$CGAL_SHA256" "$BOOST_VERSION" "$BOOST_SHA256" "$FETCH_FORMAT")
|
||||
|
||||
is_current() {
|
||||
[[ -f "$MARKER" && "$(<"$MARKER")" == "$EXPECTED" ]]
|
||||
}
|
||||
|
||||
if is_current; then
|
||||
echo "Print-remesh dependencies already current in $DEST"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$DEST")"
|
||||
LOCK="${DEST}.lock"
|
||||
attempt=0
|
||||
until mkdir "$LOCK" 2>/dev/null; do
|
||||
if is_current; then
|
||||
echo "Print-remesh dependencies already current in $DEST"
|
||||
exit 0
|
||||
fi
|
||||
((attempt += 1))
|
||||
if ((attempt >= 300)); then
|
||||
echo "Timed out waiting for dependency lock $LOCK" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
|
||||
TMP=
|
||||
cleanup() {
|
||||
rm -rf "$LOCK"
|
||||
if [[ -n "${TMP:-}" ]]; then
|
||||
rm -rf "$TMP"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
TMP=$(mktemp -d "${DEST}.tmp.XXXXXX")
|
||||
|
||||
verify_sha256() {
|
||||
local file=$1 expected=$2 actual
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual=$(sha256sum "$file")
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual=$(shasum -a 256 "$file")
|
||||
else
|
||||
echo "No SHA-256 tool found (need sha256sum or shasum)" >&2
|
||||
exit 127
|
||||
fi
|
||||
actual=${actual%% *}
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "SHA-256 mismatch for $file: expected $expected, got $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
CGAL_ARCHIVE="$TMP/cgal.zip"
|
||||
curl -fL --retry 3 \
|
||||
"https://github.com/CGAL/cgal/releases/download/v${CGAL_VERSION}/CGAL-${CGAL_VERSION}-library.zip" \
|
||||
-o "$CGAL_ARCHIVE"
|
||||
verify_sha256 "$CGAL_ARCHIVE" "$CGAL_SHA256"
|
||||
mkdir "$TMP/cgal-unpack"
|
||||
unzip -q "$CGAL_ARCHIVE" -d "$TMP/cgal-unpack"
|
||||
shopt -s nullglob
|
||||
cgal_roots=("$TMP/cgal-unpack"/*)
|
||||
shopt -u nullglob
|
||||
if [[ ${#cgal_roots[@]} -ne 1 || ! -d "${cgal_roots[0]}" ]]; then
|
||||
echo "Expected one root directory in the CGAL archive" >&2
|
||||
exit 1
|
||||
fi
|
||||
mv "${cgal_roots[0]}" "$TMP/cgal"
|
||||
|
||||
BOOST_UNDERSCORE=${BOOST_VERSION//./_}
|
||||
BOOST_ARCHIVE="$TMP/boost.tgz"
|
||||
curl -fL --retry 3 \
|
||||
"https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_UNDERSCORE}.tar.gz" \
|
||||
-o "$BOOST_ARCHIVE"
|
||||
verify_sha256 "$BOOST_ARCHIVE" "$BOOST_SHA256"
|
||||
mkdir "$TMP/boost"
|
||||
tar -xzf "$BOOST_ARCHIVE" -C "$TMP/boost" --strip-components=1 \
|
||||
"boost_${BOOST_UNDERSCORE}/boost"
|
||||
|
||||
# Raw Boost release archives intentionally have no installed CMake package.
|
||||
# Supply the header-only targets CGAL consumes so config-mode find_package
|
||||
# works on CMake 3.30+, where the legacy FindBoost module was removed.
|
||||
mkdir "$TMP/boost-cmake"
|
||||
cat > "$TMP/boost-cmake/BoostConfig.cmake" <<EOF
|
||||
set(Boost_FOUND TRUE)
|
||||
set(Boost_VERSION "$BOOST_VERSION")
|
||||
set(Boost_VERSION_STRING "$BOOST_VERSION")
|
||||
set(Boost_INCLUDE_DIR "\${CMAKE_CURRENT_LIST_DIR}/../boost")
|
||||
set(Boost_INCLUDE_DIRS "\${Boost_INCLUDE_DIR}")
|
||||
set(Boost_LIBRARIES "")
|
||||
if(NOT TARGET Boost::headers)
|
||||
add_library(Boost::headers INTERFACE IMPORTED GLOBAL)
|
||||
set_target_properties(Boost::headers PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "\${Boost_INCLUDE_DIR}")
|
||||
endif()
|
||||
if(NOT TARGET Boost::boost)
|
||||
add_library(Boost::boost ALIAS Boost::headers)
|
||||
endif()
|
||||
EOF
|
||||
cat > "$TMP/boost-cmake/BoostConfigVersion.cmake" <<EOF
|
||||
set(PACKAGE_VERSION "$BOOST_VERSION")
|
||||
if(PACKAGE_FIND_VERSION VERSION_GREATER PACKAGE_VERSION)
|
||||
set(PACKAGE_VERSION_COMPATIBLE FALSE)
|
||||
else()
|
||||
set(PACKAGE_VERSION_COMPATIBLE TRUE)
|
||||
if(PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION)
|
||||
set(PACKAGE_VERSION_EXACT TRUE)
|
||||
endif()
|
||||
endif()
|
||||
EOF
|
||||
|
||||
rm -rf "$CGAL_ARCHIVE" "$BOOST_ARCHIVE" "$TMP/cgal-unpack"
|
||||
printf '%s\n' "$EXPECTED" > "$TMP/.trellis2-print-remesh-deps"
|
||||
|
||||
OLD="${DEST}.old.$$"
|
||||
if [[ -e "$DEST" ]]; then
|
||||
mv "$DEST" "$OLD"
|
||||
fi
|
||||
if mv "$TMP" "$DEST"; then
|
||||
TMP=
|
||||
rm -rf "$OLD"
|
||||
else
|
||||
if [[ -e "$OLD" ]]; then
|
||||
mv "$OLD" "$DEST"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Fetched CGAL $CGAL_VERSION and Boost $BOOST_VERSION into $DEST"
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dump a GLB's baseColor + metallicRoughness atlas images side by side, so the
|
||||
bake's colour fidelity can be eyeballed without any renderer in the way.
|
||||
|
||||
python3 scripts/glb_atlas.py in.glb [out.png]
|
||||
"""
|
||||
import sys, numpy as np, trimesh
|
||||
from PIL import Image
|
||||
|
||||
path = sys.argv[1]; out = sys.argv[2] if len(sys.argv) > 2 else "glb_atlas.png"
|
||||
scene = trimesh.load(path, process=False)
|
||||
mesh = scene if isinstance(scene, trimesh.Trimesh) else list(scene.geometry.values())[0]
|
||||
attrs = getattr(mesh.visual, "vertex_attributes", {})
|
||||
if "color" in attrs:
|
||||
color = np.asarray(attrs["color"])
|
||||
print(f"vertex-colour GLB: {len(color):,} COLOR_0 values ({color.dtype}); no UV atlas")
|
||||
sys.exit(0)
|
||||
mat = mesh.visual.material
|
||||
bc = mat.baseColorTexture.convert("RGB")
|
||||
try:
|
||||
mr = mat.metallicRoughnessTexture.convert("RGB")
|
||||
except Exception:
|
||||
mr = Image.new("RGB", bc.size)
|
||||
W, H = bc.size
|
||||
# downscale for a quick look if huge
|
||||
scale = 1024 / max(W, H)
|
||||
if scale < 1:
|
||||
bc = bc.resize((int(W*scale), int(H*scale)))
|
||||
mr = mr.resize((int(W*scale), int(H*scale)))
|
||||
w, h = bc.size
|
||||
canvas = Image.new("RGB", (w*2 + 16, h), (30, 30, 30))
|
||||
canvas.paste(bc, (0, 0)); canvas.paste(mr, (w + 16, 0))
|
||||
canvas.save(out)
|
||||
a = np.asarray(bc, np.float64) / 255
|
||||
nz = a.reshape(-1, 3); nz = nz[nz.sum(1) > 0.05]
|
||||
print(f"atlas {W}x{H} baseColor mean(nonblack)={nz.mean(0).round(3) if len(nz) else 'n/a'} "
|
||||
f"filled={len(nz)/(w*h)*100:.0f}%", flush=True)
|
||||
print("wrote", out, flush=True)
|
||||
@@ -0,0 +1,324 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Headless Chromium smoke checks for the trellis2.cpp demo.
|
||||
|
||||
Uses only the Python standard library and Chromium's DevTools protocol so it can
|
||||
run on the headless demo host without Playwright/Selenium. It intentionally
|
||||
loads a persisted mesh and the showcase, exercising fetch, binary parsing,
|
||||
WebGL2 buffer upload, and rendering rather than merely checking the HTML.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class CDP:
|
||||
def __init__(self, url):
|
||||
parsed = urlparse(url)
|
||||
self.sock = socket.create_connection((parsed.hostname, parsed.port), timeout=15)
|
||||
key = base64.b64encode(secrets.token_bytes(16)).decode()
|
||||
request = (
|
||||
f"GET {parsed.path} HTTP/1.1\r\n"
|
||||
f"Host: {parsed.hostname}:{parsed.port}\r\n"
|
||||
"Upgrade: websocket\r\nConnection: Upgrade\r\n"
|
||||
f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n"
|
||||
)
|
||||
self.sock.sendall(request.encode())
|
||||
response = b""
|
||||
while b"\r\n\r\n" not in response:
|
||||
response += self.sock.recv(4096)
|
||||
if b" 101 " not in response.split(b"\r\n", 1)[0]:
|
||||
raise RuntimeError(f"WebSocket upgrade failed: {response[:200]!r}")
|
||||
self.next_id = 1
|
||||
self.events = []
|
||||
|
||||
def _send_frame(self, payload):
|
||||
data = payload.encode()
|
||||
mask = secrets.token_bytes(4)
|
||||
n = len(data)
|
||||
header = bytearray([0x81])
|
||||
if n < 126:
|
||||
header.append(0x80 | n)
|
||||
elif n < 65536:
|
||||
header.append(0x80 | 126)
|
||||
header.extend(struct.pack("!H", n))
|
||||
else:
|
||||
header.append(0x80 | 127)
|
||||
header.extend(struct.pack("!Q", n))
|
||||
header.extend(mask)
|
||||
header.extend(bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
|
||||
self.sock.sendall(header)
|
||||
|
||||
def _read_exact(self, n):
|
||||
chunks = []
|
||||
while n:
|
||||
chunk = self.sock.recv(n)
|
||||
if not chunk:
|
||||
raise RuntimeError("DevTools WebSocket closed")
|
||||
chunks.append(chunk)
|
||||
n -= len(chunk)
|
||||
return b"".join(chunks)
|
||||
|
||||
def _recv_frame(self):
|
||||
b0, b1 = self._read_exact(2)
|
||||
opcode, n = b0 & 0x0F, b1 & 0x7F
|
||||
if n == 126:
|
||||
n = struct.unpack("!H", self._read_exact(2))[0]
|
||||
elif n == 127:
|
||||
n = struct.unpack("!Q", self._read_exact(8))[0]
|
||||
masked = bool(b1 & 0x80)
|
||||
mask = self._read_exact(4) if masked else None
|
||||
data = self._read_exact(n)
|
||||
if mask:
|
||||
data = bytes(b ^ mask[i % 4] for i, b in enumerate(data))
|
||||
if opcode == 0x9: # ping
|
||||
self._send_frame(data.decode(errors="ignore"))
|
||||
return self._recv_frame()
|
||||
if opcode == 0x8:
|
||||
raise RuntimeError("DevTools WebSocket closed")
|
||||
return json.loads(data.decode())
|
||||
|
||||
def call(self, method, params=None, timeout=90):
|
||||
ident = self.next_id
|
||||
self.next_id += 1
|
||||
self._send_frame(json.dumps({"id": ident, "method": method, "params": params or {}}))
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
self.sock.settimeout(max(0.1, deadline - time.monotonic()))
|
||||
message = self._recv_frame()
|
||||
if message.get("id") == ident:
|
||||
if "error" in message:
|
||||
raise RuntimeError(f"{method}: {message['error']}")
|
||||
return message.get("result", {})
|
||||
self.events.append(message)
|
||||
raise TimeoutError(method)
|
||||
|
||||
def evaluate(self, expression, timeout=90):
|
||||
result = self.call("Runtime.evaluate", {
|
||||
"expression": expression,
|
||||
"awaitPromise": True,
|
||||
"returnByValue": True,
|
||||
"userGesture": True,
|
||||
}, timeout)
|
||||
value = result.get("result", {})
|
||||
if "exceptionDetails" in result:
|
||||
raise RuntimeError(result["exceptionDetails"])
|
||||
return value.get("value")
|
||||
|
||||
|
||||
def wait_json(url, timeout=15):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as response:
|
||||
return json.load(response)
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError(url)
|
||||
|
||||
|
||||
def screenshot(cdp, path):
|
||||
encoded = cdp.call("Page.captureScreenshot", {
|
||||
"format": "png", "captureBeyondViewport": False,
|
||||
}, timeout=30)["data"]
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
|
||||
|
||||
def navigate(cdp, url, timeout=30):
|
||||
cdp.call("Page.navigate", {"url": url}, timeout=timeout)
|
||||
expected = urlparse(url).path or "/"
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
state = cdp.evaluate("({path: location.pathname, ready: document.readyState})", timeout=2)
|
||||
if state and state["path"] == expected and state["ready"] == "complete":
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError(f"navigation to {url}")
|
||||
|
||||
|
||||
def browser_errors(events):
|
||||
errors = []
|
||||
for event in events:
|
||||
method, params = event.get("method"), event.get("params", {})
|
||||
if method == "Runtime.exceptionThrown":
|
||||
detail = params.get("exceptionDetails", {})
|
||||
errors.append(detail.get("text", "JavaScript exception"))
|
||||
elif method == "Log.entryAdded":
|
||||
entry = params.get("entry", {})
|
||||
if entry.get("level") in ("error", "warning"):
|
||||
location = f" ({entry['url']})" if entry.get("url") else ""
|
||||
errors.append(f"{entry.get('level')}: {entry.get('text', '')}{location}")
|
||||
elif method == "Runtime.consoleAPICalled" and params.get("type") == "error":
|
||||
args = params.get("args", [])
|
||||
errors.append("console: " + " ".join(str(a.get("value", a.get("description", ""))) for a in args))
|
||||
return errors
|
||||
|
||||
|
||||
MAIN_CHECK = r"""
|
||||
(async () => {
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
for (let i = 0; i < 100 && (!history || !history.length); i++) await sleep(100);
|
||||
if (!history.length) throw new Error('no persisted generations in history');
|
||||
// Prefer the oldest retained asset: it is stable across new generations and
|
||||
// avoids coupling this reusable smoke test to a particular server/job ID.
|
||||
const preferred = history[history.length - 1];
|
||||
await viewGeneration(preferred.id);
|
||||
await sleep(1200);
|
||||
await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
|
||||
const mesh = originalExportMesh;
|
||||
const pbr = mesh && mesh.pbr;
|
||||
const range = pbr ? {min: Array(6).fill(Infinity), max: Array(6).fill(-Infinity)} : null;
|
||||
if (pbr) {
|
||||
const stride = Math.max(6, Math.floor((pbr.length / 6) / 50000) * 6);
|
||||
for (let i = 0; i < pbr.length; i += stride) {
|
||||
for (let c = 0; c < 6; c++) {
|
||||
range.min[c] = Math.min(range.min[c], pbr[i + c]);
|
||||
range.max[c] = Math.max(range.max[c], pbr[i + c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
const canvas = document.getElementById('gl');
|
||||
const gl = canvas.getContext('webgl2');
|
||||
const pixel = new Uint8Array(4);
|
||||
gl.readPixels(canvas.width >> 1, canvas.height >> 1, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixel);
|
||||
return {
|
||||
path: location.pathname, title: document.title,
|
||||
historyCount: history.length, selected: preferred.id,
|
||||
status: document.getElementById('status').textContent,
|
||||
meshInfo: document.getElementById('meshinfo').textContent,
|
||||
mesh: mesh ? {vertices: mesh.nv, triangles: mesh.nt, textured: mesh.textured} : null,
|
||||
pbrRange: range, webgl2: !!gl, glError: gl.getError(), centerPixel: [...pixel],
|
||||
canvas: {width: canvas.width, height: canvas.height,
|
||||
cssWidth: canvas.clientWidth, cssHeight: canvas.clientHeight},
|
||||
liveSteps: {text: document.getElementById('tsteps').textContent,
|
||||
pressed: document.getElementById('tsteps').getAttribute('aria-pressed'),
|
||||
keyframesDisabled: document.getElementById('tkf').disabled},
|
||||
exportEnabled: !document.getElementById('dglb').disabled,
|
||||
regenerateEnabled: !document.getElementById('regen').disabled,
|
||||
};
|
||||
})()
|
||||
"""
|
||||
|
||||
|
||||
SHOWCASE_CHECK = r"""
|
||||
(async () => {
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
for (let i = 0; i < 600; i++) {
|
||||
if (typeof showcaseAssets !== 'undefined' && history.length && showcaseAssets.length === history.length) break;
|
||||
await sleep(100);
|
||||
}
|
||||
await sleep(1800);
|
||||
await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
|
||||
const canvas = document.getElementById('gl');
|
||||
const gl = canvas.getContext('webgl2');
|
||||
const display = selector => getComputedStyle(document.querySelector(selector)).display;
|
||||
const pixel = new Uint8Array(4);
|
||||
gl.readPixels(canvas.width >> 1, canvas.height >> 1, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixel);
|
||||
return {
|
||||
path: location.pathname, title: document.title,
|
||||
historyCount: history.length,
|
||||
loadedAssets: typeof showcaseAssets === 'undefined' ? -1 : showcaseAssets.length,
|
||||
running: typeof showcaseRunning === 'undefined' ? false : showcaseRunning,
|
||||
label: document.getElementById('showcaselabel').textContent,
|
||||
sourceClass: document.getElementById('showcasesource').className,
|
||||
chrome: {header: display('header'), side: display('#side'), timeline: display('#timeline')},
|
||||
webgl2: !!gl, glError: gl.getError(), centerPixel: [...pixel],
|
||||
canvas: {width: canvas.width, height: canvas.height,
|
||||
cssWidth: canvas.clientWidth, cssHeight: canvas.clientHeight},
|
||||
};
|
||||
})()
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="http://127.0.0.1:8742")
|
||||
parser.add_argument("--chromium", default="chromium")
|
||||
parser.add_argument("--output", default="/tmp/trellis2-headless")
|
||||
args = parser.parse_args()
|
||||
output = Path(args.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="trellis2-chrome-") as profile:
|
||||
command = [
|
||||
args.chromium, "--headless=new", "--no-sandbox", "--disable-dev-shm-usage",
|
||||
"--enable-webgl", "--use-angle=swiftshader", "--enable-unsafe-swiftshader",
|
||||
"--remote-debugging-port=0", "--remote-allow-origins=*",
|
||||
f"--user-data-dir={profile}", "--window-size=1280,900", "about:blank",
|
||||
]
|
||||
browser = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
||||
try:
|
||||
port_file = Path(profile) / "DevToolsActivePort"
|
||||
deadline = time.monotonic() + 15
|
||||
while not port_file.exists() and time.monotonic() < deadline:
|
||||
if browser.poll() is not None:
|
||||
raise RuntimeError(browser.stderr.read().decode(errors="replace"))
|
||||
time.sleep(0.1)
|
||||
port = int(port_file.read_text().splitlines()[0])
|
||||
targets = wait_json(f"http://127.0.0.1:{port}/json/list")
|
||||
page = next(t for t in targets if t["type"] == "page")
|
||||
cdp = CDP(page["webSocketDebuggerUrl"])
|
||||
for method in ("Page.enable", "Runtime.enable", "Log.enable"):
|
||||
cdp.call(method)
|
||||
|
||||
results = {}
|
||||
navigate(cdp, args.url.rstrip("/") + "/")
|
||||
results["main"] = cdp.evaluate(MAIN_CHECK, timeout=120)
|
||||
screenshot(cdp, output / "main.png")
|
||||
|
||||
navigate(cdp, args.url.rstrip("/") + "/showcase")
|
||||
results["showcase"] = cdp.evaluate(SHOWCASE_CHECK, timeout=180)
|
||||
screenshot(cdp, output / "showcase.png")
|
||||
results["browserErrors"] = browser_errors(cdp.events)
|
||||
results["screenshots"] = [str(output / "main.png"), str(output / "showcase.png")]
|
||||
failures = []
|
||||
main_result, showcase_result = results["main"], results["showcase"]
|
||||
if not main_result["webgl2"] or main_result["glError"] != 0:
|
||||
failures.append("main page WebGL2 failure")
|
||||
if not main_result["mesh"]["textured"] or not main_result["pbrRange"]:
|
||||
failures.append("main page did not load PBR data")
|
||||
elif max(main_result["pbrRange"]["max"][c] - main_result["pbrRange"]["min"][c]
|
||||
for c in range(3)) < 0.1:
|
||||
failures.append("main page base colour is effectively constant")
|
||||
if main_result["status"] != "done":
|
||||
failures.append(f"main page status is {main_result['status']!r}")
|
||||
if main_result["liveSteps"] != {
|
||||
"text": "live steps: off", "pressed": "false", "keyframesDisabled": True}:
|
||||
failures.append(f"live steps did not default clearly off: {main_result['liveSteps']!r}")
|
||||
if showcase_result["loadedAssets"] != showcase_result["historyCount"]:
|
||||
failures.append("showcase did not load every persisted asset")
|
||||
if not showcase_result["webgl2"] or showcase_result["glError"] != 0:
|
||||
failures.append("showcase WebGL2 failure")
|
||||
if any(value != "none" for value in showcase_result["chrome"].values()):
|
||||
failures.append("showcase page chrome is visible")
|
||||
significant_errors = [e for e in results["browserErrors"] if "favicon.ico" not in e]
|
||||
if significant_errors:
|
||||
failures.extend(significant_errors)
|
||||
results["failures"] = failures
|
||||
results["passed"] = not failures
|
||||
print(json.dumps(results, indent=2))
|
||||
if failures:
|
||||
raise SystemExit(1)
|
||||
finally:
|
||||
browser.terminate()
|
||||
try:
|
||||
browser.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
browser.kill()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,41 @@
|
||||
---
|
||||
license: mit
|
||||
base_model: microsoft/TRELLIS-image-large
|
||||
tags:
|
||||
- gguf
|
||||
- trellis
|
||||
- image-to-3d
|
||||
- ggml
|
||||
pipeline_tag: image-to-3d
|
||||
---
|
||||
|
||||
# TRELLIS-image-large — GGUF (f16)
|
||||
|
||||
GGUF (`f16`) conversion of the sparse-structure decoder from
|
||||
[microsoft/TRELLIS-image-large](https://huggingface.co/microsoft/TRELLIS-image-large) for
|
||||
the **[trellis2cpp](https://github.com/localai-org/trellis2cpp)** /
|
||||
[ggml](https://github.com/ggml-org/ggml) runtime.
|
||||
|
||||
The TRELLIS.2 geometry pipeline reuses this TRELLIS-1 sparse-structure decoder to turn the
|
||||
sparse-structure flow output into a 64³ voxel scaffold.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Pipeline stage | Source safetensors |
|
||||
|---|---|---|
|
||||
| `ss_dec_f16.gguf` | Sparse-structure decoder → 64³ voxel scaffold | `ss_dec_conv3d_16l8_fp16` |
|
||||
|
||||
The rest of the TRELLIS.2 pipeline is published at
|
||||
[TRELLIS.2-4B-GGUF](https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF) and
|
||||
[dinov3-vitl16-pretrain-lvd1689m-GGUF](https://huggingface.co/LocalAI-io/dinov3-vitl16-pretrain-lvd1689m-GGUF).
|
||||
|
||||
## Conversion
|
||||
|
||||
Converted from the upstream `fp16` safetensors to GGUF `f16` (ggml ftype 1) with
|
||||
`convert_ss_dec_to_gguf.py` in the trellis2cpp project. Format conversion only.
|
||||
|
||||
## License
|
||||
|
||||
MIT, inherited from
|
||||
[microsoft/TRELLIS-image-large](https://huggingface.co/microsoft/TRELLIS-image-large).
|
||||
See [`LICENSE`](LICENSE). Copyright (c) Microsoft Corporation.
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) Microsoft Corporation.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
license: mit
|
||||
base_model: microsoft/TRELLIS.2-4B
|
||||
tags:
|
||||
- gguf
|
||||
- trellis
|
||||
- image-to-3d
|
||||
- ggml
|
||||
pipeline_tag: image-to-3d
|
||||
---
|
||||
|
||||
# TRELLIS.2-4B — GGUF (f16)
|
||||
|
||||
GGUF (`f16`) conversions of [microsoft/TRELLIS.2-4B](https://huggingface.co/microsoft/TRELLIS.2-4B)
|
||||
for the **[trellis2cpp](https://github.com/localai-org/trellis2cpp)** /
|
||||
[ggml](https://github.com/ggml-org/ggml) runtime — a CUDA-free, PyTorch-free C++ port of
|
||||
the TRELLIS.2 image-to-3D pipeline. Load these directly; no safetensors conversion or
|
||||
Python is needed at inference.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Pipeline stage | Source safetensors |
|
||||
|---|---|---|
|
||||
| `ss_flow_f16.gguf` | Sparse-structure flow (64³ occupancy) | `ss_flow_img_dit_1_3B_64_bf16` |
|
||||
| `slat_flow_f16.gguf` | Shape-SLAT flow, 512 fine | `slat_flow_img2shape_dit_1_3B_512_bf16` |
|
||||
| `slat_flow_1024_f16.gguf` | Shape-SLAT flow, 1024 cascade | `slat_flow_img2shape_dit_1_3B_1024_bf16` |
|
||||
| `shape_dec_f16.gguf` | Shape decoder → dual-grid fields | `shape_dec_next_dc_f16c32_fp16` |
|
||||
| `shape_enc_f16.gguf` | Shape encoder (re-encode for texture flow) | `shape_enc_next_dc_f16c32_fp16` |
|
||||
| `tex_dec_f16.gguf` | PBR texture decoder | `tex_dec_next_dc_f16c32_fp16` |
|
||||
| `tex_slat_flow_512_f16.gguf` | Texture-SLAT flow, 512 | `slat_flow_imgshape2tex_dit_1_3B_512_bf16` |
|
||||
| `tex_slat_flow_1024_f16.gguf` | Texture-SLAT flow, 1024 | `slat_flow_imgshape2tex_dit_1_3B_1024_bf16` |
|
||||
|
||||
The image conditioning encoder (DINOv3) and the sparse-structure decoder are published
|
||||
separately:
|
||||
[dinov3-vitl16-pretrain-lvd1689m-GGUF](https://huggingface.co/LocalAI-io/dinov3-vitl16-pretrain-lvd1689m-GGUF)
|
||||
and
|
||||
[TRELLIS-image-large-GGUF](https://huggingface.co/LocalAI-io/TRELLIS-image-large-GGUF).
|
||||
|
||||
## Conversion
|
||||
|
||||
Converted from the upstream `bf16`/`fp16` safetensors to GGUF `f16` (ggml ftype 1) with
|
||||
the `convert_*_to_gguf.py` scripts in the trellis2cpp project. No weights were retrained
|
||||
or modified — this is a format conversion only.
|
||||
|
||||
## License
|
||||
|
||||
MIT, inherited from [microsoft/TRELLIS.2-4B](https://huggingface.co/microsoft/TRELLIS.2-4B).
|
||||
See [`LICENSE`](LICENSE). Copyright (c) Microsoft Corporation.
|
||||
@@ -0,0 +1,66 @@
|
||||
# DINOv3 License
|
||||
|
||||
*Last Updated: August 19, 2025*
|
||||
|
||||
**“Agreement”** means the terms and conditions for use, reproduction, distribution and modification of the DINO Materials set forth herein.
|
||||
|
||||
**“DINO Materials”** means, collectively, Documentation and the models, software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code, and other elements of the foregoing distributed by Meta and made available under this Agreement.
|
||||
|
||||
**“Documentation”** means the specifications, manuals and documentation accompanying
|
||||
DINO Materials distributed by Meta.
|
||||
|
||||
**“Licensee”** or **“you”** means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf.
|
||||
|
||||
**“Meta”** or **“we”** means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your principal place of business is in the EEA or Switzerland) or Meta Platforms, Inc. (if you are located outside of the EEA or Switzerland).
|
||||
|
||||
**“Sanctions”** means any economic or trade sanctions or restrictions administered or enforced by the United States (including the Office of Foreign Assets Control of the U.S. Department of the Treasury (“OFAC”), the U.S. Department of State and the U.S. Department of Commerce), the United Nations, the European Union, or the United Kingdom.
|
||||
|
||||
**“Trade Controls”** means any of the following: Sanctions and applicable export and import controls.
|
||||
|
||||
By clicking “I Accept” below or by using or distributing any portion or element of the DINO Materials, you agree to be bound by this Agreement.
|
||||
|
||||
## 1. License Rights and Redistribution.
|
||||
|
||||
a. <ins>Grant of Rights</ins>. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under Meta’s intellectual property or other rights owned by Meta embodied in the DINO Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the DINO Materials.
|
||||
|
||||
b. <ins>Redistribution and Use</ins>.
|
||||
|
||||
i. Distribution of DINO Materials, and any derivative works thereof, are subject to the terms of this Agreement. If you distribute or make the DINO Materials, or any derivative works thereof, available to a third party, you may only do so under the terms of this Agreement and you shall provide a copy of this Agreement with any such DINO Materials.
|
||||
|
||||
ii. If you submit for publication the results of research you perform on, using, or otherwise in connection with DINO Materials, you must acknowledge the use of DINO Materials in your publication.
|
||||
|
||||
iii. Your use of the DINO Materials must comply with applicable laws and regulations, including Trade Control Laws and applicable privacy and data protection laws.
|
||||
|
||||
iv. Your use of the DINO Materials will not involve or encourage others to reverse engineer, decompile or discover the underlying components of the DINO Materials.
|
||||
|
||||
v. You are not the target of Trade Controls and your use of DINO Materials must comply with Trade Controls. You agree not to use, or permit others to use, DINO Materials for any activities subject to the International Traffic in Arms Regulations (ITAR) or end uses prohibited by Trade Controls, including those related to military or warfare purposes, nuclear industries or applications, espionage, or the development or use of guns or illegal weapons.
|
||||
|
||||
## 2. User Support.
|
||||
|
||||
Your use of the DINO Materials is done at your own discretion; Meta does not process any information nor provide any service in relation to such use. Meta is under no obligation to provide any support services for the DINO Materials. Any support provided is “as is”, “with all faults”, and without warranty of any kind.
|
||||
|
||||
## 3. Disclaimer of Warranty.
|
||||
|
||||
UNLESS REQUIRED BY APPLICABLE LAW, THE DINO MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE DINO MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE DINO MATERIALS AND ANY OUTPUT AND RESULTS.
|
||||
|
||||
## 4. Limitation of Liability.
|
||||
|
||||
IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT OR INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING.
|
||||
|
||||
## 5. Intellectual Property.
|
||||
|
||||
a. Subject to Meta’s ownership of DINO Materials and derivatives made by or for Meta, with respect to any derivative works and modifications of the DINO Materials that are made by you, as between you and Meta, you are and will be the owner of such derivative works and modifications.
|
||||
|
||||
b. If you institute litigation or other proceedings against Meta or any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the DINO Materials, outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Meta from and against any claim by any third party arising out of or related to your use or distribution of the DINO Materials.
|
||||
|
||||
## 6. Term and Termination.
|
||||
|
||||
The term of this Agreement will commence upon your acceptance of this Agreement or access to the DINO Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Meta may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the DINO Materials. Sections 3, 4 and 7 shall survive the termination of this Agreement.
|
||||
|
||||
## 7. Governing Law and Jurisdiction.
|
||||
|
||||
This Agreement will be governed and construed under the laws of the State of California without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. The courts of California shall have exclusive jurisdiction of any dispute arising out of this Agreement.
|
||||
|
||||
## 8. Modifications and Amendments.
|
||||
|
||||
Meta may modify this Agreement from time to time; provided that they are similar in spirit to the current version of the Agreement, but may differ in detail to address new problems or concerns. All such changes will be effective immediately. Your continued use of the DINO Materials after any modification to this Agreement constitutes your agreement to such modification. Except as provided in this Agreement, no modification or addition to any provision of this Agreement will be binding unless it is in writing and signed by an authorized representative of both you and Meta.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
license: other
|
||||
license_name: dinov3-license
|
||||
license_link: LICENSE
|
||||
base_model: facebook/dinov3-vitl16-pretrain-lvd1689m
|
||||
tags:
|
||||
- gguf
|
||||
- dinov3
|
||||
- vision-encoder
|
||||
- ggml
|
||||
---
|
||||
|
||||
# DINOv3 ViT-L/16 (lvd1689m) — GGUF (f16)
|
||||
|
||||
**Built with DINOv3.**
|
||||
|
||||
GGUF (`f16`) conversion of the DINOv3 ViT-L/16 (`lvd1689m`) vision encoder for the
|
||||
**[trellis2cpp](https://github.com/localai-org/trellis2cpp)** /
|
||||
[ggml](https://github.com/ggml-org/ggml) runtime, where it produces the image conditioning
|
||||
features for the TRELLIS.2 image-to-3D pipeline.
|
||||
|
||||
The weights derive from Meta's
|
||||
[facebook/dinov3-vitl16-pretrain-lvd1689m](https://huggingface.co/facebook/dinov3-vitl16-pretrain-lvd1689m)
|
||||
(obtained via the ungated
|
||||
[camenduru](https://huggingface.co/camenduru/dinov3-vitl16-pretrain-lvd1689m) mirror) and
|
||||
are redistributed under Meta's **DINOv3 License**, a full copy of which is included as
|
||||
[`LICENSE`](LICENSE). Your use of these weights is subject to that Agreement.
|
||||
|
||||
The source `model.safetensors` is **byte-identical to the official Meta release** —
|
||||
verified SHA256 `dcb2e45127cccbf1601e5f42fef165eea275c8e5213197e8dcf3f48822718179`
|
||||
(1,212,559,808 bytes), matching `facebook/dinov3-vitl16-pretrain-lvd1689m` exactly.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Pipeline stage | Source |
|
||||
|---|---|---|
|
||||
| `dino_f16.gguf` | Image conditioning encoder (DINOv3 ViT-L/16) | `model.safetensors` |
|
||||
|
||||
## Conversion
|
||||
|
||||
Converted from the upstream safetensors to GGUF `f16` (ggml ftype 1) with
|
||||
`convert_dino_to_gguf.py` in the trellis2cpp project. No weights were retrained or
|
||||
modified — this is a format conversion only.
|
||||
|
||||
## License & attribution
|
||||
|
||||
Meta **DINOv3 License** — see [`LICENSE`](LICENSE). This model is **Built with DINOv3**.
|
||||
DINOv3 is © Meta Platforms, Inc. Redistribution of these weights (or derivatives) must
|
||||
provide a copy of the Agreement and prominently display "Built with DINOv3"; downstream
|
||||
use is bound by the Agreement's terms (including its acceptable-use and trade-control
|
||||
provisions).
|
||||
|
||||
## Companion models
|
||||
|
||||
- [TRELLIS.2-4B-GGUF](https://huggingface.co/LocalAI-io/TRELLIS.2-4B-GGUF)
|
||||
- [TRELLIS-image-large-GGUF](https://huggingface.co/LocalAI-io/TRELLIS-image-large-GGUF)
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Shared plumbing for the PyTorch reference dumps.
|
||||
|
||||
The reference environment is deliberately stock PyTorch (docker/Dockerfile.ref):
|
||||
none of the custom CUDA extensions (flash-attn, FlexGEMM, cumesh, o-voxel) are
|
||||
installed. This module makes the trellis2 package importable and usable in that
|
||||
environment:
|
||||
|
||||
* stubs out `cumesh` (only needed by Mesh postprocess methods we don't call
|
||||
while dumping activations),
|
||||
* replaces the sparse attention dispatcher with a plain-SDPA implementation
|
||||
(mathematically identical for the var-len batch-1 case we validate),
|
||||
* registers a pure-PyTorch submanifold sparse-conv backend under the name
|
||||
'none' (gather + GEMM per kernel offset — slow, but it runs anywhere and
|
||||
doubles as the executable spec for the C++ implementation).
|
||||
|
||||
Import this before importing anything from `trellis2`.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
|
||||
TRELLIS2_PY = os.environ.get("TRELLIS2_PY", "/trellis2")
|
||||
if TRELLIS2_PY not in sys.path:
|
||||
sys.path.insert(0, TRELLIS2_PY)
|
||||
|
||||
os.environ.setdefault("ATTN_BACKEND", "sdpa")
|
||||
os.environ.setdefault("SPARSE_CONV_BACKEND", "none")
|
||||
# The sparse attention config only accepts xformers/flash_attn/flash_attn_3;
|
||||
# we leave it alone and monkeypatch the dispatcher below instead.
|
||||
|
||||
# --- cumesh stub (postprocess-only dependency of representations.mesh.base) ---
|
||||
if "cumesh" not in sys.modules:
|
||||
stub = types.ModuleType("cumesh")
|
||||
|
||||
class _CuMeshStub:
|
||||
def __init__(self, *_a, **_k):
|
||||
raise RuntimeError("cumesh is stubbed out in the reference container")
|
||||
|
||||
stub.CuMesh = _CuMeshStub
|
||||
sys.modules["cumesh"] = stub
|
||||
|
||||
# --- o_voxel stub (CUDA hashmap mesher; we dump the decoder's raw 7-channel
|
||||
# --- output and do mesh comparison with scripts/ref_dual_grid.py instead) ---
|
||||
if "o_voxel" not in sys.modules:
|
||||
ovx = types.ModuleType("o_voxel")
|
||||
ovx_convert = types.ModuleType("o_voxel.convert")
|
||||
|
||||
def flexible_dual_grid_to_mesh(*_a, **_k):
|
||||
raise RuntimeError("o_voxel is stubbed out in the reference container")
|
||||
|
||||
ovx_convert.flexible_dual_grid_to_mesh = flexible_dual_grid_to_mesh
|
||||
ovx.convert = ovx_convert
|
||||
sys.modules["o_voxel"] = ovx
|
||||
sys.modules["o_voxel.convert"] = ovx_convert
|
||||
|
||||
# --- flex_gemm stub (CUDA kernels; representations.mesh.base imports
|
||||
# --- grid_sample_3d for texture baking, unused on the geometry path) ---
|
||||
if "flex_gemm" not in sys.modules:
|
||||
fg = types.ModuleType("flex_gemm")
|
||||
fg_ops = types.ModuleType("flex_gemm.ops")
|
||||
fg_gs = types.ModuleType("flex_gemm.ops.grid_sample")
|
||||
fg_sp = types.ModuleType("flex_gemm.ops.spconv")
|
||||
|
||||
def _fg_unavailable(*_a, **_k):
|
||||
raise RuntimeError("flex_gemm is stubbed out in the reference container")
|
||||
|
||||
fg_gs.grid_sample_3d = _fg_unavailable
|
||||
fg_sp.sparse_submanifold_conv3d = _fg_unavailable
|
||||
fg_ops.grid_sample = fg_gs
|
||||
fg_ops.spconv = fg_sp
|
||||
fg.ops = fg_ops
|
||||
sys.modules["flex_gemm"] = fg
|
||||
sys.modules["flex_gemm.ops"] = fg_ops
|
||||
sys.modules["flex_gemm.ops.grid_sample"] = fg_gs
|
||||
sys.modules["flex_gemm.ops.spconv"] = fg_sp
|
||||
|
||||
|
||||
def _install_sdpa_sparse_attention():
|
||||
"""Replace trellis2's sparse attention with a dense-SDPA equivalent."""
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from trellis2.modules.sparse.attention import full_attn
|
||||
from trellis2.modules.sparse import VarLenTensor
|
||||
|
||||
def sdpa_varlen(q, k, v, q_seqlen, kv_seqlen):
|
||||
# q: [Tq, H, C], k/v: [Tkv, H, C] concatenated over batch. Query-chunked
|
||||
# so the [H, chunk, L] score matrix stays bounded — the math SDPA backend
|
||||
# (forced for true fp32) would otherwise OOM at the cascade's HR token
|
||||
# counts. Chunking queries is mathematically exact (each query's softmax
|
||||
# is independent), so the golden values are unchanged.
|
||||
CHUNK = 2048
|
||||
out = torch.empty_like(q)
|
||||
qo = ko = 0
|
||||
for ql, kl in zip(q_seqlen, kv_seqlen):
|
||||
ks = k[ko:ko + kl].transpose(0, 1).unsqueeze(0) # [1,H,kl,C]
|
||||
vs = v[ko:ko + kl].transpose(0, 1).unsqueeze(0)
|
||||
for s in range(0, ql, CHUNK):
|
||||
e = min(s + CHUNK, ql)
|
||||
qs = q[qo + s:qo + e].transpose(0, 1).unsqueeze(0) # [1,H,chunk,C]
|
||||
o = F.scaled_dot_product_attention(qs, ks, vs)
|
||||
out[qo + s:qo + e] = o.squeeze(0).transpose(0, 1)
|
||||
qo += ql
|
||||
ko += kl
|
||||
return out
|
||||
|
||||
def sparse_sdpa(*args, **kwargs):
|
||||
num = len(args) + len(kwargs)
|
||||
if num == 1:
|
||||
qkv = args[0] if args else kwargs["qkv"]
|
||||
assert isinstance(qkv, VarLenTensor)
|
||||
q_seqlen = [qkv.layout[i].stop - qkv.layout[i].start for i in range(qkv.shape[0])]
|
||||
q, k, v = qkv.feats.unbind(dim=1) # [T,3,H,C] -> 3x[T,H,C]
|
||||
out = sdpa_varlen(q, k, v, q_seqlen, q_seqlen)
|
||||
return qkv.replace(out)
|
||||
if num == 2:
|
||||
q = args[0] if len(args) > 0 else kwargs["q"]
|
||||
kv = args[1] if len(args) > 1 else kwargs["kv"]
|
||||
s = q if isinstance(q, VarLenTensor) else None
|
||||
if isinstance(q, VarLenTensor):
|
||||
q_seqlen = [q.layout[i].stop - q.layout[i].start for i in range(q.shape[0])]
|
||||
qf = q.feats
|
||||
else:
|
||||
N, L = q.shape[:2]
|
||||
q_seqlen = [L] * N
|
||||
qf = q.reshape(N * L, *q.shape[2:])
|
||||
if isinstance(kv, VarLenTensor):
|
||||
kv_seqlen = [kv.layout[i].stop - kv.layout[i].start for i in range(kv.shape[0])]
|
||||
kvf = kv.feats
|
||||
else:
|
||||
N, L = kv.shape[:2]
|
||||
kv_seqlen = [L] * N
|
||||
kvf = kv.reshape(N * L, *kv.shape[2:])
|
||||
k, v = kvf.unbind(dim=1)
|
||||
out = sdpa_varlen(qf, k, v, q_seqlen, kv_seqlen)
|
||||
if s is not None:
|
||||
return s.replace(out)
|
||||
N = len(q_seqlen)
|
||||
return out.reshape(N, q_seqlen[0], *out.shape[1:])
|
||||
if num == 3:
|
||||
q = args[0] if len(args) > 0 else kwargs["q"]
|
||||
k = args[1] if len(args) > 1 else kwargs["k"]
|
||||
v = args[2] if len(args) > 2 else kwargs["v"]
|
||||
s = q if isinstance(q, VarLenTensor) else None
|
||||
if isinstance(q, VarLenTensor):
|
||||
q_seqlen = [q.layout[i].stop - q.layout[i].start for i in range(q.shape[0])]
|
||||
qf = q.feats
|
||||
else:
|
||||
N, L = q.shape[:2]
|
||||
q_seqlen = [L] * N
|
||||
qf = q.reshape(N * L, *q.shape[2:])
|
||||
if isinstance(k, VarLenTensor):
|
||||
kv_seqlen = [k.layout[i].stop - k.layout[i].start for i in range(k.shape[0])]
|
||||
kf, vf = k.feats, v.feats
|
||||
else:
|
||||
N, L = k.shape[:2]
|
||||
kv_seqlen = [L] * N
|
||||
kf = k.reshape(N * L, *k.shape[2:])
|
||||
vf = v.reshape(N * L, *v.shape[2:])
|
||||
out = sdpa_varlen(qf, kf, vf, q_seqlen, kv_seqlen)
|
||||
if s is not None:
|
||||
return s.replace(out)
|
||||
N = len(q_seqlen)
|
||||
return out.reshape(N, q_seqlen[0], *out.shape[1:])
|
||||
raise AssertionError("bad arg count")
|
||||
|
||||
full_attn.sparse_scaled_dot_product_attention = sparse_sdpa
|
||||
# modules.py imported the symbol by value; patch it there too.
|
||||
from trellis2.modules.sparse.attention import modules as attn_modules
|
||||
attn_modules.sparse_scaled_dot_product_attention = sparse_sdpa
|
||||
|
||||
|
||||
def _install_torch_sparse_conv():
|
||||
"""Register a pure-PyTorch submanifold conv backend as 'none'."""
|
||||
import math
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from trellis2.modules.sparse.conv import conv as conv_dispatch
|
||||
|
||||
mod = types.ModuleType("trellis2.modules.sparse.conv.conv_none")
|
||||
|
||||
def sparse_conv3d_init(self, in_channels, out_channels, kernel_size,
|
||||
stride=1, dilation=1, padding=None, bias=True,
|
||||
indice_key=None):
|
||||
assert stride == 1 and padding is None, "submanifold only"
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
ks = tuple(kernel_size) if isinstance(kernel_size, (list, tuple)) else (kernel_size,) * 3
|
||||
self.kernel_size = ks
|
||||
self.stride = (1, 1, 1)
|
||||
self.dilation = tuple(dilation) if isinstance(dilation, (list, tuple)) else (dilation,) * 3
|
||||
# flex_gemm weight layout: (Co, Kd, Kh, Kw, Ci)
|
||||
self.weight = nn.Parameter(torch.empty(out_channels, *ks, in_channels))
|
||||
if bias:
|
||||
self.bias = nn.Parameter(torch.zeros(out_channels))
|
||||
else:
|
||||
self.register_parameter("bias", None)
|
||||
nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
|
||||
|
||||
def _coord_key(coords, spatial_shape):
|
||||
# coords: [N,4] int (b,x,y,z) -> int64 linear key
|
||||
b, x, y, z = coords.unbind(-1)
|
||||
sx, sy, sz = spatial_shape
|
||||
return ((b.long() * sx + x.long()) * sy + y.long()) * sz + z.long()
|
||||
|
||||
def sparse_conv3d_forward(self, x):
|
||||
coords = x.coords
|
||||
feats = x.feats
|
||||
n = feats.shape[0]
|
||||
spatial = tuple(x.spatial_shape)
|
||||
keys = _coord_key(coords, spatial)
|
||||
order = torch.argsort(keys)
|
||||
keys_sorted = keys[order]
|
||||
|
||||
Co, Kd, Kh, Kw, Ci = self.weight.shape
|
||||
w = self.weight
|
||||
out = feats.new_zeros(n, Co)
|
||||
if self.bias is not None:
|
||||
out += self.bias.to(out.dtype)
|
||||
rd, rh, rw = Kd // 2, Kh // 2, Kw // 2
|
||||
dd, dh, dw = self.dilation
|
||||
for kd in range(Kd):
|
||||
for kh in range(Kh):
|
||||
for kw in range(Kw):
|
||||
off = coords.new_tensor([0, (kd - rd) * dd, (kh - rh) * dh, (kw - rw) * dw])
|
||||
ncoords = coords + off
|
||||
inb = ((ncoords[:, 1] >= 0) & (ncoords[:, 1] < spatial[0]) &
|
||||
(ncoords[:, 2] >= 0) & (ncoords[:, 2] < spatial[1]) &
|
||||
(ncoords[:, 3] >= 0) & (ncoords[:, 3] < spatial[2]))
|
||||
nkeys = _coord_key(ncoords, spatial)
|
||||
pos = torch.searchsorted(keys_sorted, nkeys)
|
||||
pos_c = pos.clamp(max=n - 1)
|
||||
hit = inb & (keys_sorted[pos_c] == nkeys)
|
||||
src = order[pos_c[hit]]
|
||||
# out[i] += feats[neighbor(i, offset)] @ w[:, kd, kh, kw, :]^T
|
||||
contrib = feats[src] @ w[:, kd, kh, kw, :].to(feats.dtype).t()
|
||||
out[hit] += contrib
|
||||
return x.replace(out)
|
||||
|
||||
def sparse_inverse_conv3d_init(self, *a, **k):
|
||||
raise NotImplementedError
|
||||
|
||||
def sparse_inverse_conv3d_forward(self, x):
|
||||
raise NotImplementedError
|
||||
|
||||
mod.sparse_conv3d_init = sparse_conv3d_init
|
||||
mod.sparse_conv3d_forward = sparse_conv3d_forward
|
||||
mod.sparse_inverse_conv3d_init = sparse_inverse_conv3d_init
|
||||
mod.sparse_inverse_conv3d_forward = sparse_inverse_conv3d_forward
|
||||
conv_dispatch._backends["none"] = mod
|
||||
from trellis2.modules import sparse as sp
|
||||
sp.config.CONV = "none"
|
||||
|
||||
|
||||
def _force_true_fp32():
|
||||
"""Make CUDA math bit-comparable to a real fp32 reference.
|
||||
|
||||
PyTorch's default CUDA matmul/attention uses TF32 (≈10-bit mantissa) and
|
||||
flash/mem-efficient SDPA kernels that accumulate in reduced precision —
|
||||
that shows up as ~1e-3 relative error versus a true fp32 (or ggml-CPU)
|
||||
forward, which would otherwise masquerade as a port bug. Force full-width
|
||||
fp32 so the golden dumps are the real reference regardless of --device.
|
||||
"""
|
||||
import torch
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
if hasattr(torch.backends.cuda, "enable_flash_sdp"):
|
||||
torch.backends.cuda.enable_flash_sdp(False)
|
||||
torch.backends.cuda.enable_mem_efficient_sdp(False)
|
||||
torch.backends.cuda.enable_math_sdp(True)
|
||||
|
||||
|
||||
def setup():
|
||||
_force_true_fp32()
|
||||
_install_sdpa_sparse_attention()
|
||||
_install_torch_sparse_conv()
|
||||
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MODELS = os.path.join(REPO, "models")
|
||||
DUMPS = os.path.join(REPO, "dumps")
|
||||
|
||||
|
||||
def write_dinodata(path, arr):
|
||||
""".dinodata: DINOCOND | u32 version | u32 dtype(0=f32) | u32 ndim | dims | f32 payload"""
|
||||
import struct
|
||||
import numpy as np
|
||||
arr = np.ascontiguousarray(arr, dtype="<f4")
|
||||
with open(path, "wb") as f:
|
||||
f.write(b"DINOCOND")
|
||||
f.write(struct.pack("<III", 1, 0, arr.ndim))
|
||||
f.write(struct.pack("<%dI" % arr.ndim, *arr.shape))
|
||||
f.write(arr.tobytes())
|
||||
|
||||
|
||||
def preprocess_rgba(img):
|
||||
"""pipeline.preprocess_image for an RGBA input (no rembg): downscale to
|
||||
<=1024, alpha bbox square crop, premultiply onto black. Returns RGB PIL."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
assert img.mode == "RGBA", "fixture must have an alpha channel"
|
||||
max_size = max(img.size)
|
||||
scale = min(1, 1024 / max_size)
|
||||
if scale < 1:
|
||||
img = img.resize((int(img.width * scale), int(img.height * scale)),
|
||||
Image.Resampling.LANCZOS)
|
||||
out = np.array(img)
|
||||
alpha = out[:, :, 3]
|
||||
bbox = np.argwhere(alpha > 0.8 * 255)
|
||||
bbox = np.min(bbox[:, 1]), np.min(bbox[:, 0]), np.max(bbox[:, 1]), np.max(bbox[:, 0])
|
||||
center = (bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2
|
||||
size = max(bbox[2] - bbox[0], bbox[3] - bbox[1])
|
||||
size = int(size * 1)
|
||||
bbox = center[0] - size // 2, center[1] - size // 2, center[0] + size // 2, center[1] + size // 2
|
||||
img = img.crop(bbox)
|
||||
out = np.array(img).astype(np.float32) / 255
|
||||
out = out[:, :, :3] * out[:, :, 3:4]
|
||||
return Image.fromarray((out * 255).astype(np.uint8))
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reference PBR texturing, end-to-end, on OUR generated mesh -- the golden target
|
||||
the C++/ggml texture port validates against, and a quick eyeball of the texture
|
||||
NN stages.
|
||||
|
||||
Runs the real Trellis2TexturingPipeline NN stages (shape encoder -> tex SLAT
|
||||
flow -> tex decoder) with sparse ops monkeypatched to pure torch (ref_common),
|
||||
so it needs NO custom CUDA kernels except o-voxel's CPU mesh->dual-grid.
|
||||
|
||||
For the eyeball we skip the CUDA-only UV bake (nvdiffrast/cumesh/flexgemm) and
|
||||
instead trilinear-sample the decoded 6-channel PBR voxels at each mesh vertex
|
||||
(base_color / metallic / roughness), then dump a coloured mesh to render.
|
||||
|
||||
python scripts/ref_texture.py --mesh <T2MESH01.bin> --image <rgba.png> \
|
||||
--resolution 512 --out dumps/tex_vcolor.bin
|
||||
"""
|
||||
import argparse, json, os, struct, sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import o_voxel # noqa: E402 (import the REAL o-voxel BEFORE ref_common so its
|
||||
import o_voxel.convert # stub guard skips it; we need mesh_to_flexible_dual_grid)
|
||||
import ref_common # noqa: E402
|
||||
ref_common.setup()
|
||||
|
||||
import numpy as np # noqa: E402
|
||||
import torch # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
import trimesh # noqa: E402
|
||||
|
||||
from safetensors.torch import load_file # noqa: E402
|
||||
from trellis2.models.sc_vaes.fdg_vae import FlexiDualGridVaeEncoder # noqa: E402
|
||||
from trellis2.models.sc_vaes.sparse_unet_vae import SparseUnetVaeDecoder # noqa: E402
|
||||
from trellis2.models.structured_latent_flow import SLatFlowModel # noqa: E402
|
||||
from trellis2.pipelines.samplers import FlowEulerGuidanceIntervalSampler # noqa: E402
|
||||
from trellis2.pipelines import Trellis2TexturingPipeline # noqa: E402
|
||||
from trellis2.modules.image_feature_extractor import DinoV3FeatureExtractor # noqa: E402
|
||||
|
||||
|
||||
def load_t2mesh(path):
|
||||
b = open(path, "rb").read()
|
||||
assert b[:8] == b"T2MESH01", b[:8]
|
||||
nv, nt = struct.unpack("<II", b[8:16]); o = 16
|
||||
V = np.frombuffer(b, "<f4", 3 * nv, o).reshape(-1, 3).copy(); o += 12 * nv
|
||||
o += 12 * nv # skip normals
|
||||
F = np.frombuffer(b, "<i4", 3 * nt, o).reshape(-1, 3).copy()
|
||||
return V, F
|
||||
|
||||
|
||||
def trilinear_sample_sparse(feats, coords_xyz, query_vox):
|
||||
"""feats [M,C], coords_xyz [M,3] int voxel coords, query_vox [Q,3] float.
|
||||
Returns (sampled [Q,C], weight [Q]) trilinear over present voxels."""
|
||||
dev = feats.device
|
||||
MULT = 4096
|
||||
enc = lambda c: (c[:, 0].long() * MULT + c[:, 1].long()) * MULT + c[:, 2].long()
|
||||
ck = enc(coords_xyz)
|
||||
order = torch.argsort(ck)
|
||||
ck_s = ck[order]; f_s = feats[order]
|
||||
base = torch.floor(query_vox)
|
||||
frac = query_vox - base
|
||||
out = torch.zeros(query_vox.shape[0], feats.shape[1], device=dev)
|
||||
wsum = torch.zeros(query_vox.shape[0], device=dev)
|
||||
for dx in (0, 1):
|
||||
for dy in (0, 1):
|
||||
for dz in (0, 1):
|
||||
corner = base + torch.tensor([dx, dy, dz], device=dev, dtype=base.dtype)
|
||||
w = (frac[:, 0] if dx else 1 - frac[:, 0]) * \
|
||||
(frac[:, 1] if dy else 1 - frac[:, 1]) * \
|
||||
(frac[:, 2] if dz else 1 - frac[:, 2])
|
||||
cck = enc(corner)
|
||||
pos = torch.searchsorted(ck_s, cck).clamp(max=ck_s.shape[0] - 1)
|
||||
found = (ck_s[pos] == cck)
|
||||
out += (w * found)[:, None] * f_s[pos]
|
||||
wsum += w * found
|
||||
return out / wsum.clamp(min=1e-6)[:, None], wsum
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--mesh", required=True)
|
||||
ap.add_argument("--image", required=True)
|
||||
ap.add_argument("--resolution", type=int, default=512, choices=[512, 1024])
|
||||
ap.add_argument("--seed", type=int, default=42)
|
||||
ap.add_argument("--out", default=os.path.join(ref_common.DUMPS, "tex_vcolor.bin"))
|
||||
args = ap.parse_args()
|
||||
dev = "cuda"
|
||||
CK = os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "ckpts")
|
||||
tpa = json.load(open(os.path.join(ref_common.MODELS, "TRELLIS.2-4B", "texturing_pipeline.json")))["args"]
|
||||
|
||||
def load_model(cls, stem, **extra):
|
||||
cfg = json.load(open(os.path.join(CK, stem + ".json")))["args"]
|
||||
m = cls(**{**cfg, **extra})
|
||||
sd = {k: v.float() for k, v in load_file(os.path.join(CK, stem + ".safetensors")).items()}
|
||||
m.load_state_dict(sd)
|
||||
return m.eval().to(dev)
|
||||
|
||||
R = args.resolution
|
||||
print(f"loading models (res {R}) ...", flush=True)
|
||||
shape_enc = load_model(FlexiDualGridVaeEncoder, "shape_enc_next_dc_f16c32_fp16")
|
||||
tex_dec = load_model(SparseUnetVaeDecoder, "tex_dec_next_dc_f16c32_fp16")
|
||||
tex_flow = load_model(SLatFlowModel, f"slat_flow_imgshape2tex_dit_1_3B_{R}_bf16", dtype="float32")
|
||||
|
||||
dino = DinoV3FeatureExtractor(os.path.join(ref_common.MODELS, "dinov3-vitl16"), image_size=R)
|
||||
dino.model = dino.model.to(dev).eval()
|
||||
|
||||
sampler = FlowEulerGuidanceIntervalSampler(**tpa["tex_slat_sampler"]["args"])
|
||||
pipe = Trellis2TexturingPipeline(
|
||||
models={"shape_slat_encoder": shape_enc, "tex_slat_decoder": tex_dec,
|
||||
f"tex_slat_flow_model_{R}": tex_flow},
|
||||
tex_slat_sampler=sampler,
|
||||
tex_slat_sampler_params=tpa["tex_slat_sampler"]["params"],
|
||||
shape_slat_normalization=tpa["shape_slat_normalization"],
|
||||
tex_slat_normalization=tpa["tex_slat_normalization"],
|
||||
image_cond_model=dino, rembg_model=None, low_vram=False,
|
||||
)
|
||||
pipe._device = dev
|
||||
|
||||
V, F = load_t2mesh(args.mesh)
|
||||
mesh = trimesh.Trimesh(vertices=V, faces=F, process=False)
|
||||
img = Image.open(args.image).convert("RGBA")
|
||||
print(f"mesh {len(V)} verts / {len(F)} tris; image {img.size}", flush=True)
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
with torch.no_grad():
|
||||
image = pipe.preprocess_image(img)
|
||||
mesh = pipe.preprocess_mesh(mesh)
|
||||
cond = pipe.get_cond([image], R)
|
||||
print("encoding shape SLAT (o-voxel dual grid + shape encoder) ...", flush=True)
|
||||
shape_slat = pipe.encode_shape_slat(mesh, R)
|
||||
print(f"shape_slat: {shape_slat.feats.shape} coords {shape_slat.coords.shape}", flush=True)
|
||||
print("sampling tex SLAT (flow, concat_cond) ...", flush=True)
|
||||
tex_slat = pipe.sample_tex_slat(cond, tex_flow, shape_slat)
|
||||
print("decoding tex SLAT -> PBR voxels ...", flush=True)
|
||||
pbr = pipe.decode_tex_slat(tex_slat) # SparseTensor, 6ch, already *0.5+0.5
|
||||
print(f"pbr voxels: {pbr.feats.shape} coords {pbr.coords.shape} "
|
||||
f"range [{pbr.feats.min():.3f},{pbr.feats.max():.3f}]", flush=True)
|
||||
|
||||
# per-vertex trilinear sample of the PBR voxels (mesh is now normalized to [-.5,.5])
|
||||
Vt = torch.from_numpy(mesh.vertices).float().to(dev)
|
||||
qvox = (Vt + 0.5) * R
|
||||
feats = pbr.feats.float()
|
||||
coords_xyz = pbr.coords[:, 1:].to(dev)
|
||||
vals, w = trilinear_sample_sparse(feats, coords_xyz, qvox)
|
||||
vals = vals.clamp(0, 1).cpu().numpy()
|
||||
hit = (w > 1e-4).float().mean().item()
|
||||
print(f"per-vertex sample hit-rate {hit*100:.1f}%", flush=True)
|
||||
|
||||
base_color = vals[:, 0:3]
|
||||
metallic = vals[:, 3:4]; roughness = vals[:, 4:5]
|
||||
Vout = mesh.vertices.astype("<f4")
|
||||
Fout = mesh.faces.astype("<i4")
|
||||
with open(args.out, "wb") as f:
|
||||
f.write(b"T2VCOL01")
|
||||
f.write(struct.pack("<II", len(Vout), len(Fout)))
|
||||
f.write(Vout.tobytes())
|
||||
f.write(base_color.astype("<f4").tobytes())
|
||||
f.write(np.concatenate([metallic, roughness], 1).astype("<f4").tobytes())
|
||||
f.write(Fout.tobytes())
|
||||
print(f"wrote {args.out} (verts+rgb+metal/rough+tris)", flush=True)
|
||||
print(f"base_color mean {base_color.mean(0)} metallic {metallic.mean():.3f} rough {roughness.mean():.3f}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate all PyTorch reference dumps inside the reference container.
|
||||
# Usage: scripts/refgen.sh [fixture-image]
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
TRELLIS2_PY_HOST="${TRELLIS2_PY_HOST:-/home/rich/python/TRELLIS.2}"
|
||||
FIXTURE="${1:-/trellis2/assets/example_image/0a34fae7ba57cb8870df5325b9c30ea474def1b0913c19c596655b85a79fdee4.webp}"
|
||||
|
||||
run() {
|
||||
docker run --rm --device nvidia.com/gpu=all \
|
||||
-v "$ROOT":/work -v "$TRELLIS2_PY_HOST":/trellis2 \
|
||||
-e PYTHONPATH=/trellis2 -e TRELLIS2_PY=/trellis2 \
|
||||
-e ATTN_BACKEND=sdpa -e SPARSE_CONV_BACKEND=none \
|
||||
-e HF_HUB_OFFLINE=1 \
|
||||
-w /work trellis2-ref "$@"
|
||||
}
|
||||
|
||||
mkdir -p "$ROOT/dumps"
|
||||
run python scripts/dump_dino_reference.py --image "$FIXTURE"
|
||||
run python tests/ref_ss_flow.py # CPU (true fp32 golden)
|
||||
run python tests/ref_ss_sample.py --device cuda
|
||||
run python tests/ref_ss_dec.py --device cuda
|
||||
run python scripts/dump_slat_reference.py --device cuda # TF32 disabled in ref_common
|
||||
run python scripts/dump_cascade_reference.py --device cuda # 1024 HR stage
|
||||
echo "reference dumps regenerated:"
|
||||
ls -la "$ROOT/dumps" "$ROOT"/tests/*.bin
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Load a GLB and render it three ways from vertex colour or a baked UV atlas.
|
||||
Also prints structural stats (a sanity check that the glTF is well-formed).
|
||||
|
||||
python3 scripts/render_glb.py in.glb [out.png]
|
||||
|
||||
Runs anywhere with trimesh + numpy + Pillow (e.g. the t2tex container).
|
||||
"""
|
||||
import sys, numpy as np, trimesh
|
||||
from PIL import Image
|
||||
|
||||
path = sys.argv[1]; out = sys.argv[2] if len(sys.argv) > 2 else "glb_render.png"
|
||||
scene = trimesh.load(path, process=False)
|
||||
mesh = scene if isinstance(scene, trimesh.Trimesh) else list(scene.geometry.values())[0]
|
||||
|
||||
V = np.asarray(mesh.vertices, np.float64)
|
||||
F = np.asarray(mesh.faces, np.int64)
|
||||
visual_attrs = getattr(mesh.visual, "vertex_attributes", {})
|
||||
if "color" in visual_attrs:
|
||||
raw = np.asarray(visual_attrs["color"])
|
||||
scale = np.iinfo(raw.dtype).max if np.issubdtype(raw.dtype, np.integer) else 1.0
|
||||
base_v = np.asarray(raw[:, :3], np.float64) / scale
|
||||
# glTF COLOR_0 is already linear.
|
||||
base_linear_v = np.clip(base_v, 0, 1)
|
||||
custom = getattr(mesh, "vertex_attributes", {})
|
||||
if "_METALLIC_ROUGHNESS" in custom:
|
||||
mr = np.asarray(custom["_METALLIC_ROUGHNESS"])
|
||||
mr_scale = np.iinfo(mr.dtype).max if np.issubdtype(mr.dtype, np.integer) else 1.0
|
||||
metal_v = np.asarray(mr[:, 0], np.float64) / mr_scale
|
||||
rough_v = np.asarray(mr[:, 1], np.float64) / mr_scale
|
||||
else:
|
||||
metal_v = np.zeros(len(V)); rough_v = np.full(len(V), 0.6)
|
||||
print(f"GLB: {len(V):,} verts {len(F):,} faces COLOR_0 {raw.dtype} "
|
||||
f"custom metalRough={'yes' if '_METALLIC_ROUGHNESS' in custom else 'no'}", flush=True)
|
||||
else:
|
||||
uv = np.asarray(mesh.visual.uv, np.float64)
|
||||
mat = mesh.visual.material
|
||||
base_img = np.asarray(mat.baseColorTexture.convert("RGB"), np.float64) / 255.0
|
||||
try:
|
||||
mr_img = np.asarray(mat.metallicRoughnessTexture.convert("RGB"), np.float64) / 255.0
|
||||
except Exception:
|
||||
mr_img = None
|
||||
TH, TW = base_img.shape[:2]
|
||||
def sample(img, u, v):
|
||||
x = np.clip((u % 1.0) * (TW - 1), 0, TW - 1).astype(np.int32)
|
||||
y = np.clip((v % 1.0) * (TH - 1), 0, TH - 1).astype(np.int32)
|
||||
return img[y, x]
|
||||
base_v = sample(base_img, uv[:, 0], uv[:, 1])
|
||||
base_linear_v = np.clip(base_v, 0, 1) ** 2.2
|
||||
if mr_img is not None:
|
||||
rough_v = sample(mr_img, uv[:, 0], uv[:, 1])[:, 1]
|
||||
metal_v = sample(mr_img, uv[:, 0], uv[:, 1])[:, 2]
|
||||
else:
|
||||
rough_v = np.full(len(V), 0.6); metal_v = np.zeros(len(V))
|
||||
print(f"GLB: {len(V):,} verts {len(F):,} faces uv[{uv.min():.3f},{uv.max():.3f}] "
|
||||
f"baseColor {base_img.shape} metalRough {None if mr_img is None else mr_img.shape}", flush=True)
|
||||
|
||||
N = np.zeros_like(V)
|
||||
fn = np.cross(V[F[:, 1]] - V[F[:, 0]], V[F[:, 2]] - V[F[:, 0]])
|
||||
for k in range(3): np.add.at(N, F[:, k], fn)
|
||||
N /= np.linalg.norm(N, axis=1, keepdims=True) + 1e-9
|
||||
|
||||
SS = 2; W = H = 720 * SS
|
||||
c = (V.max(0) + V.min(0)) / 2; Vc = V - c; Vc *= 0.92 / np.abs(Vc).max()
|
||||
LC = [(np.array([.32, .55, .77]), np.array([1., .96, .88]), .85),
|
||||
(np.array([-.65, .20, .45]), np.array([.70, .80, 1.]), .45),
|
||||
(np.array([.10, -.75, .35]), np.array([.85, .85, .90]), .25)]
|
||||
|
||||
|
||||
def Rmat(el, az):
|
||||
a, e = np.radians(az), np.radians(el)
|
||||
Ry = np.array([[np.cos(a), 0, np.sin(a)], [0, 1, 0], [-np.sin(a), 0, np.cos(a)]])
|
||||
Rx = np.array([[1, 0, 0], [0, np.cos(e), -np.sin(e)], [0, np.sin(e), np.cos(e)]])
|
||||
return Rx @ Ry
|
||||
|
||||
|
||||
def render(el, az):
|
||||
r = Rmat(el, az); P = Vc @ r.T; Nr = N @ r.T
|
||||
Nr /= np.linalg.norm(Nr, axis=1, keepdims=True) + 1e-9
|
||||
sx = ((P[:, 0]*.5+.5)*(W-1)).astype(np.int32); sy = ((.5-P[:, 1]*.5)*(H-1)).astype(np.int32)
|
||||
z = P[:, 2]
|
||||
view = np.array([0., 0., 1.]); nv_ = np.abs(Nr @ view)
|
||||
F0 = 0.04*(1-metal_v)[:, None] + base_linear_v*metal_v[:, None]
|
||||
Fr = F0 + (1-F0)*((1-nv_)[:, None]**5)
|
||||
shin = 6.0 + 214.0*np.clip(1-rough_v, 0, 1)**1.5
|
||||
sn = (shin+2.0)/(2*np.pi)
|
||||
diffuse = np.zeros_like(base_v); specular = np.zeros_like(base_v)
|
||||
for Ld, Lc, w in LC:
|
||||
l = Ld/np.linalg.norm(Ld); ndl = np.abs(Nr @ l)
|
||||
diffuse += Lc[None]*w*ndl[:, None]
|
||||
h = l+view; h = h/np.linalg.norm(h); nh = np.abs(Nr @ h)
|
||||
specular += Lc[None]*w*(nh[:, None]**shin[:, None])*sn[:, None]*ndl[:, None]
|
||||
albedo = base_linear_v*(1-metal_v)[:, None]
|
||||
hemi = np.abs(N[:, 1])*0.5+0.5
|
||||
amb = (1-hemi)[:, None]*np.array([.20, .19, .22]) + hemi[:, None]*np.array([.55, .58, .66])
|
||||
col = np.clip(albedo*(amb*0.55 + diffuse*0.75) + specular*Fr + Fr*0.25, 0, 1)
|
||||
img = np.ones((H, W, 3))
|
||||
idx = np.argsort(z); sxi, syi, coli = sx[idx], sy[idx], col[idx]
|
||||
for dx in range(SS+1):
|
||||
for dy in range(SS+1):
|
||||
xx = sxi+dx; yy = syi+dy
|
||||
m = (xx >= 0) & (xx < W) & (yy >= 0) & (yy < H)
|
||||
img[yy[m], xx[m]] = coli[m]
|
||||
return (img**(1/2.2)*255).reshape(H//SS, SS, W//SS, SS, 3).mean((1, 3)).astype(np.uint8)
|
||||
|
||||
|
||||
row = np.concatenate([render(12, 25), render(12, 150), render(65, 20)], 1)
|
||||
Image.fromarray(row).save(out)
|
||||
print("wrote", out, flush=True)
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render a T2MESH02/03 (per-vertex PBR) binary mesh to a PNG for eyeballing the
|
||||
demo's textured output. Mirrors the viewer's orientation-independent shading."""
|
||||
import struct, sys, numpy as np
|
||||
from PIL import Image
|
||||
|
||||
path = sys.argv[1]; out = sys.argv[2] if len(sys.argv) > 2 else "mesh_pbr.png"
|
||||
b = open(path, "rb").read()
|
||||
magic = b[:8]; nv, nt = struct.unpack("<II", b[8:16]); o = 16
|
||||
V = np.frombuffer(b, "<f4", 3*nv, o).reshape(-1, 3).astype(np.float64); o += 12*nv
|
||||
o += 12*nv # skip normals (recompute from faces)
|
||||
pbr = None
|
||||
if magic == b"T2MESH03":
|
||||
pbr = np.frombuffer(b, "<f4", 6*nv, o).reshape(-1, 6).astype(np.float64); o += 24*nv
|
||||
elif magic == b"T2MESH02":
|
||||
old = np.frombuffer(b, "<f4", 5*nv, o).reshape(-1, 5).astype(np.float64); o += 20*nv
|
||||
pbr = np.concatenate([old, np.ones((nv, 1))], axis=1)
|
||||
F = np.frombuffer(b, "<i4", 3*nt, o).reshape(-1, 3)
|
||||
C = pbr[:, 0:3] if pbr is not None else np.full((nv, 3), 0.65)
|
||||
C_lin = np.clip(C, 0, 1) ** 2.2
|
||||
metal = pbr[:, 3] if pbr is not None else np.zeros(nv)
|
||||
print(f"{magic} {nv:,} verts base_color mean {C.mean(0).round(3)}", flush=True)
|
||||
|
||||
fn = np.cross(V[F[:, 1]] - V[F[:, 0]], V[F[:, 2]] - V[F[:, 0]])
|
||||
N = np.zeros_like(V)
|
||||
for k in range(3): np.add.at(N, F[:, k], fn)
|
||||
N /= np.linalg.norm(N, axis=1, keepdims=True) + 1e-9
|
||||
|
||||
SS = 2; W = H = 720 * SS
|
||||
c = (V.max(0) + V.min(0)) / 2; Vc = V - c; Vc *= 0.92 / np.abs(Vc).max()
|
||||
l1 = np.array([0.5, 0.8, 0.6]); l1 /= np.linalg.norm(l1)
|
||||
l2 = np.array([-0.6, -0.2, -0.7]); l2 /= np.linalg.norm(l2)
|
||||
|
||||
|
||||
def R(el, az):
|
||||
a, e = np.radians(az), np.radians(el)
|
||||
Ry = np.array([[np.cos(a), 0, np.sin(a)], [0, 1, 0], [-np.sin(a), 0, np.cos(a)]])
|
||||
Rx = np.array([[1, 0, 0], [0, np.cos(e), -np.sin(e)], [0, np.sin(e), np.cos(e)]])
|
||||
return Rx @ Ry
|
||||
|
||||
|
||||
rough = pbr[:, 4] if pbr is not None else np.full(nv, 0.55)
|
||||
# view-space metallic-roughness PBR, matching server/web/index.html
|
||||
LC = [(np.array([.32, .55, .77]), np.array([1., .96, .88]), .85),
|
||||
(np.array([-.65, .20, .45]), np.array([.70, .80, 1.]), .45),
|
||||
(np.array([.10, -.75, .35]), np.array([.85, .85, .90]), .25)]
|
||||
|
||||
|
||||
def render(el, az):
|
||||
r = R(el, az); P = Vc @ r.T; Nr = N @ r.T
|
||||
Nr /= np.linalg.norm(Nr, axis=1, keepdims=True) + 1e-9
|
||||
sx = ((P[:, 0]*.5+.5)*(W-1)).astype(np.int32); sy = ((.5-P[:, 1]*.5)*(H-1)).astype(np.int32)
|
||||
z = P[:, 2]
|
||||
V = np.array([0., 0., 1.])
|
||||
nv_ = np.abs(Nr @ V)
|
||||
F0 = 0.04*(1-metal)[:, None] + C_lin*metal[:, None]
|
||||
F = F0 + (1-F0)*((1-nv_)[:, None]**5)
|
||||
shin = 6.0 + (220.0-6.0)*np.clip(1-rough, 0, 1)**1.5
|
||||
sn = (shin+2.0)/(2*np.pi)
|
||||
diffuse = np.zeros((len(C), 3)); specular = np.zeros((len(C), 3))
|
||||
for Ld, Lc, w in LC:
|
||||
l = Ld/np.linalg.norm(Ld); ndl = np.abs(Nr @ l)
|
||||
diffuse += Lc[None]*w*ndl[:, None]
|
||||
h = l+V; h = h/np.linalg.norm(h); nh = np.abs(Nr @ h)
|
||||
specular += Lc[None]*w*(nh[:, None]**shin[:, None])*sn[:, None]*ndl[:, None]
|
||||
albedo = C_lin*(1-metal)[:, None]
|
||||
hemi = np.abs(N[:, 1])*0.5+0.5
|
||||
amb = (1-hemi)[:, None]*np.array([.20, .19, .22]) + hemi[:, None]*np.array([.55, .58, .66])
|
||||
col = np.clip(albedo*(amb*0.55 + diffuse*0.75) + specular*F + F*0.25, 0, 1)
|
||||
img = np.ones((H, W, 3))
|
||||
idx = np.argsort(z); sx, sy, col = sx[idx], sy[idx], col[idx]
|
||||
for dx in range(SS+1):
|
||||
for dy in range(SS+1):
|
||||
xx = sx+dx; yy = sy+dy
|
||||
m = (xx >= 0) & (xx < W) & (yy >= 0) & (yy < H)
|
||||
img[yy[m], xx[m]] = col[m]
|
||||
return (img**(1/2.2)*255).reshape(H//SS, SS, W//SS, SS, 3).mean((1, 3)).astype(np.uint8)
|
||||
|
||||
|
||||
row = np.concatenate([render(12, 25), render(12, 150), render(65, 20)], 1)
|
||||
Image.fromarray(row).save(out)
|
||||
print("wrote", out, flush=True)
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render a T2VCOL01 coloured mesh (verts + per-vertex base_color) with a numpy
|
||||
z-buffer, a few views, for eyeballing the PBR texture NN output."""
|
||||
import struct, sys, numpy as np
|
||||
from PIL import Image
|
||||
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else "dumps/tex_vcolor.bin"
|
||||
out = sys.argv[2] if len(sys.argv) > 2 else "dumps/tex_vcolor.png"
|
||||
b = open(path, "rb").read()
|
||||
assert b[:8] == b"T2VCOL01", b[:8]
|
||||
nv, nt = struct.unpack("<II", b[8:16]); o = 16
|
||||
V = np.frombuffer(b, "<f4", 3 * nv, o).reshape(-1, 3).astype(np.float64); o += 12 * nv
|
||||
C = np.frombuffer(b, "<f4", 3 * nv, o).reshape(-1, 3).astype(np.float64); o += 12 * nv
|
||||
o += 8 * nv # skip metal/rough
|
||||
F = np.frombuffer(b, "<i4", 3 * nt, o).reshape(-1, 3)
|
||||
print(f"{nv:,} verts {nt:,} tris color mean {C.mean(0)}", flush=True)
|
||||
|
||||
# face normals -> per-vertex (for a little shading on top of albedo)
|
||||
fn = np.cross(V[F[:, 1]] - V[F[:, 0]], V[F[:, 2]] - V[F[:, 0]])
|
||||
N = np.zeros_like(V)
|
||||
for k in range(3):
|
||||
np.add.at(N, F[:, k], fn)
|
||||
N /= np.linalg.norm(N, axis=1, keepdims=True) + 1e-9
|
||||
|
||||
SS = 2; W = H = 640 * SS
|
||||
c = (V.max(0) + V.min(0)) / 2; Vc = V - c; Vc *= 0.9 / np.abs(Vc).max()
|
||||
l1 = np.array([0.5, 0.8, 0.6]); l1 /= np.linalg.norm(l1)
|
||||
l2 = np.array([-0.6, -0.2, -0.7]); l2 /= np.linalg.norm(l2)
|
||||
|
||||
|
||||
def R(el, az):
|
||||
a = np.radians(az); e = np.radians(el)
|
||||
Ry = np.array([[np.cos(a), 0, np.sin(a)], [0, 1, 0], [-np.sin(a), 0, np.cos(a)]])
|
||||
Rx = np.array([[1, 0, 0], [0, np.cos(e), -np.sin(e)], [0, np.sin(e), np.cos(e)]])
|
||||
return Rx @ Ry
|
||||
|
||||
|
||||
def render(el, az):
|
||||
r = R(el, az); P = Vc @ r.T; Nr = N @ r.T
|
||||
sx = ((P[:, 0] * .5 + .5) * (W - 1)).astype(np.int32)
|
||||
sy = ((.5 - P[:, 1] * .5) * (H - 1)).astype(np.int32)
|
||||
z = P[:, 2]
|
||||
shade = (np.abs(Nr @ l1) * .7 + np.abs(Nr @ l2) * .3) * 0.6 + 0.4 # gentle, keep albedo
|
||||
col = np.clip(C * shade[:, None], 0, 1)
|
||||
img = np.ones((H, W, 3)); zb = np.full((H, W), -1e9)
|
||||
idx = np.argsort(z); sx, sy, z, col = sx[idx], sy[idx], z[idx], col[idx]
|
||||
for dx in range(SS + 1):
|
||||
for dy in range(SS + 1):
|
||||
xx = sx + dx; yy = sy + dy
|
||||
m = (xx >= 0) & (xx < W) & (yy >= 0) & (yy < H)
|
||||
img[yy[m], xx[m]] = col[m]; zb[yy[m], xx[m]] = z[m]
|
||||
im = (img ** (1 / 2.2) * 255).reshape(H // SS, SS, W // SS, SS, 3).mean((1, 3))
|
||||
return im.astype(np.uint8)
|
||||
|
||||
|
||||
row = np.concatenate([render(15, 25), render(15, 120), render(70, 20)], 1)
|
||||
Image.fromarray(row).save(out)
|
||||
print("wrote", out, flush=True)
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# One-time (idempotent) publish of the prebuilt f16 GGUFs to the LocalAI-io org on
|
||||
# Hugging Face. Splits by upstream provenance so each repo carries a single clean
|
||||
# license:
|
||||
# TRELLIS.2-4B-GGUF (MIT) 8 files
|
||||
# TRELLIS-image-large-GGUF (MIT) ss_dec
|
||||
# dinov3-vitl16-pretrain-lvd1689m-GGUF (DINOv3 License) dino
|
||||
#
|
||||
# Cards + LICENSE files live under scripts/hf/<repo>/ and are uploaded alongside the
|
||||
# weights. Requires an authenticated `hf` (huggingface_hub >= 1.x) with write access
|
||||
# to the org. `hf` is often not on PATH; set HF=/path/to/venv/bin/hf to override.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
GGUFS="${GGUFS:-$ROOT/ggufs}"
|
||||
CARDS="$ROOT/scripts/hf"
|
||||
ORG="${GGUF_ORG:-LocalAI-io}"
|
||||
|
||||
# Locate the hf CLI: explicit $HF, then PATH, then known project venvs.
|
||||
HF="${HF:-}"
|
||||
if [ -z "$HF" ]; then
|
||||
if command -v hf >/dev/null 2>&1; then HF="$(command -v hf)"
|
||||
else
|
||||
for c in "$HOME"/.venvs/*/bin/hf "$HOME"/c/*/.venv*/bin/hf; do
|
||||
[ -x "$c" ] && { HF="$c"; break; }
|
||||
done
|
||||
fi
|
||||
fi
|
||||
[ -n "$HF" ] && [ -x "$HF" ] || { echo "error: hf CLI not found; set HF=/path/to/hf" >&2; exit 1; }
|
||||
echo "using hf: $HF ($("$HF" --version 2>/dev/null))"
|
||||
"$HF" auth whoami >/dev/null || { echo "error: not logged in (hf auth login)" >&2; exit 1; }
|
||||
|
||||
T2=TRELLIS.2-4B-GGUF
|
||||
T1=TRELLIS-image-large-GGUF
|
||||
DINO=dinov3-vitl16-pretrain-lvd1689m-GGUF
|
||||
|
||||
T2_FILES=(ss_flow_f16.gguf slat_flow_f16.gguf slat_flow_1024_f16.gguf shape_dec_f16.gguf \
|
||||
shape_enc_f16.gguf tex_dec_f16.gguf tex_slat_flow_512_f16.gguf tex_slat_flow_1024_f16.gguf)
|
||||
T1_FILES=(ss_dec_f16.gguf)
|
||||
DINO_FILES=(dino_f16.gguf)
|
||||
|
||||
publish() { # publish <repo> <file>...
|
||||
local repo="$1"; shift
|
||||
local id="$ORG/$repo"
|
||||
echo "== $id =="
|
||||
"$HF" repo create "$id" --repo-type model --exist-ok
|
||||
local f
|
||||
for f in "$@"; do
|
||||
[ -s "$GGUFS/$f" ] || { echo "error: missing $GGUFS/$f" >&2; exit 1; }
|
||||
echo "-- upload $f"
|
||||
"$HF" upload "$id" "$GGUFS/$f" "$f" --commit-message "Add $f"
|
||||
done
|
||||
"$HF" upload "$id" "$CARDS/$repo/README.md" README.md --commit-message "Add model card"
|
||||
"$HF" upload "$id" "$CARDS/$repo/LICENSE" LICENSE --commit-message "Add license"
|
||||
}
|
||||
|
||||
publish "$T2" "${T2_FILES[@]}"
|
||||
publish "$T1" "${T1_FILES[@]}"
|
||||
publish "$DINO" "${DINO_FILES[@]}"
|
||||
|
||||
echo "done. repos:"
|
||||
echo " https://huggingface.co/$ORG/$T2"
|
||||
echo " https://huggingface.co/$ORG/$T1"
|
||||
echo " https://huggingface.co/$ORG/$DINO"
|
||||
Reference in New Issue
Block a user