Initial release
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Move every bare `<name>.safetensors` in a folder into `<name>/` and rename
|
||||
* it to the canonical `adapter_model.safetensors`:
|
||||
*
|
||||
* dit-xl-base-turbo/abba.safetensors -> dit-xl-base-turbo/abba/adapter_model.safetensors
|
||||
*
|
||||
* The filename stops carrying the trigger word — that now lives in the file's
|
||||
* own __metadata__ (hot_step_trigger, stamped 2026-07-28) — so the name's only
|
||||
* remaining job is labelling the folder. Same-volume renames, no data copied,
|
||||
* safe to re-run. DRY RUN by default.
|
||||
*
|
||||
* node server/scripts/enfolder-bare-adapters.mjs --folder M:\...\dit-xl-base-turbo
|
||||
* node server/scripts/enfolder-bare-adapters.mjs --folder ... --apply
|
||||
*
|
||||
* NOTE deliberately does NOT write an adapter_config.json: these are
|
||||
* LyCORIS-format adapters whose alphas live per-tensor in the file itself; a
|
||||
* fabricated PEFT config would mis-scale them.
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const APPLY = argv.includes('--apply');
|
||||
const fi = argv.indexOf('--folder');
|
||||
const FOLDER = fi >= 0 && fi + 1 < argv.length ? path.resolve(argv[fi + 1]) : '';
|
||||
|
||||
if (!FOLDER || !fs.existsSync(FOLDER) || !fs.statSync(FOLDER).isDirectory()) {
|
||||
console.error('usage: enfolder-bare-adapters.mjs --folder <dir> [--apply]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const moves = [];
|
||||
const warns = [];
|
||||
for (const e of fs.readdirSync(FOLDER, { withFileTypes: true })) {
|
||||
if (!e.isFile() || !e.name.endsWith('.safetensors') || e.name.startsWith('.')) continue;
|
||||
const stem = e.name.replace(/\.safetensors$/i, '');
|
||||
if (stem === 'adapter_model') { warns.push(`${e.name}: already canonical at top level — left alone`); continue; }
|
||||
const destDir = path.join(FOLDER, stem);
|
||||
const dest = path.join(destDir, 'adapter_model.safetensors');
|
||||
if (fs.existsSync(dest)) { warns.push(`${stem}: ${path.basename(destDir)}/adapter_model.safetensors already exists — skipped`); continue; }
|
||||
moves.push({ from: path.join(FOLDER, e.name), destDir, dest, stem });
|
||||
}
|
||||
|
||||
console.log(`\nFolder: ${FOLDER} Mode: ${APPLY ? 'APPLY' : 'DRY RUN'}`);
|
||||
for (const m of moves.slice(0, 8)) console.log(` ${m.stem}.safetensors -> ${m.stem}/adapter_model.safetensors`);
|
||||
if (moves.length > 8) console.log(` … and ${moves.length - 8} more, same shape`);
|
||||
for (const w of warns) console.log(`WARN ${w}`);
|
||||
console.log(`${moves.length} move(s), ${warns.length} warning(s).`);
|
||||
|
||||
if (!APPLY) { console.log('DRY RUN — nothing moved. Re-run with --apply.\n'); process.exit(0); }
|
||||
|
||||
let ok = 0, failed = 0;
|
||||
for (const m of moves) {
|
||||
try {
|
||||
fs.mkdirSync(m.destDir, { recursive: true });
|
||||
fs.renameSync(m.from, m.dest);
|
||||
ok++;
|
||||
} catch (err) {
|
||||
console.error(`FAILED ${m.stem}: ${err.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
console.log(`Moved ${ok}${failed ? `, ${failed} FAILED` : ''}.`);
|
||||
process.exit(failed ? 1 : 0);
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Migrate an adapter corpus to the per-base layout (2026-07-28):
|
||||
*
|
||||
* lm/<artist>-0.6B -> lm-06b/<artist>
|
||||
* lm/<artist>-1.7B -> lm-17b/<artist>
|
||||
* lm/<artist>-4B -> lm-4b/<artist>
|
||||
* <root>/<dit-dir> -> dit-<shorthand>/<dit-dir> (base read from its own
|
||||
* dit_train_log.json)
|
||||
*
|
||||
* Everything is a same-volume rename — no data is copied, so this is fast and
|
||||
* safe to re-run (already-migrated entries are skipped). DRY RUN by default.
|
||||
*
|
||||
* node server/scripts/migrate-adapter-layout.mjs --adapters M:\HOT-Step-CPP\Adapters
|
||||
* node server/scripts/migrate-adapter-layout.mjs --adapters ... --apply
|
||||
*
|
||||
* --adapters <dir> adapters root (default: ../../adapters from this file)
|
||||
* --default-dit <s> shorthand folder for a DiT adapter dir with no readable
|
||||
* dit_train_log.json (default: leave in place + warn)
|
||||
* --apply actually move things
|
||||
*
|
||||
* Companion of server/src/services/training/adapterLayout.ts — the shorthand
|
||||
* map here must stay in sync with DIT_SHORTHANDS there.
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (n) => argv.includes(`--${n}`);
|
||||
const opt = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 && i + 1 < argv.length ? argv[i + 1] : d; };
|
||||
|
||||
const APPLY = flag('apply');
|
||||
const ROOT = path.resolve(opt('adapters', path.join(HERE, '..', '..', 'adapters')));
|
||||
const DEFAULT_DIT = opt('default-dit', '');
|
||||
|
||||
// Mirror of adapterLayout.ts (keep in sync).
|
||||
const LM_DIRS = { '0.6B': 'lm-06b', '1.7B': 'lm-17b', '4B': 'lm-4b' };
|
||||
const DIT_SHORTHANDS = {
|
||||
'acestep-v15-merge-base-sft-turbo-xl-thirds': 'dit-xl-thirds',
|
||||
'acestep-v15-merge-base-turbo-xl-ta-0.5': 'dit-xl-base-turbo',
|
||||
'acestep-v15-xl-sftturbo50': 'dit-xl-sft-turbo',
|
||||
'acestep-v15-merge-base-sft-xl-ta-0.5': 'dit-xl-base-sft',
|
||||
'acestep-v15-merge-sft-turbo-xl-ta-0.3': 'dit-xl-sft-turbo-ta03',
|
||||
'acestep-v15-merge-sft-turbo-xl-ta-0.7': 'dit-xl-sft-turbo-ta07',
|
||||
'acestep-v15-xl-base': 'dit-xl-base',
|
||||
'acestep-v15-xl-sft': 'dit-xl-sft',
|
||||
'acestep-v15-xl-turbo': 'dit-xl-turbo',
|
||||
'acestep-v15-base': 'dit-base',
|
||||
'acestep-v15-sft': 'dit-sft',
|
||||
'acestep-v15-turbo': 'dit-turbo',
|
||||
'acestep-v15-sftturbo50': 'dit-sft-turbo',
|
||||
'acestep-v15-turbo-continuous': 'dit-turbo-continuous',
|
||||
'acestep-v15-turbo-shift1': 'dit-turbo-shift1',
|
||||
'acestep-v15-turbo-shift3': 'dit-turbo-shift3',
|
||||
'acestep-v15-merge-base-sft-turbo-xl-thirds-convrot-ref': 'dit-xl-thirds-convrot',
|
||||
'sa3-dit': 'dit-sa3',
|
||||
};
|
||||
const QUANT_RE = /-(BF16|F16|F32|MXFP4|NVFP4|IQ\d[A-Z_]*|Q\d[\w]*)$/i;
|
||||
|
||||
function ditShorthand(model) {
|
||||
let stem = path.basename(String(model || '')).replace(/\.gguf$/i, '').replace(QUANT_RE, '');
|
||||
if (!stem) return '';
|
||||
if (DIT_SHORTHANDS[stem]) return DIT_SHORTHANDS[stem];
|
||||
const fb = stem.replace(/^acestep-v15-/i, '').replace(/\./g, '');
|
||||
return fb ? `dit-${fb}` : '';
|
||||
}
|
||||
|
||||
if (!fs.existsSync(ROOT)) { console.error(`adapters root not found: ${ROOT}`); process.exit(2); }
|
||||
|
||||
const moves = []; // { from, to, note }
|
||||
const warns = [];
|
||||
|
||||
// ── planner-LM adapters ─────────────────────────────────────────────────────
|
||||
const lmRoot = path.join(ROOT, 'lm');
|
||||
if (fs.existsSync(lmRoot)) {
|
||||
for (const e of fs.readdirSync(lmRoot, { withFileTypes: true })) {
|
||||
if (e.name.startsWith('.')) continue;
|
||||
const from = path.join(lmRoot, e.name);
|
||||
const stem = e.isFile() ? e.name.replace(/\.safetensors$/i, '') : e.name;
|
||||
if (e.isFile() && !e.name.endsWith('.safetensors')) { warns.push(`lm/${e.name}: not an adapter — left in place`); continue; }
|
||||
const m = /-(0\.6B|1\.7B|4B)$/i.exec(stem);
|
||||
if (!m) {
|
||||
if (e.isDirectory() && fs.readdirSync(from).length === 0) {
|
||||
moves.push({ from, to: '', note: 'empty dir — remove' });
|
||||
} else {
|
||||
warns.push(`lm/${e.name}: no -<size> suffix — cannot tell the base, left in place`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const size = ['0.6B', '1.7B', '4B'].find(s => s.toLowerCase() === m[1].toLowerCase());
|
||||
const bare = stem.slice(0, -(m[1].length + 1)) + (e.isFile() ? '.safetensors' : '');
|
||||
moves.push({ from, to: path.join(ROOT, LM_DIRS[size], bare), note: `lm ${size}` });
|
||||
}
|
||||
}
|
||||
|
||||
// ── already-migrated lm-<size> dirs: strip legacy -<size> suffixes ──────────
|
||||
// A hand-moved corpus (rename lm -> lm-4b) still carries the suffix on every
|
||||
// child; the parent folder now says the size, so the suffix goes. A child whose
|
||||
// suffix DISAGREES with its folder is moved to the right size folder instead.
|
||||
for (const [size, dirName] of Object.entries(LM_DIRS)) {
|
||||
const sizeRoot = path.join(ROOT, dirName);
|
||||
if (!fs.existsSync(sizeRoot)) continue;
|
||||
for (const e of fs.readdirSync(sizeRoot, { withFileTypes: true })) {
|
||||
if (e.name.startsWith('.')) continue;
|
||||
const from = path.join(sizeRoot, e.name);
|
||||
const stem = e.isFile() ? e.name.replace(/\.safetensors$/i, '') : e.name;
|
||||
if (e.isFile() && !e.name.endsWith('.safetensors')) continue;
|
||||
const m = /-(0\.6B|1\.7B|4B)$/i.exec(stem);
|
||||
if (!m) {
|
||||
if (e.isDirectory() && fs.readdirSync(from).length === 0) {
|
||||
moves.push({ from, to: '', note: 'empty dir — remove' });
|
||||
}
|
||||
continue; // already unsuffixed — nothing to do
|
||||
}
|
||||
const suffixSize = ['0.6B', '1.7B', '4B'].find(s => s.toLowerCase() === m[1].toLowerCase());
|
||||
const bare = stem.slice(0, -(m[1].length + 1)) + (e.isFile() ? '.safetensors' : '');
|
||||
const targetDir = suffixSize === size ? sizeRoot : path.join(ROOT, LM_DIRS[suffixSize]);
|
||||
moves.push({
|
||||
from, to: path.join(targetDir, bare),
|
||||
note: suffixSize === size ? 'strip suffix' : `RELOCATE — suffix says ${suffixSize}, folder says ${size}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── canonicalise PEFT weight filenames ──────────────────────────────────────
|
||||
// MUST run (and execute) before the relocation pass below: a dir can need both
|
||||
// a weights-rename and a relocation, and the rename addresses the old path.
|
||||
// A dir with adapter_config.json whose weights were renamed <name>.safetensors
|
||||
// (the old make-the-filename-the-trigger workaround) goes back to the
|
||||
// adapter_model.safetensors the scanners and loaders look for — the embedded
|
||||
// trigger has made the rename pointless.
|
||||
for (const e of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (!e.isDirectory() || !/^(dit-|lm-)/i.test(e.name)) continue;
|
||||
for (const sub of fs.readdirSync(path.join(ROOT, e.name), { withFileTypes: true })) {
|
||||
if (!sub.isDirectory() || sub.name.startsWith('.')) continue;
|
||||
const dir = path.join(ROOT, e.name, sub.name);
|
||||
if (!fs.existsSync(path.join(dir, 'adapter_config.json'))) continue;
|
||||
if (fs.existsSync(path.join(dir, 'adapter_model.safetensors'))) continue;
|
||||
const st = fs.readdirSync(dir).filter(f => f.endsWith('.safetensors'));
|
||||
if (st.length === 1) {
|
||||
moves.push({ from: path.join(dir, st[0]), to: path.join(dir, 'adapter_model.safetensors'),
|
||||
note: 'canonicalise weights filename' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── dit-* dirs: verify each PEFT child sits under its own base ──────────────
|
||||
// The folder is a claim about the training base; the adapter's own
|
||||
// dit_train_log.json is the evidence. Disagreement = move to the right folder.
|
||||
// Bare .safetensors files record no base and are left where the user put them.
|
||||
for (const e of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (!e.isDirectory() || !/^dit-/i.test(e.name)) continue;
|
||||
const shorthandDir = path.join(ROOT, e.name);
|
||||
for (const sub of fs.readdirSync(shorthandDir, { withFileTypes: true })) {
|
||||
if (!sub.isDirectory() || sub.name.startsWith('.')) continue;
|
||||
const from = path.join(shorthandDir, sub.name);
|
||||
let base = '';
|
||||
try {
|
||||
const log = JSON.parse(fs.readFileSync(path.join(from, 'dit_train_log.json'), 'utf8'));
|
||||
base = log?.config?.dit_name || log?.config?.dit_path || '';
|
||||
} catch { continue; } // no log — nothing to verify against
|
||||
const want = base ? ditShorthand(base) : '';
|
||||
if (want && want.toLowerCase() !== e.name.toLowerCase()) {
|
||||
moves.push({ from, to: path.join(ROOT, want, sub.name), note: `RELOCATE — trained on ${base}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── DiT adapters at the root ────────────────────────────────────────────────
|
||||
for (const e of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (e.name.startsWith('.') || e.name === 'lm') continue;
|
||||
if (/^(lm-06b|lm-17b|lm-4b|dit-)/i.test(e.name)) continue; // already migrated trees
|
||||
const from = path.join(ROOT, e.name);
|
||||
|
||||
if (e.isDirectory()) {
|
||||
const looksAdapter = fs.existsSync(path.join(from, 'adapter_config.json')) ||
|
||||
fs.readdirSync(from).some(f => f.endsWith('.safetensors'));
|
||||
if (!looksAdapter) { warns.push(`${e.name}: not an adapter dir — left in place`); continue; }
|
||||
let base = '';
|
||||
try {
|
||||
const log = JSON.parse(fs.readFileSync(path.join(from, 'dit_train_log.json'), 'utf8'));
|
||||
base = log?.config?.dit_name || log?.config?.dit_path || '';
|
||||
} catch { /* no log — fall through */ }
|
||||
const shorthand = base ? ditShorthand(base) : DEFAULT_DIT;
|
||||
if (!shorthand) { warns.push(`${e.name}: no dit_train_log.json and no --default-dit — left in place`); continue; }
|
||||
moves.push({ from, to: path.join(ROOT, shorthand, e.name), note: base ? `dit (${base})` : 'dit (--default-dit)' });
|
||||
} else if (e.isFile() && e.name.endsWith('.safetensors')) {
|
||||
// A bare DiT adapter at the root records no base anywhere — only move it
|
||||
// when the user says where it belongs.
|
||||
if (DEFAULT_DIT) moves.push({ from, to: path.join(ROOT, DEFAULT_DIT, e.name), note: 'dit bare (--default-dit)' });
|
||||
else warns.push(`${e.name}: bare file with no recorded base — left in place (use --default-dit to move)`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── report ──────────────────────────────────────────────────────────────────
|
||||
console.log(`\nAdapters root: ${ROOT} Mode: ${APPLY ? 'APPLY' : 'DRY RUN'}\n`);
|
||||
const pad = (s, n) => String(s).padEnd(n);
|
||||
for (const mv of moves) {
|
||||
const rel = (p) => p ? path.relative(ROOT, p) : '(delete)';
|
||||
console.log(`${pad(rel(mv.from), 46)} -> ${pad(rel(mv.to), 40)} ${mv.note}`);
|
||||
}
|
||||
for (const w of warns) console.log(`WARN ${w}`);
|
||||
console.log(`\n${moves.length} move(s), ${warns.length} warning(s).`);
|
||||
|
||||
if (!APPLY) { console.log('DRY RUN — nothing moved. Re-run with --apply.\n'); process.exit(0); }
|
||||
|
||||
let ok = 0, failed = 0;
|
||||
for (const mv of moves) {
|
||||
try {
|
||||
if (!mv.to) { fs.rmdirSync(mv.from); ok++; continue; }
|
||||
if (fs.existsSync(mv.to)) throw new Error(`target exists: ${mv.to}`);
|
||||
fs.mkdirSync(path.dirname(mv.to), { recursive: true });
|
||||
fs.renameSync(mv.from, mv.to);
|
||||
ok++;
|
||||
} catch (e) {
|
||||
console.error(`FAILED ${mv.from}: ${e.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
// Remove the legacy lm/ root if the migration emptied it.
|
||||
try {
|
||||
if (fs.existsSync(lmRoot) && fs.readdirSync(lmRoot).length === 0) { fs.rmdirSync(lmRoot); console.log('removed empty lm/'); }
|
||||
} catch { /* leave it */ }
|
||||
console.log(`\nMoved ${ok}${failed ? `, ${failed} FAILED` : ''}.`);
|
||||
process.exit(failed ? 1 : 0);
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Repoint every album preset's adapter paths at the per-base adapter layout
|
||||
* (2026-07-28). The artist stem is taken from the currently stored path's
|
||||
* filename and looked up in the new hierarchy:
|
||||
*
|
||||
* adapter_path -> <adapters>/dit-xl-thirds/<stem> when trained there
|
||||
* <adapters>/dit-xl-base-turbo/<stem> otherwise (archive)
|
||||
* lm_adapter_path -> <adapters>/lm-4b/<stem>
|
||||
*
|
||||
* A stem with no folder in the new layout is left untouched and warned about.
|
||||
* Rows already pointing inside <adapters> keep their setting (hand-fixed).
|
||||
* The DB is backed up via SQLite's online backup API before any write.
|
||||
* DRY RUN by default.
|
||||
*
|
||||
* node server/scripts/retarget-album-presets.mjs
|
||||
* node server/scripts/retarget-album-presets.mjs --apply
|
||||
*
|
||||
* --adapters <dir> adapters root (default M:\HOT-Step-CPP\Adapters)
|
||||
* --db <file> SQLite db (default server/data/hotstep.db)
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createRequire } from 'module';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const require = createRequire(path.join(HERE, '..', 'package.json'));
|
||||
const Database = require('better-sqlite3');
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const APPLY = argv.includes('--apply');
|
||||
const opt = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 && i + 1 < argv.length ? argv[i + 1] : d; };
|
||||
const ROOT = path.resolve(opt('adapters', 'M:\\HOT-Step-CPP\\Adapters'));
|
||||
const DB_PATH = path.resolve(opt('db', path.join(HERE, '..', 'data', 'hotstep.db')));
|
||||
|
||||
const stemOf = (p) => {
|
||||
if (!p) return '';
|
||||
const base = String(p).split(/[\\/]/).filter(Boolean).pop() || '';
|
||||
// Bare file, canonical weights file (stem = its folder), or already a folder.
|
||||
if (/^adapter_model\.safetensors$/i.test(base)) {
|
||||
const parts = String(p).split(/[\\/]/).filter(Boolean);
|
||||
return parts[parts.length - 2] || '';
|
||||
}
|
||||
return base.replace(/\.safetensors$/i, '').replace(/-(0\.6B|1\.7B|4B)$/i, '');
|
||||
};
|
||||
|
||||
const hasAdapter = (dir) => fs.existsSync(path.join(dir, 'adapter_model.safetensors'));
|
||||
|
||||
const ditTarget = (stem) => {
|
||||
for (const basedir of ['dit-xl-thirds', 'dit-xl-base-turbo']) {
|
||||
const d = path.join(ROOT, basedir, stem);
|
||||
if (hasAdapter(d)) return d;
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const lmTarget = (stem) => {
|
||||
const d = path.join(ROOT, 'lm-4b', stem);
|
||||
return hasAdapter(d) ? d : '';
|
||||
};
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
const rows = db.prepare(
|
||||
`SELECT ap.lyrics_set_id AS id, ls.album, ap.adapter_path, ap.lm_adapter_path
|
||||
FROM album_presets ap LEFT JOIN lyrics_sets ls ON ls.id = ap.lyrics_set_id`).all();
|
||||
|
||||
const updates = [];
|
||||
const warns = [];
|
||||
for (const r of rows) {
|
||||
const upd = { id: r.id, album: r.album || `(set ${r.id})` };
|
||||
|
||||
for (const [field, target] of [['adapter_path', ditTarget], ['lm_adapter_path', lmTarget]]) {
|
||||
const cur = r[field] || '';
|
||||
if (!cur) continue;
|
||||
if (path.resolve(cur).toLowerCase().startsWith(ROOT.toLowerCase())) continue; // hand-fixed already
|
||||
const stem = stemOf(cur);
|
||||
const dest = stem ? target(stem) : '';
|
||||
if (dest) upd[field] = dest;
|
||||
else warns.push(`${upd.album}: ${field} stem "${stem}" has no folder in the new layout — left as ${cur}`);
|
||||
}
|
||||
if (upd.adapter_path || upd.lm_adapter_path) updates.push(upd);
|
||||
}
|
||||
|
||||
console.log(`\nDB: ${DB_PATH}\nAdapters: ${ROOT}\nMode: ${APPLY ? 'APPLY' : 'DRY RUN'}\n`);
|
||||
console.log(`${rows.length} preset(s), ${updates.length} to update, ${warns.length} warning(s).\n`);
|
||||
for (const u of updates.slice(0, 6)) {
|
||||
console.log(`${u.album}`);
|
||||
if (u.adapter_path) console.log(` dit -> ${path.relative(ROOT, u.adapter_path)}`);
|
||||
if (u.lm_adapter_path) console.log(` lm -> ${path.relative(ROOT, u.lm_adapter_path)}`);
|
||||
}
|
||||
if (updates.length > 6) console.log(`… and ${updates.length - 6} more, same shape`);
|
||||
for (const w of warns) console.log(`WARN ${w}`);
|
||||
|
||||
if (!APPLY) { console.log('\nDRY RUN — nothing written. Re-run with --apply.\n'); db.close(); process.exit(0); }
|
||||
|
||||
// Online backup first — safe against the running server's open handle.
|
||||
const stampNow = new Date().toISOString().replace(/[:T]/g, '-').slice(0, 19);
|
||||
const bak = DB_PATH.replace(/\.db$/i, `_backup_presets_${stampNow}.db`);
|
||||
await db.backup(bak);
|
||||
console.log(`\nBacked up to ${path.basename(bak)}`);
|
||||
|
||||
const setBoth = db.prepare('UPDATE album_presets SET adapter_path = ?, lm_adapter_path = ? WHERE lyrics_set_id = ?');
|
||||
const cur = db.prepare('SELECT adapter_path, lm_adapter_path FROM album_presets WHERE lyrics_set_id = ?');
|
||||
let n = 0;
|
||||
for (const u of updates) {
|
||||
const c = cur.get(u.id);
|
||||
setBoth.run(u.adapter_path ?? c.adapter_path, u.lm_adapter_path ?? c.lm_adapter_path, u.id);
|
||||
n++;
|
||||
}
|
||||
db.close();
|
||||
console.log(`Updated ${n} preset(s).`);
|
||||
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Stamp trigger words into adapters that were trained before HOT-Step embedded
|
||||
* them (every Side-Step-trained adapter, and every HOT-Step one from before
|
||||
* 2026-07-28).
|
||||
*
|
||||
* docs/plans/2026-07-28-adapter-trigger-embedding.md §6
|
||||
*
|
||||
* Adds `hot_step_trigger`, `hot_step_trigger_position` and
|
||||
* `modelspec.trigger_phrase` to a safetensors file's `__metadata__`. Tensor
|
||||
* bytes are copied verbatim, so the weights are provably unchanged — only the
|
||||
* JSON header grows. Unknown metadata keys are ignored by every other consumer,
|
||||
* so a stamped adapter still loads in ComfyUI / PEFT / Side-Step.
|
||||
*
|
||||
* DRY RUN BY DEFAULT. Nothing is written without --apply.
|
||||
*
|
||||
* node server/scripts/stamp-adapter-triggers.mjs
|
||||
* node server/scripts/stamp-adapter-triggers.mjs --apply
|
||||
*
|
||||
* --adapters <dir> adapters root (default: ../../adapters from this file)
|
||||
* --datasets <dir> a Side-Step corpus root; each <dir>/<name>/dataset.json
|
||||
* supplies that adapter's REAL custom_tag + tag_position,
|
||||
* which beats guessing from the filename
|
||||
* --map <file.json> {"<adapterName>": "<trigger>"} explicit overrides
|
||||
* --position <p> prepend|append fallback when no dataset says otherwise
|
||||
* --only <lm|dit|all> which subtree to walk (default all)
|
||||
* --no-backup skip the .bak copy (not recommended)
|
||||
* --keep-backup keep every .bak instead of releasing it after verify
|
||||
* --force restamp adapters that already carry a trigger
|
||||
*/
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const MAX_HEADER_BYTES = 64 * 1024 * 1024;
|
||||
const SIZE_SUFFIX = /-(?:0\.6B|1\.7B|4B)$/;
|
||||
|
||||
// ── args ────────────────────────────────────────────────────────────────────
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name) => argv.includes(`--${name}`);
|
||||
const opt = (name, dflt) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : dflt;
|
||||
};
|
||||
|
||||
const APPLY = flag('apply');
|
||||
const FORCE = flag('force');
|
||||
const BACKUP = !flag('no-backup');
|
||||
// A .bak is taken per file and released as soon as that file verifies, so peak
|
||||
// extra disk is one adapter rather than a second copy of the whole corpus.
|
||||
// --keep-backup retains them all instead.
|
||||
const KEEP_BACKUP = flag('keep-backup');
|
||||
const POSITION = opt('position', 'prepend');
|
||||
const ONLY = opt('only', 'all');
|
||||
const ADAPTERS_ROOT = path.resolve(opt('adapters', path.join(HERE, '..', '..', 'adapters')));
|
||||
const DATASETS_DIR = opt('datasets', '');
|
||||
const MAP_FILE = opt('map', '');
|
||||
|
||||
if (POSITION !== 'prepend' && POSITION !== 'append') {
|
||||
console.error(`--position must be prepend|append (got "${POSITION}")`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
let explicitMap = {};
|
||||
if (MAP_FILE) {
|
||||
try {
|
||||
explicitMap = JSON.parse(fs.readFileSync(path.resolve(MAP_FILE), 'utf8'));
|
||||
} catch (e) {
|
||||
console.error(`cannot read --map ${MAP_FILE}: ${e.message}`);
|
||||
process.exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
// ── safetensors header I/O ──────────────────────────────────────────────────
|
||||
|
||||
function readHeader(file) {
|
||||
const fd = fs.openSync(file, 'r');
|
||||
try {
|
||||
const lenBuf = Buffer.allocUnsafe(8);
|
||||
if (fs.readSync(fd, lenBuf, 0, 8, 0) !== 8) return null;
|
||||
const headerLen = Number(lenBuf.readBigUInt64LE(0));
|
||||
if (!Number.isSafeInteger(headerLen) || headerLen <= 0 || headerLen > MAX_HEADER_BYTES) return null;
|
||||
const hdr = Buffer.allocUnsafe(headerLen);
|
||||
if (fs.readSync(fd, hdr, 0, headerLen, 8) !== headerLen) return null;
|
||||
return { json: JSON.parse(hdr.toString('utf8')), headerLen };
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite `file` with `metadata` merged into its `__metadata__`.
|
||||
*
|
||||
* Streams `newLen(8) + newHeader + originalPayload` to a temp file, then swaps.
|
||||
* Never edits in place, so an interrupted run leaves the original intact.
|
||||
*/
|
||||
function stamp(file, metadata) {
|
||||
const head = readHeader(file);
|
||||
if (!head) throw new Error('unreadable safetensors header');
|
||||
|
||||
const json = { ...head.json, __metadata__: { ...(head.json.__metadata__ || {}), ...metadata } };
|
||||
const newHeader = Buffer.from(JSON.stringify(json), 'utf8');
|
||||
// safetensors requires the header to be 8-byte aligned; pad with spaces,
|
||||
// which are legal JSON whitespace outside string literals.
|
||||
const pad = (8 - (newHeader.length % 8)) % 8;
|
||||
const padded = Buffer.concat([newHeader, Buffer.alloc(pad, 0x20)]);
|
||||
const lenBuf = Buffer.allocUnsafe(8);
|
||||
lenBuf.writeBigUInt64LE(BigInt(padded.length), 0);
|
||||
|
||||
const originalSize = fs.statSync(file).size;
|
||||
const payloadBytes = originalSize - 8 - head.headerLen;
|
||||
const expectedSize = 8 + padded.length + payloadBytes;
|
||||
const originalNames = Object.keys(head.json).filter(k => k !== '__metadata__').sort().join('\n');
|
||||
|
||||
const tmp = `${file}.stamping`;
|
||||
const out = fs.openSync(tmp, 'w');
|
||||
const src = fs.openSync(file, 'r');
|
||||
try {
|
||||
fs.writeSync(out, lenBuf);
|
||||
fs.writeSync(out, padded);
|
||||
const buf = Buffer.allocUnsafe(4 * 1024 * 1024);
|
||||
let pos = 8 + head.headerLen;
|
||||
for (;;) {
|
||||
const n = fs.readSync(src, buf, 0, buf.length, pos);
|
||||
if (n <= 0) break;
|
||||
fs.writeSync(out, buf, 0, n);
|
||||
pos += n;
|
||||
}
|
||||
} finally {
|
||||
fs.closeSync(src);
|
||||
fs.closeSync(out);
|
||||
}
|
||||
|
||||
const bak = `${file}.bak`;
|
||||
if (BACKUP) fs.copyFileSync(file, bak);
|
||||
fs.rmSync(file);
|
||||
fs.renameSync(tmp, file);
|
||||
|
||||
// Verify the swapped file before moving on. The payload is a verbatim byte
|
||||
// copy, so an exact size match plus an identical tensor-entry set is enough
|
||||
// to catch truncation, a short write or a mangled header — the only ways this
|
||||
// operation can go wrong. On failure, put the original back.
|
||||
try {
|
||||
const size = fs.statSync(file).size;
|
||||
const check = readHeader(file);
|
||||
if (size !== expectedSize) throw new Error(`size ${size} != expected ${expectedSize}`);
|
||||
if (!check) throw new Error('rewritten header does not parse');
|
||||
if (check.json.__metadata__?.hot_step_trigger !== metadata.hot_step_trigger) {
|
||||
throw new Error('trigger missing from the rewritten header');
|
||||
}
|
||||
const names = Object.keys(check.json).filter(k => k !== '__metadata__').sort().join('\n');
|
||||
if (names !== originalNames) throw new Error('tensor entry set changed');
|
||||
} catch (e) {
|
||||
if (BACKUP && fs.existsSync(bak)) {
|
||||
fs.rmSync(file, { force: true });
|
||||
fs.renameSync(bak, file);
|
||||
throw new Error(`${e.message} — ORIGINAL RESTORED from .bak`);
|
||||
}
|
||||
throw new Error(`${e.message} — NO BACKUP TO RESTORE (ran with --no-backup)`);
|
||||
}
|
||||
|
||||
if (BACKUP && !KEEP_BACKUP) fs.rmSync(bak, { force: true });
|
||||
}
|
||||
|
||||
// ── discovery ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** Every adapter under `root`: PEFT dirs (unversioned, and each stamped run
|
||||
* subfolder — per-run layout) plus bare .safetensors files. `name` stays the
|
||||
* ARTIST name for runs, which is what the trigger proposal keys on. */
|
||||
function findAdapters(root, kind) {
|
||||
const out = [];
|
||||
if (!fs.existsSync(root)) return out;
|
||||
for (const e of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
if (e.name.startsWith('.')) continue;
|
||||
const full = path.join(root, e.name);
|
||||
if (e.isDirectory()) {
|
||||
const model = path.join(full, 'adapter_model.safetensors');
|
||||
if (fs.existsSync(model)) out.push({ name: e.name, file: model, kind });
|
||||
for (const run of fs.readdirSync(full, { withFileTypes: true })) {
|
||||
if (!run.isDirectory() || run.name.startsWith('.') || run.name === 'milestones') continue;
|
||||
const runModel = path.join(full, run.name, 'adapter_model.safetensors');
|
||||
if (fs.existsSync(runModel)) out.push({ name: e.name, file: runModel, kind });
|
||||
}
|
||||
} else if (e.isFile() && e.name.endsWith('.safetensors')) {
|
||||
out.push({ name: e.name.replace(/\.safetensors$/i, ''), file: full, kind });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* `custom_tag` + `tag_position` per dataset name — the ground truth for what an
|
||||
* adapter was actually trained with. Reads both the Training Studio's own
|
||||
* `dataset_meta.json` and any Side-Step corpus passed via --datasets, whose
|
||||
* `dataset.json` carries the same fields under `metadata`.
|
||||
*/
|
||||
function loadDatasetTags() {
|
||||
const tags = new Map();
|
||||
|
||||
const addStudio = (dsRoot) => {
|
||||
if (!fs.existsSync(dsRoot)) return;
|
||||
for (const e of fs.readdirSync(dsRoot, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
try {
|
||||
const meta = JSON.parse(fs.readFileSync(path.join(dsRoot, e.name, 'dataset_meta.json'), 'utf8'));
|
||||
const tag = (meta.customTag || meta.custom_tag || '').trim();
|
||||
const pos = (meta.tagPosition || meta.tag_position || '').trim();
|
||||
if (tag) tags.set(e.name, { tag, pos });
|
||||
} catch { /* a dataset without readable meta simply contributes nothing */ }
|
||||
}
|
||||
};
|
||||
|
||||
const addSideStep = (root) => {
|
||||
if (!root || !fs.existsSync(root)) return;
|
||||
for (const e of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
try {
|
||||
const j = JSON.parse(fs.readFileSync(path.join(root, e.name, 'dataset.json'), 'utf8'));
|
||||
const md = j.metadata || {};
|
||||
const tag = (md.custom_tag || '').trim();
|
||||
const pos = (md.tag_position || '').trim();
|
||||
if (tag) tags.set(e.name, { tag, pos });
|
||||
} catch { /* likewise */ }
|
||||
}
|
||||
};
|
||||
|
||||
addStudio(path.join(HERE, '..', 'data', 'training', 'datasets'));
|
||||
if (DATASETS_DIR) addSideStep(path.resolve(DATASETS_DIR));
|
||||
return tags;
|
||||
}
|
||||
|
||||
/** explicit map > dataset custom_tag > dir name minus the -<size> suffix. */
|
||||
function proposeTrigger(name, datasetTags) {
|
||||
if (explicitMap[name]) return { trigger: String(explicitMap[name]).trim(), position: POSITION, source: 'map' };
|
||||
const bare = name.replace(SIZE_SUFFIX, '');
|
||||
const hit = datasetTags.get(name) || datasetTags.get(bare);
|
||||
if (hit) {
|
||||
// A dataset whose tag_position is "replace" never put the tag in its
|
||||
// captions at all (preprocess-run.h:203), so that adapter has no trigger.
|
||||
if (hit.pos === 'replace') return { trigger: '', position: '', source: 'replace' };
|
||||
return { trigger: hit.tag, position: hit.pos === 'append' ? 'append' : 'prepend', source: 'dataset' };
|
||||
}
|
||||
return { trigger: bare, position: POSITION, source: 'filename' };
|
||||
}
|
||||
|
||||
// ── run ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
const datasetTags = loadDatasetTags();
|
||||
const targets = [];
|
||||
// Per-base layout (adapterLayout.ts): planner adapters under lm/ (legacy) and
|
||||
// lm-06b/lm-17b/lm-4b; DiT adapters at the root (legacy) and under dit-*.
|
||||
if (ONLY === 'all' || ONLY === 'dit') {
|
||||
// Root level: LEGACY bare files / PEFT dirs only. The lm*/dit-* trees are
|
||||
// walked by their own passes — descending into them here double-counts.
|
||||
for (const e of fs.existsSync(ADAPTERS_ROOT) ? fs.readdirSync(ADAPTERS_ROOT, { withFileTypes: true }) : []) {
|
||||
if (e.name.startsWith('.') || /^(lm|lm-|dit-)/i.test(e.name)) continue;
|
||||
const full = path.join(ADAPTERS_ROOT, e.name);
|
||||
if (e.isDirectory()) {
|
||||
const model = path.join(full, 'adapter_model.safetensors');
|
||||
if (fs.existsSync(model)) targets.push({ name: e.name, file: model, kind: 'dit' });
|
||||
} else if (e.isFile() && e.name.endsWith('.safetensors')) {
|
||||
targets.push({ name: e.name.replace(/\.safetensors$/i, ''), file: full, kind: 'dit' });
|
||||
}
|
||||
}
|
||||
for (const e of fs.existsSync(ADAPTERS_ROOT) ? fs.readdirSync(ADAPTERS_ROOT, { withFileTypes: true }) : []) {
|
||||
if (e.isDirectory() && /^dit-/i.test(e.name)) {
|
||||
targets.push(...findAdapters(path.join(ADAPTERS_ROOT, e.name), 'dit'));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ONLY === 'all' || ONLY === 'lm') {
|
||||
for (const sub of ['lm', 'lm-06b', 'lm-17b', 'lm-4b']) {
|
||||
targets.push(...findAdapters(path.join(ADAPTERS_ROOT, sub), 'lm'));
|
||||
}
|
||||
}
|
||||
|
||||
if (!targets.length) {
|
||||
console.error(`No adapters found under ${ADAPTERS_ROOT} (--only ${ONLY}).`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const rows = [];
|
||||
let skipped = 0;
|
||||
for (const t of targets) {
|
||||
const head = readHeader(t.file);
|
||||
if (!head) { rows.push({ ...t, trigger: '', source: 'UNREADABLE', action: 'skip' }); continue; }
|
||||
const existing = (head.json.__metadata__ || {}).hot_step_trigger;
|
||||
if (existing && !FORCE) { skipped++; continue; }
|
||||
const { trigger, position, source } = proposeTrigger(t.name, datasetTags);
|
||||
rows.push({ ...t, trigger, position, source, action: trigger ? (existing ? 'restamp' : 'stamp') : 'skip' });
|
||||
}
|
||||
|
||||
const pad = (s, n) => String(s).padEnd(n);
|
||||
console.log(`\nAdapters root: ${ADAPTERS_ROOT}`);
|
||||
console.log(`${targets.length} adapter(s) found, ${skipped} already stamped (skipped), ${rows.length} to review.`);
|
||||
console.log(`Fallback position: ${POSITION} Mode: ${APPLY ? 'APPLY' : 'DRY RUN'}${BACKUP ? ' (keeping .bak)' : ''}`);
|
||||
console.log(`Dataset tags loaded: ${datasetTags.size}${DATASETS_DIR ? ` (incl. ${DATASETS_DIR})` : ''}\n`);
|
||||
console.log(`${pad('KIND', 5)} ${pad('ADAPTER', 42)} ${pad('PROPOSED TRIGGER', 28)} ${pad('POS', 8)} ${pad('SOURCE', 9)} ACTION`);
|
||||
console.log('-'.repeat(110));
|
||||
for (const r of rows) {
|
||||
console.log(`${pad(r.kind, 5)} ${pad(r.name.slice(0, 42), 42)} ${pad(r.trigger.slice(0, 28), 28)} ${pad(r.position || '-', 8)} ${pad(r.source, 9)} ${r.action}`);
|
||||
}
|
||||
const bySource = rows.reduce((m, r) => ({ ...m, [r.source]: (m[r.source] || 0) + 1 }), {});
|
||||
console.log(`\nBy source: ${Object.entries(bySource).map(([k, v]) => `${k}=${v}`).join(' ')}`);
|
||||
|
||||
if (!APPLY) {
|
||||
console.log(`\nDRY RUN — nothing written. Re-run with --apply once the table above is right.`);
|
||||
console.log(`Fix individual rows with --map triggers.json, e.g. {"abba-4B": "abba"}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
let ok = 0, failed = 0;
|
||||
for (const r of rows) {
|
||||
if (r.action === 'skip') continue;
|
||||
try {
|
||||
stamp(r.file, {
|
||||
hot_step_trigger: r.trigger,
|
||||
hot_step_trigger_position: r.position || POSITION,
|
||||
'modelspec.trigger_phrase': r.trigger,
|
||||
});
|
||||
ok++;
|
||||
} catch (e) {
|
||||
console.error(`FAILED ${r.name}: ${e.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
console.log(`\nStamped ${ok} adapter(s)${failed ? `, ${failed} failed` : ''}.`);
|
||||
process.exit(failed ? 1 : 0);
|
||||
Reference in New Issue
Block a user