Initial release
This commit is contained in:
civ
@@ -0,0 +1,85 @@
|
||||
// exportService.ts — Export lyric generations to JSON + TXT files
|
||||
//
|
||||
// Port of Python export_service.py
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { config } from '../../config.js';
|
||||
|
||||
export interface ExportData {
|
||||
title: string;
|
||||
lyrics: string;
|
||||
artistName: string;
|
||||
albumName?: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
bpm?: number;
|
||||
key?: string;
|
||||
caption?: string;
|
||||
duration?: number;
|
||||
subject?: string;
|
||||
extraInstructions?: string;
|
||||
createdAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a generation to both JSON and TXT files.
|
||||
* Returns the paths of the exported files.
|
||||
*/
|
||||
export function exportGeneration(data: ExportData): { jsonPath: string; txtPath: string } {
|
||||
const exportDir = config.lireek.exportDir;
|
||||
fs.mkdirSync(exportDir, { recursive: true });
|
||||
|
||||
// Build safe filename: "Artist - Title" or "Artist - Album - Title"
|
||||
const safeName = (s: string) => s.replace(/[<>:"/\\|?*]/g, '_').trim();
|
||||
const parts = [safeName(data.artistName)];
|
||||
if (data.albumName) parts.push(safeName(data.albumName));
|
||||
parts.push(safeName(data.title || 'Untitled'));
|
||||
const baseName = parts.join(' - ');
|
||||
|
||||
// Deduplicate: if file exists, append (2), (3), etc.
|
||||
let finalBase = baseName;
|
||||
let counter = 1;
|
||||
while (fs.existsSync(path.join(exportDir, `${finalBase}.json`))) {
|
||||
counter++;
|
||||
finalBase = `${baseName} (${counter})`;
|
||||
}
|
||||
|
||||
const jsonPath = path.join(exportDir, `${finalBase}.json`);
|
||||
const txtPath = path.join(exportDir, `${finalBase}.txt`);
|
||||
|
||||
// JSON export (full metadata)
|
||||
const jsonData = {
|
||||
title: data.title,
|
||||
artist: data.artistName,
|
||||
album: data.albumName ?? null,
|
||||
lyrics: data.lyrics,
|
||||
provider: data.provider,
|
||||
model: data.model,
|
||||
bpm: data.bpm ?? null,
|
||||
key: data.key ?? null,
|
||||
caption: data.caption ?? null,
|
||||
duration: data.duration ?? null,
|
||||
subject: data.subject ?? null,
|
||||
extra_instructions: data.extraInstructions ?? null,
|
||||
exported_at: new Date().toISOString(),
|
||||
created_at: data.createdAt ?? null,
|
||||
};
|
||||
fs.writeFileSync(jsonPath, JSON.stringify(jsonData, null, 2), 'utf-8');
|
||||
|
||||
// TXT export (human-readable)
|
||||
const txtLines = [
|
||||
`Title: ${data.title || 'Untitled'}`,
|
||||
`Artist: ${data.artistName}`,
|
||||
];
|
||||
if (data.albumName) txtLines.push(`Album Style: ${data.albumName}`);
|
||||
if (data.bpm) txtLines.push(`BPM: ${data.bpm}`);
|
||||
if (data.key) txtLines.push(`Key: ${data.key}`);
|
||||
if (data.caption) txtLines.push(`Caption: ${data.caption}`);
|
||||
if (data.duration) txtLines.push(`Duration: ${data.duration}s`);
|
||||
txtLines.push('', '---', '', data.lyrics);
|
||||
fs.writeFileSync(txtPath, txtLines.join('\n'), 'utf-8');
|
||||
|
||||
console.log(`[Export] Saved ${jsonPath}`);
|
||||
return { jsonPath, txtPath };
|
||||
}
|
||||
@@ -0,0 +1,476 @@
|
||||
// geniusService.ts — Lyrics acquisition via the Genius API
|
||||
//
|
||||
// Port of Python genius_service.py.
|
||||
// Uses fetch() for the Genius REST API and cheerio for HTML parsing.
|
||||
|
||||
import * as cheerio from 'cheerio';
|
||||
import { config } from '../../config.js';
|
||||
|
||||
const API_ROOT = 'https://api.genius.com';
|
||||
|
||||
const BROWSER_HEADERS: Record<string, string> = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
};
|
||||
|
||||
export interface SongLyrics {
|
||||
title: string;
|
||||
album: string | null;
|
||||
lyrics: string;
|
||||
}
|
||||
|
||||
export interface LyricsSearchResponse {
|
||||
artist: string;
|
||||
album: string | null;
|
||||
songs: SongLyrics[];
|
||||
total_songs: number;
|
||||
}
|
||||
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
function getAuthHeaders(): Record<string, string> {
|
||||
const token = config.lireek.geniusAccessToken;
|
||||
if (!token) {
|
||||
throw new Error('GENIUS_ACCESS_TOKEN is not set. Please add it to your .env file.');
|
||||
}
|
||||
return { Authorization: `Bearer ${token}` };
|
||||
}
|
||||
|
||||
function cleanLyrics(raw: string): string {
|
||||
if (!raw) return '';
|
||||
// Remove contributor/title header (captures up to first section header)
|
||||
let text = raw.replace(/^\d+\s*Contributors?.*?Lyrics.*?(?=\[)/is, '');
|
||||
// Simpler header strip if no section header found
|
||||
text = text.replace(/^\d+\s*Contributors?.*?Lyrics\s*\n?/is, '');
|
||||
// Remove 'You might also like'
|
||||
text = text.replace(/You might also like\s*/g, '');
|
||||
// Remove trailing 'Embed'
|
||||
text = text.replace(/\d*Embed$/, '').trim();
|
||||
// Collapse 3+ consecutive blank lines into 2
|
||||
text = text.replace(/\n{3,}/g, '\n\n');
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
async function scrapeLyrics(songUrl: string): Promise<string | null> {
|
||||
try {
|
||||
const resp = await fetch(songUrl, {
|
||||
headers: BROWSER_HEADERS,
|
||||
redirect: 'follow',
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.warn(`[Genius] Failed to scrape ${songUrl}: ${resp.status}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const html = await resp.text();
|
||||
const $ = cheerio.load(html.replace(/<br\/?>/gi, '\n'));
|
||||
|
||||
// Modern Genius layout: data-lyrics-container divs
|
||||
const containers = $('div[data-lyrics-container="true"]');
|
||||
if (containers.length) {
|
||||
const parts: string[] = [];
|
||||
containers.each((_, el) => { parts.push($(el).text()); });
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
// Fallback: root lyrics div
|
||||
const root = $('div.lyrics, div[class*="Lyrics__Root"]');
|
||||
if (root.length) return root.text();
|
||||
|
||||
console.warn(`[Genius] No lyrics div found on: ${songUrl}`);
|
||||
return null;
|
||||
} catch (err) {
|
||||
console.warn(`[Genius] Scrape error for ${songUrl}:`, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
|
||||
// ── Authenticated Genius API calls ──────────────────────────────────────────
|
||||
|
||||
async function apiSearch(query: string, perPage = 20): Promise<any[]> {
|
||||
const headers = getAuthHeaders();
|
||||
const url = `${API_ROOT}/search?q=${encodeURIComponent(query)}&per_page=${perPage}`;
|
||||
const resp = await fetch(url, { headers });
|
||||
if (!resp.ok) throw new Error(`Genius search failed: ${resp.status}`);
|
||||
const data = await resp.json() as any;
|
||||
return data.response.hits;
|
||||
}
|
||||
|
||||
async function apiGetArtistSongs(artistId: number, perPage = 20, sort = 'popularity'): Promise<any[]> {
|
||||
const headers = getAuthHeaders();
|
||||
const url = `${API_ROOT}/artists/${artistId}/songs?per_page=${perPage}&page=1&sort=${sort}`;
|
||||
const resp = await fetch(url, { headers });
|
||||
if (!resp.ok) throw new Error(`Genius artist songs failed: ${resp.status}`);
|
||||
const data = await resp.json() as any;
|
||||
return data.response.songs;
|
||||
}
|
||||
|
||||
async function apiGetAlbumTracks(albumId: number): Promise<any[]> {
|
||||
const headers = getAuthHeaders();
|
||||
const tracks: any[] = [];
|
||||
let page: number | null = 1;
|
||||
|
||||
while (page) {
|
||||
const url = `${API_ROOT}/albums/${albumId}/tracks?per_page=50&page=${page}`;
|
||||
const resp = await fetch(url, { headers });
|
||||
if (!resp.ok) throw new Error(`Genius album tracks failed: ${resp.status}`);
|
||||
const data = await resp.json() as any;
|
||||
tracks.push(...data.response.tracks);
|
||||
page = data.response.next_page ?? null;
|
||||
}
|
||||
return tracks;
|
||||
}
|
||||
|
||||
async function apiGetArtistDetails(artistId: number): Promise<any> {
|
||||
const headers = getAuthHeaders();
|
||||
const resp = await fetch(`${API_ROOT}/artists/${artistId}`, { headers });
|
||||
if (!resp.ok) throw new Error(`Genius artist details failed: ${resp.status}`);
|
||||
const data = await resp.json() as any;
|
||||
return data.response.artist;
|
||||
}
|
||||
|
||||
async function getArtistIdFromUrl(url: string): Promise<number | null> {
|
||||
try {
|
||||
const resp = await fetch(url, { headers: BROWSER_HEADERS, redirect: 'follow' });
|
||||
if (!resp.ok) return null;
|
||||
const html = await resp.text();
|
||||
const match = html.match(/\\?"artist_id\\?":\s*(\d+)/) ?? html.match(/content="genius:\/\/artists\/(\d+)"/);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
async function findArtistId(artistName: string): Promise<number | null> {
|
||||
const hits = await apiSearch(artistName, 5);
|
||||
const nameLower = artistName.toLowerCase();
|
||||
for (const hit of hits) {
|
||||
const primary = hit.result?.primary_artist;
|
||||
if (primary?.name?.toLowerCase() === nameLower) return primary.id;
|
||||
}
|
||||
return hits[0]?.result?.primary_artist?.id ?? null;
|
||||
}
|
||||
|
||||
// Patterns indicating bonus/non-original tracks
|
||||
const BONUS_PATTERN = /\([^)]*(?:Demo|Live|Outtake|Cassette|Remix|Acoustic|Remaster|Session)[^)]*\)/i;
|
||||
function isBonusTrack(title: string): boolean {
|
||||
return BONUS_PATTERN.test(title);
|
||||
}
|
||||
|
||||
function slugifyForGenius(name: string): string {
|
||||
return name.replace(/[^\w\s-]/g, '').trim().replace(/\s+/g, '-');
|
||||
}
|
||||
|
||||
async function findAlbumIdByPage(albumName: string, artistName: string): Promise<number | null> {
|
||||
const slug = `${slugifyForGenius(artistName)}/${slugifyForGenius(albumName)}`;
|
||||
const url = `https://genius.com/albums/${slug}`;
|
||||
const artistLower = artistName.toLowerCase();
|
||||
|
||||
try {
|
||||
const resp = await fetch(url, { headers: BROWSER_HEADERS, redirect: 'follow' });
|
||||
if (!resp.ok) return null;
|
||||
const html = await resp.text();
|
||||
const $ = cheerio.load(html);
|
||||
const links = $('div.chart_row a.u-display_block').toArray();
|
||||
if (!links.length) return null;
|
||||
|
||||
for (const link of links.slice(0, 3)) {
|
||||
const h3 = $(link).find('h3');
|
||||
let titleText = (h3.length ? h3.text() : $(link).text()).trim();
|
||||
titleText = titleText.replace(/\s*Lyrics$/, '').trim();
|
||||
if (!titleText) continue;
|
||||
|
||||
const hits = await apiSearch(`${titleText} ${artistName}`, 5);
|
||||
for (const hit of hits) {
|
||||
const result = hit.result ?? {};
|
||||
const songId = result.id;
|
||||
if (!songId) continue;
|
||||
if (result.primary_artist?.name?.toLowerCase() !== artistLower) continue;
|
||||
|
||||
const headers = getAuthHeaders();
|
||||
const sResp = await fetch(`${API_ROOT}/songs/${songId}`, { headers });
|
||||
if (!sResp.ok) continue;
|
||||
const sData = await sResp.json() as any;
|
||||
const album = sData.response.song.album;
|
||||
if (album && albumName.toLowerCase().includes(album.name?.toLowerCase())) {
|
||||
return album.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function findAlbumId(albumName: string, artistName: string): Promise<number | null> {
|
||||
const headers = getAuthHeaders();
|
||||
const query = `${albumName} ${artistName}`;
|
||||
const hits = await apiSearch(query, 10);
|
||||
const albumLower = albumName.toLowerCase();
|
||||
const artistLower = artistName.toLowerCase();
|
||||
|
||||
// Sort: artist matches first
|
||||
const sorted = [...hits].sort((a, b) => {
|
||||
const aMatch = a.result?.primary_artist?.name?.toLowerCase() === artistLower ? 1 : 0;
|
||||
const bMatch = b.result?.primary_artist?.name?.toLowerCase() === artistLower ? 1 : 0;
|
||||
return bMatch - aMatch;
|
||||
});
|
||||
|
||||
const candidates: { id: number; name: string }[] = [];
|
||||
const seenIds = new Set<number>();
|
||||
|
||||
for (const hit of sorted) {
|
||||
const songId = hit.result?.id;
|
||||
if (!songId || candidates.length >= 3) break;
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${API_ROOT}/songs/${songId}`, { headers });
|
||||
if (!resp.ok) continue;
|
||||
const data = await resp.json() as any;
|
||||
const album = data.response.song.album;
|
||||
if (album && album.name?.toLowerCase().includes(albumLower)) {
|
||||
if (!seenIds.has(album.id)) {
|
||||
seenIds.add(album.id);
|
||||
candidates.push({ id: album.id, name: album.name });
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
if (candidates.length) {
|
||||
// Prefer exact name match, then shortest name (less likely deluxe/expanded)
|
||||
const exact = candidates.find(c => c.name.toLowerCase() === albumLower);
|
||||
if (exact) return exact.id;
|
||||
candidates.sort((a, b) => a.name.length - b.name.length);
|
||||
return candidates[0].id;
|
||||
}
|
||||
|
||||
// Fallback: direct page scrape
|
||||
return findAlbumIdByPage(albumName, artistName);
|
||||
}
|
||||
|
||||
async function scrapeAlbumPageTracks(
|
||||
albumName = '', artistName = '', url?: string,
|
||||
): Promise<{ title: string; url: string }[]> {
|
||||
if (!url) {
|
||||
const slug = `${slugifyForGenius(artistName)}/${slugifyForGenius(albumName)}`;
|
||||
url = `https://genius.com/albums/${slug}`;
|
||||
}
|
||||
try {
|
||||
const resp = await fetch(url, { headers: BROWSER_HEADERS, redirect: 'follow' });
|
||||
if (!resp.ok) return [];
|
||||
const html = await resp.text();
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const tracks: { title: string; url: string }[] = [];
|
||||
$('div.chart_row a.u-display_block').each((_, el) => {
|
||||
let href = $(el).attr('href') ?? '';
|
||||
if (!href) return;
|
||||
if (!href.startsWith('http')) href = `https://genius.com${href}`;
|
||||
const h3 = $(el).find('h3');
|
||||
let title = (h3.length ? h3.text() : $(el).text()).trim();
|
||||
title = title.replace(/\s*Lyrics$/, '').trim();
|
||||
if (title) tracks.push({ title, url: href });
|
||||
});
|
||||
return tracks;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
|
||||
// ── Public API ──────────────────────────────────────────────────────────────
|
||||
|
||||
export async function fetchLyrics(
|
||||
artistName: string,
|
||||
albumName?: string | null,
|
||||
maxSongs = 10,
|
||||
): Promise<LyricsSearchResponse> {
|
||||
const scrapeDelay = 300;
|
||||
const songs: SongLyrics[] = [];
|
||||
let artistId: number | null = null;
|
||||
|
||||
// Handle Genius artist URL input
|
||||
if (artistName.includes('genius.com/artists/')) {
|
||||
const urlMatch = artistName.match(/(https?:\/\/(?:www\.)?genius\.com\/artists\/[^\s]+)/);
|
||||
const url = urlMatch ? urlMatch[1] : artistName.trim();
|
||||
artistId = await getArtistIdFromUrl(url);
|
||||
if (artistId) {
|
||||
try {
|
||||
const details = await apiGetArtistDetails(artistId);
|
||||
if (details.name) artistName = details.name;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle album URL input
|
||||
let albumUrl: string | undefined;
|
||||
if (albumName && albumName.includes('genius.com/albums/')) {
|
||||
const urlMatch = albumName.match(/(https?:\/\/(?:www\.)?genius\.com\/albums\/[^\s]+)/);
|
||||
albumUrl = urlMatch ? urlMatch[1] : albumName.trim();
|
||||
const parts = albumUrl.replace(/\/$/, '').split('/');
|
||||
if (parts.length >= 2) {
|
||||
albumName = parts[parts.length - 1].replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
if (!artistId) {
|
||||
const artistSlug = parts[parts.length - 2];
|
||||
artistName = artistSlug.replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (albumName) {
|
||||
console.log(`[Genius] Fetching album '${albumName}' by '${artistName}'`);
|
||||
|
||||
let albumIdResolved: number | null = null;
|
||||
if (!albumUrl) {
|
||||
albumIdResolved = await findAlbumId(albumName, artistName);
|
||||
}
|
||||
|
||||
if (!albumUrl && albumIdResolved != null) {
|
||||
// API-based album track fetch
|
||||
const tracks = await apiGetAlbumTracks(albumIdResolved);
|
||||
const seenTitles = new Set<string>();
|
||||
|
||||
for (const track of tracks) {
|
||||
const songInfo = track.song ?? {};
|
||||
const url = songInfo.url;
|
||||
const title = songInfo.title ?? 'Unknown';
|
||||
if (!url || songInfo.lyrics_state !== 'complete' || songInfo.instrumental) continue;
|
||||
if (isBonusTrack(title)) continue;
|
||||
const baseTitle = title.replace(/\s*\(.*\)/, '').trim().toLowerCase();
|
||||
if (seenTitles.has(baseTitle)) continue;
|
||||
seenTitles.add(baseTitle);
|
||||
|
||||
try {
|
||||
await sleep(scrapeDelay);
|
||||
const raw = await scrapeLyrics(url);
|
||||
if (raw) songs.push({ title, album: albumName, lyrics: cleanLyrics(raw) });
|
||||
} catch (err) {
|
||||
console.warn(`[Genius] Failed to scrape '${title}':`, err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Fallback: scrape tracks from album page
|
||||
const pageTracks = await scrapeAlbumPageTracks(albumName, artistName, albumUrl);
|
||||
for (const pt of pageTracks) {
|
||||
if (isBonusTrack(pt.title)) continue;
|
||||
try {
|
||||
await sleep(scrapeDelay);
|
||||
const raw = await scrapeLyrics(pt.url);
|
||||
if (raw) songs.push({ title: pt.title, album: albumName, lyrics: cleanLyrics(raw) });
|
||||
} catch (err) {
|
||||
console.warn(`[Genius] Failed to scrape '${pt.title}':`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// General artist search (no specific album)
|
||||
console.log(`[Genius] Fetching up to ${maxSongs} songs by '${artistName}'`);
|
||||
if (!artistId) artistId = await findArtistId(artistName);
|
||||
if (!artistId) {
|
||||
throw new Error(`Could not find artist '${artistName}' on Genius.`);
|
||||
}
|
||||
|
||||
const apiSongs = await apiGetArtistSongs(artistId, maxSongs);
|
||||
for (const songInfo of apiSongs.slice(0, maxSongs)) {
|
||||
const url = songInfo.url;
|
||||
const title = songInfo.title ?? 'Unknown';
|
||||
const songAlbum = songInfo.album?.name ?? null;
|
||||
if (!url) continue;
|
||||
|
||||
try {
|
||||
await sleep(scrapeDelay);
|
||||
const raw = await scrapeLyrics(url);
|
||||
if (raw) songs.push({ title, album: songAlbum, lyrics: cleanLyrics(raw) });
|
||||
} catch (err) {
|
||||
console.warn(`[Genius] Failed to scrape '${title}':`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!songs.length) {
|
||||
throw new Error(
|
||||
`No lyrics found for '${artistName}'` +
|
||||
(albumName ? ` – album '${albumName}'` : '') +
|
||||
'. Please check the spelling and try again.',
|
||||
);
|
||||
}
|
||||
|
||||
return { artist: artistName, album: albumName ?? null, songs, total_songs: songs.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for a single song's lyrics on Genius.
|
||||
*/
|
||||
/** Lowercase, strip parentheticals/brackets and punctuation, collapse spaces. */
|
||||
function normalizeTitle(s: string): string {
|
||||
return s.toLowerCase()
|
||||
.replace(/\([^)]*\)|\[[^\]]*\]/g, ' ')
|
||||
.replace(/[^\p{L}\p{N}\s]/gu, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function searchSongLyrics(
|
||||
artist: string, title: string,
|
||||
opts?: { relaxed?: boolean },
|
||||
): Promise<{ title: string; lyrics: string; url: string } | null> {
|
||||
const hits = await apiSearch(`${title} ${artist}`, 5);
|
||||
const artistLower = artist.toLowerCase().trim();
|
||||
const titleNorm = normalizeTitle(title);
|
||||
|
||||
for (const hit of hits) {
|
||||
const result = hit.result ?? {};
|
||||
const primaryLower = (result.primary_artist?.name ?? '').toLowerCase().trim();
|
||||
if (opts?.relaxed) {
|
||||
// Collabs and feats break exact matching ("Electric Callboy & BABYMETAL",
|
||||
// "Artist feat. X") — accept containment either way, but then require the
|
||||
// titles to agree so a popular unrelated song can't slip in.
|
||||
const artistOk = !!primaryLower
|
||||
&& (primaryLower.includes(artistLower) || artistLower.includes(primaryLower));
|
||||
if (!artistOk) continue;
|
||||
const resultNorm = normalizeTitle(result.title ?? '');
|
||||
const titleOk = !!titleNorm && !!resultNorm
|
||||
&& (resultNorm.includes(titleNorm) || titleNorm.includes(resultNorm));
|
||||
if (!titleOk) continue;
|
||||
} else if (primaryLower !== artistLower) {
|
||||
continue;
|
||||
}
|
||||
const songUrl = result.url;
|
||||
if (!songUrl) continue;
|
||||
|
||||
const raw = await scrapeLyrics(songUrl);
|
||||
if (raw) {
|
||||
return { title: result.title ?? title, lyrics: cleanLyrics(raw), url: songUrl };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh an artist's image URL from Genius.
|
||||
* Returns the image URL or null if not found.
|
||||
*/
|
||||
export async function getArtistImageUrl(artistName: string): Promise<string | null> {
|
||||
try {
|
||||
const id = await findArtistId(artistName);
|
||||
if (!id) return null;
|
||||
const details = await apiGetArtistDetails(id);
|
||||
return details.image_url ?? null;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an album cover image URL from Genius.
|
||||
*/
|
||||
export async function getAlbumImageUrl(albumName: string, artistName: string): Promise<string | null> {
|
||||
try {
|
||||
const albumId = await findAlbumId(albumName, artistName);
|
||||
if (!albumId) return null;
|
||||
const headers = getAuthHeaders();
|
||||
const resp = await fetch(`${API_ROOT}/albums/${albumId}`, { headers });
|
||||
if (!resp.ok) return null;
|
||||
const data = await resp.json() as any;
|
||||
return data.response.album?.cover_art_url ?? null;
|
||||
} catch { return null; }
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// llm/anthropic.ts — Anthropic / Claude provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE } from './base.js';
|
||||
import type { ChunkCallback } from './types.js';
|
||||
|
||||
export class AnthropicProvider extends LLMProvider {
|
||||
id = 'anthropic';
|
||||
name = 'Anthropic / Claude';
|
||||
get defaultModel() { return config.lireek.anthropicModel; }
|
||||
availableModels = ['claude-3-5-sonnet-20241022', 'claude-3-5-haiku-20241022', 'claude-3-opus-20240229'];
|
||||
|
||||
isAvailable() { return !!config.lireek.anthropicApiKey; }
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback): Promise<string> {
|
||||
const url = 'https://api.anthropic.com/v1/messages';
|
||||
const payload = {
|
||||
model: model || this.defaultModel,
|
||||
max_tokens: 4096,
|
||||
system: systemPrompt,
|
||||
messages: [{ role: 'user', content: userPrompt }],
|
||||
stream: !!onChunk,
|
||||
};
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': config.lireek.anthropicApiKey,
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`Anthropic error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
let fullText = '';
|
||||
await readSSE(resp, (text) => { fullText += text; onChunk(text); }, (data) => {
|
||||
if (data.type === 'content_block_delta' && data.delta?.text) return data.delta.text;
|
||||
return null;
|
||||
});
|
||||
return fullText;
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.content?.[0]?.text || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
// llm/base.ts — LLMProvider abstract base class + SSE streaming helper
|
||||
|
||||
import type { ProviderInfo, ChunkCallback, CallOptions } from './types.js';
|
||||
import { skipThinkingSignal } from './types.js';
|
||||
|
||||
export abstract class LLMProvider {
|
||||
abstract id: string;
|
||||
abstract name: string;
|
||||
abstract defaultModel: string;
|
||||
availableModels: string[] = [];
|
||||
|
||||
abstract isAvailable(): boolean;
|
||||
|
||||
abstract call(
|
||||
systemPrompt: string,
|
||||
userPrompt: string,
|
||||
model?: string,
|
||||
onChunk?: ChunkCallback,
|
||||
options?: CallOptions
|
||||
): Promise<string>;
|
||||
|
||||
toInfo(): ProviderInfo {
|
||||
return {
|
||||
id: this.id,
|
||||
name: this.name,
|
||||
available: this.isAvailable(),
|
||||
models: this.availableModels.length ? this.availableModels : (this.defaultModel ? [this.defaultModel] : []),
|
||||
default_model: this.defaultModel,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Qwen3-family soft switch: a bare `/no_think` in the system prompt makes the
|
||||
// chat template skip the thinking block. Plain text to every other model, so
|
||||
// it is safe to send unconditionally when CallOptions.noThink is set.
|
||||
export function noThinkSystemPrompt(systemPrompt: string): string {
|
||||
return `${systemPrompt}\n\n/no_think`;
|
||||
}
|
||||
|
||||
export async function readSSE(
|
||||
response: Response,
|
||||
onChunk: ChunkCallback,
|
||||
extractText: (data: any) => string | null,
|
||||
extractDisplayOnly?: (data: any) => string | null
|
||||
): Promise<string> {
|
||||
if (!response.body) return '';
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
let fullText = '';
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || ''; // Keep the last incomplete line in buffer
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ')) {
|
||||
const dataStr = line.slice(6).trim();
|
||||
if (dataStr === '[DONE]') {
|
||||
reader.cancel();
|
||||
return fullText;
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(dataStr);
|
||||
|
||||
// Break on finish_reason (OpenAI-compatible sentinel)
|
||||
if (parsed.choices?.[0]?.finish_reason) {
|
||||
const lastText = extractText(parsed);
|
||||
if (lastText) {
|
||||
fullText += lastText;
|
||||
onChunk(lastText);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Display-only content (e.g. reasoning/thinking) — stream to UI but don't keep
|
||||
if (extractDisplayOnly) {
|
||||
const displayText = extractDisplayOnly(parsed);
|
||||
if (displayText) {
|
||||
onChunk(displayText);
|
||||
// Track for skip-thinking detection but don't add to returned text
|
||||
if (skipThinkingSignal) {
|
||||
const thinkCheck = fullText + displayText;
|
||||
if (thinkCheck.includes('<think>') && !thinkCheck.includes('</think>')) {
|
||||
reader.cancel();
|
||||
return fullText;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const text = extractText(parsed);
|
||||
if (text) {
|
||||
fullText += text;
|
||||
onChunk(text);
|
||||
|
||||
if (skipThinkingSignal && fullText.includes('<think>') && !fullText.includes('</think>')) {
|
||||
reader.cancel();
|
||||
return fullText;
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
return fullText;
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// llm/gemini.ts — Google Gemini provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback } from './types.js';
|
||||
|
||||
export class GeminiProvider extends LLMProvider {
|
||||
id = 'gemini';
|
||||
name = 'Google Gemini';
|
||||
get defaultModel() { return config.lireek.geminiModel; }
|
||||
availableModels = ['gemini-2.5-flash']; // initial fallback, replaced by API fetch
|
||||
|
||||
/** Cache fetched models so we don't hit the API on every listProviders() call */
|
||||
private _cachedModels: string[] | null = null;
|
||||
private _cacheExpiry = 0;
|
||||
private static CACHE_TTL = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
isAvailable() { return !!config.lireek.geminiApiKey; }
|
||||
|
||||
/** Fetch available models from the Gemini API, filtered to those that support generateContent */
|
||||
private async getRemoteModels(): Promise<string[]> {
|
||||
const now = Date.now();
|
||||
if (this._cachedModels && now < this._cacheExpiry) return this._cachedModels;
|
||||
|
||||
try {
|
||||
const allModels: Array<{ name: string; supportedGenerationMethods?: string[] }> = [];
|
||||
let pageToken: string | undefined;
|
||||
|
||||
// Paginate through all models
|
||||
do {
|
||||
const url = new URL('https://generativelanguage.googleapis.com/v1beta/models');
|
||||
url.searchParams.set('key', config.lireek.geminiApiKey);
|
||||
url.searchParams.set('pageSize', '100');
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken);
|
||||
|
||||
const resp = await fetch(url.toString(), { signal: AbortSignal.timeout(8000) });
|
||||
if (!resp.ok) {
|
||||
console.warn(`[Gemini] models.list failed: ${resp.status}`);
|
||||
break;
|
||||
}
|
||||
const data = await resp.json();
|
||||
if (Array.isArray(data.models)) allModels.push(...data.models);
|
||||
pageToken = data.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
if (allModels.length === 0) return this.availableModels;
|
||||
|
||||
// Filter to models that support generateContent (excludes embedding, AQA, etc.)
|
||||
const generative = allModels
|
||||
.filter(m => m.supportedGenerationMethods?.includes('generateContent'))
|
||||
.map(m => m.name.replace(/^models\//, ''))
|
||||
// Exclude tuning, embedding, and legacy models cluttering the list
|
||||
.filter(name => !name.includes('embedding') && !name.includes('aqa') && !name.includes('tunedModels'));
|
||||
|
||||
if (generative.length === 0) return this.availableModels;
|
||||
|
||||
// Sort: prefer 2.5 > 2.0 > 1.5, flash before pro, shorter names first (base > dated variants)
|
||||
generative.sort((a, b) => {
|
||||
// Extract version number for primary sort
|
||||
const verA = parseFloat(a.match(/(\d+\.\d+)/)?.[1] || '0');
|
||||
const verB = parseFloat(b.match(/(\d+\.\d+)/)?.[1] || '0');
|
||||
if (verB !== verA) return verB - verA;
|
||||
// Same version: flash before pro
|
||||
const isFlashA = a.includes('flash') ? 0 : 1;
|
||||
const isFlashB = b.includes('flash') ? 0 : 1;
|
||||
if (isFlashA !== isFlashB) return isFlashA - isFlashB;
|
||||
// Same tier: shorter names (base model) before dated variants
|
||||
return a.length - b.length;
|
||||
});
|
||||
|
||||
this._cachedModels = generative;
|
||||
this._cacheExpiry = now + GeminiProvider.CACHE_TTL;
|
||||
this.availableModels = generative;
|
||||
return generative;
|
||||
} catch (err: any) {
|
||||
console.warn(`[Gemini] Failed to fetch models: ${err.message}`);
|
||||
return this.availableModels;
|
||||
}
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getRemoteModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : [this.defaultModel],
|
||||
default_model: this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback): Promise<string> {
|
||||
const modelName = model || this.defaultModel;
|
||||
const url = `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:${onChunk ? 'streamGenerateContent?alt=sse&' : 'generateContent?'}key=${config.lireek.geminiApiKey}`;
|
||||
|
||||
const payload = {
|
||||
systemInstruction: { parts: [{ text: systemPrompt }] },
|
||||
contents: [{ role: 'user', parts: [{ text: userPrompt }] }],
|
||||
};
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`Gemini API error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
return await readSSE(resp, onChunk, (data) => data.candidates?.[0]?.content?.parts?.[0]?.text || null);
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.candidates?.[0]?.content?.parts?.[0]?.text || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// llm/index.ts — Barrel re-export for the LLM provider module
|
||||
//
|
||||
// All external consumers should import from this module.
|
||||
// Internal files within llm/ import from their specific submodules directly.
|
||||
|
||||
// Types
|
||||
export type { ProviderInfo, GenerationResponse, ChunkCallback, CallOptions } from './types.js';
|
||||
export { skipThinkingSignal, setSkipThinking, resetSkipThinking } from './types.js';
|
||||
|
||||
// Base class (for type use / extension)
|
||||
export { LLMProvider } from './base.js';
|
||||
|
||||
// Registry
|
||||
export { getProvider, listProviders } from './registry.js';
|
||||
|
||||
// Post-processing
|
||||
export { stripThinkingBlocks, postprocessLyrics, fixSectionLabels, enforceLineCounts, fixAPrefix, stripLyricQuotes, estimateDuration } from './postprocess.js';
|
||||
|
||||
// Orchestration (high-level generation functions)
|
||||
export { generateLyricsStreaming, refineLyricsStreaming } from './orchestration.js';
|
||||
@@ -0,0 +1,85 @@
|
||||
// llm/llamacpp.ts — llama.cpp server provider
|
||||
//
|
||||
// Connects to a llama.cpp server (llama-server / llama-cli --server)
|
||||
// which exposes an OpenAI-compatible API at /v1/chat/completions and /v1/models.
|
||||
// Default endpoint: http://127.0.0.1:8080/v1
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE, noThinkSystemPrompt } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback, CallOptions } from './types.js';
|
||||
|
||||
export class LlamaCppProvider extends LLMProvider {
|
||||
id = 'llamacpp';
|
||||
name = 'llama.cpp';
|
||||
get defaultModel() { return config.lireek.llamacppModel; }
|
||||
|
||||
isAvailable() { return true; }
|
||||
|
||||
private async getLocalModels(): Promise<string[]> {
|
||||
try {
|
||||
const baseUrl = config.lireek.llamacppBaseUrl.replace(/\/+$/, '');
|
||||
const resp = await fetch(`${baseUrl}/models`, { signal: AbortSignal.timeout(3000) });
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
this.availableModels = data.data?.map((m: any) => m.id).sort().reverse() || [];
|
||||
return this.availableModels;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getLocalModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : (this.defaultModel ? [this.defaultModel] : []),
|
||||
default_model: models.length ? models[0] : this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback, options?: CallOptions): Promise<string> {
|
||||
const baseUrl = config.lireek.llamacppBaseUrl.replace(/\/+$/, '');
|
||||
const url = `${baseUrl}/chat/completions`;
|
||||
const modelName = model || (await this.getLocalModels())[0] || this.defaultModel;
|
||||
|
||||
if (!modelName) throw new Error('No models available on llama.cpp server');
|
||||
|
||||
const noThink = !!options?.noThink;
|
||||
const payload: Record<string, any> = {
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: 'system', content: noThink ? noThinkSystemPrompt(systemPrompt) : systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
stream: !!onChunk,
|
||||
};
|
||||
// Officially supported by llama-server: skips the thinking block for
|
||||
// templates that take an enable_thinking kwarg (Qwen3, GLM, ...).
|
||||
if (noThink) {
|
||||
payload.chat_template_kwargs = { enable_thinking: false };
|
||||
// Qwen's official non-thinking sampling profile — presence_penalty=1.5
|
||||
// is their documented guard against degenerate repetition loops in this
|
||||
// mode. Explicit CallOptions values win; max_tokens bounds runaways.
|
||||
payload.temperature = options?.temperature ?? 0.7;
|
||||
payload.top_p = options?.top_p ?? 0.8;
|
||||
payload.top_k = 20;
|
||||
payload.presence_penalty = 1.5;
|
||||
payload.max_tokens = 8192;
|
||||
}
|
||||
|
||||
// 5-min timeout — prevents hung requests from blocking the generation queue
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`llama.cpp error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
return await readSSE(resp, onChunk, (data) => data.choices?.[0]?.delta?.content || null, (data) => data.choices?.[0]?.delta?.reasoning_content || null);
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// llm/lmstudio.ts — LM Studio provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE, noThinkSystemPrompt } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback, CallOptions } from './types.js';
|
||||
|
||||
export class LMStudioProvider extends LLMProvider {
|
||||
id = 'lmstudio';
|
||||
name = 'LM Studio';
|
||||
get defaultModel() { return config.lireek.lmstudioModel; }
|
||||
|
||||
isAvailable() { return true; }
|
||||
|
||||
private async getLocalModels(): Promise<string[]> {
|
||||
try {
|
||||
const baseUrl = config.lireek.lmstudioBaseUrl.replace('/v1', '');
|
||||
const resp = await fetch(`${baseUrl}/v1/models`, { signal: AbortSignal.timeout(3000) });
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
this.availableModels = data.data?.map((m: any) => m.id).sort().reverse() || [];
|
||||
return this.availableModels;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getLocalModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : (this.defaultModel ? [this.defaultModel] : []),
|
||||
default_model: models.length ? models[0] : this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback, options?: CallOptions): Promise<string> {
|
||||
const baseUrl = config.lireek.lmstudioBaseUrl;
|
||||
const url = `${baseUrl}/chat/completions`;
|
||||
const modelName = model || (await this.getLocalModels())[0] || this.defaultModel;
|
||||
|
||||
if (!modelName) throw new Error("No models loaded in LM Studio");
|
||||
|
||||
const noThink = !!options?.noThink;
|
||||
const payload: Record<string, any> = {
|
||||
model: modelName,
|
||||
messages: [
|
||||
{ role: 'system', content: noThink ? noThinkSystemPrompt(systemPrompt) : systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
stream: !!onChunk,
|
||||
};
|
||||
if (noThink) {
|
||||
// Empirically verified against LM Studio + Qwen3.6 (2026-07-09): this is
|
||||
// the field LM Studio honours — reasoning drops to zero and content is
|
||||
// answered directly. Non-thinking models (gemma) accept it harmlessly.
|
||||
payload.reasoning_effort = 'none';
|
||||
// llama.cpp-style template kwarg — ignored by LM Studio today (verified),
|
||||
// kept because it is harmless and honoured if support lands.
|
||||
payload.chat_template_kwargs = { enable_thinking: false };
|
||||
// Qwen's OFFICIAL non-thinking sampling profile. Without thinking, low-
|
||||
// entropy sampling degenerates into endless repetition loops ("the cord
|
||||
// is a square / the cord is a circle" ...); presence_penalty=1.5 is
|
||||
// Qwen's documented anti-loop knob for this mode. Explicit CallOptions
|
||||
// values win. max_tokens bounds any residual runaway.
|
||||
payload.temperature = options?.temperature ?? 0.7;
|
||||
payload.top_p = options?.top_p ?? 0.8;
|
||||
payload.top_k = 20;
|
||||
payload.presence_penalty = 1.5;
|
||||
payload.max_tokens = 8192;
|
||||
}
|
||||
|
||||
const doFetch = () => fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
let resp = await doFetch();
|
||||
// If a server/model combination rejects the non-standard fields, retry
|
||||
// once without them rather than failing the generation.
|
||||
if (!resp.ok && noThink && resp.status === 400) {
|
||||
delete payload.chat_template_kwargs;
|
||||
delete payload.reasoning_effort;
|
||||
resp = await doFetch();
|
||||
}
|
||||
|
||||
if (!resp.ok) throw new Error(`LM Studio error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
return await readSSE(resp, onChunk, (data) => data.choices?.[0]?.delta?.content || null, (data) => data.choices?.[0]?.delta?.reasoning_content || null);
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// llm/ollama.ts — Ollama (Local) provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, noThinkSystemPrompt } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback, CallOptions } from './types.js';
|
||||
import { skipThinkingSignal } from './types.js';
|
||||
|
||||
export class OllamaProvider extends LLMProvider {
|
||||
id = 'ollama';
|
||||
name = 'Ollama (Local)';
|
||||
get defaultModel() { return config.lireek.ollamaModel; }
|
||||
|
||||
isAvailable() { return true; }
|
||||
|
||||
private async getLocalModels(): Promise<string[]> {
|
||||
try {
|
||||
const resp = await fetch(`${config.lireek.ollamaBaseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
this.availableModels = data.models?.map((m: any) => m.name) || [];
|
||||
return this.availableModels;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getLocalModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : [this.defaultModel],
|
||||
default_model: models.length ? models[0] : this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback, options?: CallOptions): Promise<string> {
|
||||
const url = `${config.lireek.ollamaBaseUrl}/api/chat`;
|
||||
const noThink = !!options?.noThink;
|
||||
const payload: Record<string, any> = {
|
||||
model: model || this.defaultModel,
|
||||
messages: [
|
||||
{ role: 'system', content: noThink ? noThinkSystemPrompt(systemPrompt) : systemPrompt },
|
||||
{ role: 'user', content: userPrompt },
|
||||
],
|
||||
stream: !!onChunk,
|
||||
options: { num_predict: 8196 }
|
||||
};
|
||||
// Native Ollama switch for thinking models (qwen3, deepseek-r1, ...).
|
||||
if (noThink) {
|
||||
payload.think = false;
|
||||
// Qwen's official non-thinking sampling profile — presence_penalty=1.5
|
||||
// is their documented guard against degenerate repetition loops.
|
||||
payload.options = {
|
||||
...payload.options,
|
||||
temperature: options?.temperature ?? 0.7,
|
||||
top_p: options?.top_p ?? 0.8,
|
||||
top_k: 20,
|
||||
presence_penalty: 1.5,
|
||||
};
|
||||
}
|
||||
|
||||
const doFetch = () => fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
let resp = await doFetch();
|
||||
// Non-thinking models reject the `think` field — retry once without it.
|
||||
if (!resp.ok && noThink && resp.status === 400 && 'think' in payload) {
|
||||
delete payload.think;
|
||||
resp = await doFetch();
|
||||
}
|
||||
|
||||
if (!resp.ok) throw new Error(`Ollama error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
if (!resp.body) return '';
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let fullText = '';
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunkLines = decoder.decode(value, { stream: true }).split('\n');
|
||||
for (const line of chunkLines) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
// Display-only: stream reasoning to UI but don't keep in result
|
||||
const reasoning = parsed.message?.reasoning_content;
|
||||
if (reasoning) {
|
||||
onChunk(reasoning);
|
||||
continue;
|
||||
}
|
||||
const content = parsed.message?.content;
|
||||
if (content) {
|
||||
fullText += content;
|
||||
onChunk(content);
|
||||
if (skipThinkingSignal && fullText.includes('<think>') && !fullText.includes('</think>')) {
|
||||
reader.cancel();
|
||||
return fullText;
|
||||
}
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
}
|
||||
} finally { reader.releaseLock(); }
|
||||
return fullText;
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.message?.content || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// llm/openai-compat.ts — Generic OpenAI-compatible provider
|
||||
//
|
||||
// Connects to any server that implements the OpenAI API format
|
||||
// (oMLX, vLLM, text-generation-webui, LocalAI, etc.)
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback } from './types.js';
|
||||
|
||||
export class OpenAICompatProvider extends LLMProvider {
|
||||
id = 'openai-compat';
|
||||
get name() { return config.lireek.openaiCompatName || 'OpenAI Compatible'; }
|
||||
get defaultModel() { return config.lireek.openaiCompatModel; }
|
||||
|
||||
isAvailable() { return !!config.lireek.openaiCompatBaseUrl; }
|
||||
|
||||
private async getRemoteModels(): Promise<string[]> {
|
||||
try {
|
||||
const baseUrl = config.lireek.openaiCompatBaseUrl.replace(/\/+$/, '');
|
||||
const headers: Record<string, string> = {};
|
||||
if (config.lireek.openaiCompatApiKey) {
|
||||
headers['Authorization'] = `Bearer ${config.lireek.openaiCompatApiKey}`;
|
||||
}
|
||||
const resp = await fetch(`${baseUrl}/models`, { headers, signal: AbortSignal.timeout(3000) });
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
this.availableModels = data.data?.map((m: any) => m.id).sort().reverse() || [];
|
||||
return this.availableModels;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getRemoteModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : (this.defaultModel ? [this.defaultModel] : []),
|
||||
default_model: models.length ? models[0] : this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback): Promise<string> {
|
||||
const baseUrl = config.lireek.openaiCompatBaseUrl.replace(/\/+$/, '');
|
||||
const url = `${baseUrl}/chat/completions`;
|
||||
const modelName = model || (await this.getRemoteModels())[0] || this.defaultModel;
|
||||
|
||||
if (!modelName) throw new Error(`No models available on ${this.name}`);
|
||||
|
||||
const payload: Record<string, any> = {
|
||||
model: modelName,
|
||||
messages: [{ role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }],
|
||||
stream: !!onChunk,
|
||||
// Force thinking/reasoning for Qwen3-style models on oMLX/vLLM.
|
||||
// Servers that don't support this parameter will safely ignore it.
|
||||
enable_thinking: true,
|
||||
};
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (config.lireek.openaiCompatApiKey) {
|
||||
headers['Authorization'] = `Bearer ${config.lireek.openaiCompatApiKey}`;
|
||||
}
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`${this.name} error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
return await readSSE(resp, onChunk, (data) => data.choices?.[0]?.delta?.content || null, (data) => data.choices?.[0]?.delta?.reasoning_content || null);
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// llm/openai.ts — OpenAI / ChatGPT provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE } from './base.js';
|
||||
import type { ChunkCallback } from './types.js';
|
||||
|
||||
export class OpenAIProvider extends LLMProvider {
|
||||
id = 'openai';
|
||||
name = 'OpenAI / ChatGPT';
|
||||
get defaultModel() { return config.lireek.openaiModel; }
|
||||
availableModels = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo', 'gpt-3.5-turbo'];
|
||||
|
||||
isAvailable() { return !!config.lireek.openaiApiKey; }
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback): Promise<string> {
|
||||
const url = 'https://api.openai.com/v1/chat/completions';
|
||||
const payload = {
|
||||
model: model || this.defaultModel,
|
||||
messages: [{ role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }],
|
||||
stream: !!onChunk,
|
||||
};
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${config.lireek.openaiApiKey}`,
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`OpenAI error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
if (onChunk) {
|
||||
return await readSSE(resp, onChunk, (data) => data.choices?.[0]?.delta?.content || null, (data) => data.choices?.[0]?.delta?.reasoning_content || null);
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
return data.choices?.[0]?.message?.content || '';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
// llm/orchestration.ts — High-level lyric generation and refinement functions
|
||||
//
|
||||
// All prompt TEXT and prompt BUILDERS live in ../prompts.ts (the canonical
|
||||
// single source, shared with tools/mcp-lyricstudio). This file owns the
|
||||
// call orchestration: provider calls, JSON parsing, postprocessing, retries.
|
||||
|
||||
import * as slopDetector from '../slopDetector.js';
|
||||
import {
|
||||
GENERATION_SYSTEM_PROMPT,
|
||||
SONG_METADATA_SYSTEM_PROMPT,
|
||||
REFINEMENT_SYSTEM_PROMPT,
|
||||
TITLE_DERIVATION_PROMPT,
|
||||
buildMetadataPrompt,
|
||||
buildGenerationPrompt,
|
||||
buildRefinementPrompt,
|
||||
buildTitlePrompt,
|
||||
} from '../prompts.js';
|
||||
import type { LyricsProfile } from '../profilerService.js';
|
||||
import { withModelSuffix } from '../modelName.js';
|
||||
import type { GenerationResponse, ChunkCallback, CallOptions } from './types.js';
|
||||
import { getProvider } from './registry.js';
|
||||
import {
|
||||
stripThinkingBlocks, postprocessLyrics, fixSectionLabels,
|
||||
enforceLineCounts, fixAPrefix,
|
||||
estimateDuration
|
||||
} from './postprocess.js';
|
||||
|
||||
// Words banned from titles — enforced programmatically since prompt-only bans leak
|
||||
const BANNED_TITLE_WORDS = new Set([
|
||||
'glass', 'steel', 'plastic', 'concrete', 'midnight', 'mirror',
|
||||
'heavy', 'terminal', 'altar', 'confessional', 'ledger', 'gospel',
|
||||
'chrome', 'gilded', 'puppet', 'halo', 'protocol', 'eden', 'digital',
|
||||
'algorithm', 'code', 'circuit', 'grid', 'data', 'wire',
|
||||
'sanctuary', 'void', 'ethereal', 'neon', 'silhouette', 'static',
|
||||
'embers', 'fluorescent', 'shimmering', 'tapestry', 'weight',
|
||||
'skin', 'signal', 'platform',
|
||||
]);
|
||||
|
||||
// Structural title formula patterns that produce repetitive titles
|
||||
const BANNED_TITLE_PATTERNS: RegExp[] = [
|
||||
/^watch\s+(it|me|them|us|him|her)\b/i,
|
||||
/^let\s+it\s+(burn|fade|go|fall|bleed|break|rot|die|end)\b/i,
|
||||
/^burn\s+it\s+(all|down)\b/i,
|
||||
/^nothing\s+left/i,
|
||||
/^nowhere\s+left/i,
|
||||
];
|
||||
|
||||
/**
|
||||
* Check if a title contains banned words or matches banned patterns.
|
||||
* Returns an array of issues found (empty = title is clean).
|
||||
*/
|
||||
function validateTitle(title: string): string[] {
|
||||
const issues: string[] = [];
|
||||
const words = title.toLowerCase().split(/\W+/).filter(Boolean);
|
||||
for (const w of words) {
|
||||
if (BANNED_TITLE_WORDS.has(w)) issues.push(`banned word: "${w}"`);
|
||||
}
|
||||
for (const pat of BANNED_TITLE_PATTERNS) {
|
||||
if (pat.test(title)) issues.push(`banned pattern: ${pat.source}`);
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a unique nonce to a system prompt to bust oMLX's KV cache.
|
||||
* Without this, oMLX reuses cached prefix states for identical system prompts,
|
||||
* which can cause thinking models to skip reasoning on subsequent calls.
|
||||
*/
|
||||
function cacheBustPrompt(prompt: string): string {
|
||||
return `${prompt}\n\n(session ${Date.now()}-${Math.random().toString(36).slice(2, 8)})`;
|
||||
}
|
||||
|
||||
interface PlannedMetadata {
|
||||
subject: string;
|
||||
bpm: number;
|
||||
key: string;
|
||||
caption: string;
|
||||
duration: number;
|
||||
structure?: string;
|
||||
}
|
||||
|
||||
async function planSongMetadata(
|
||||
profile: LyricsProfile,
|
||||
usedSubjects: string[],
|
||||
usedBpms: number[],
|
||||
usedKeys: string[],
|
||||
usedDurations: number[],
|
||||
providerName: string,
|
||||
modelName: string,
|
||||
onChunk?: ChunkCallback,
|
||||
userSubject?: string,
|
||||
callOptions?: CallOptions
|
||||
): Promise<PlannedMetadata> {
|
||||
const provider = getProvider(providerName);
|
||||
const prompt = buildMetadataPrompt(profile, usedSubjects, usedBpms, usedKeys, usedDurations, userSubject);
|
||||
console.log('[LLM] Planning song metadata via', providerName, modelName);
|
||||
const responseJsonStr = await provider.call(cacheBustPrompt(SONG_METADATA_SYSTEM_PROMPT), prompt, modelName, onChunk, callOptions);
|
||||
const cleaned = stripThinkingBlocks(responseJsonStr);
|
||||
const cleanJson = cleaned.replace(/^```(?:json)?\s*|\s*```$/gm, '').trim();
|
||||
try {
|
||||
return JSON.parse(cleanJson);
|
||||
} catch (err) {
|
||||
const start = cleanJson.indexOf('{');
|
||||
if (start !== -1) {
|
||||
let depth = 0;
|
||||
for (let i = start; i < cleanJson.length; i++) {
|
||||
if (cleanJson[i] === '{') depth++;
|
||||
else if (cleanJson[i] === '}') { depth--; if (depth === 0) { try { return JSON.parse(cleanJson.slice(start, i + 1)); } catch {} break; } }
|
||||
}
|
||||
}
|
||||
console.error("Failed to parse metadata JSON:", cleanJson.slice(0, 300));
|
||||
return { subject: '', bpm: 0, key: '', caption: '', duration: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateLyricsStreaming(
|
||||
profile: LyricsProfile, providerName: string, model?: string,
|
||||
extraInstructions?: string, usedSubjects: string[] = [],
|
||||
usedBpms: number[] = [], usedKeys: string[] = [],
|
||||
usedTitles: string[] = [], usedDurations: number[] = [],
|
||||
onChunk?: ChunkCallback, onPhase?: (phase: string) => void,
|
||||
userSubject?: string, callOptions?: CallOptions
|
||||
): Promise<GenerationResponse> {
|
||||
const provider = getProvider(providerName);
|
||||
const effectiveModel = model || provider.defaultModel;
|
||||
|
||||
if (onPhase) onPhase("Planning song metadata…");
|
||||
let metadata: PlannedMetadata = { subject: '', bpm: 0, key: '', caption: '', duration: 0 };
|
||||
if (profile.song_subjects || (profile.themes && profile.themes.length) || userSubject) {
|
||||
try {
|
||||
metadata = await planSongMetadata(profile, usedSubjects, usedBpms, usedKeys, usedDurations, providerName, effectiveModel, onChunk, userSubject, callOptions);
|
||||
if (userSubject) metadata.subject = userSubject;
|
||||
console.log("Planned metadata:", metadata);
|
||||
} catch(e) { console.warn("Failed to plan metadata", e); }
|
||||
}
|
||||
if (userSubject && !metadata.subject) metadata.subject = userSubject;
|
||||
|
||||
if (metadata.subject) extraInstructions = `The song must be about: ${metadata.subject}\n\n${extraInstructions || ''}`;
|
||||
if (onPhase) onPhase("Writing lyrics…");
|
||||
const userPrompt = buildGenerationPrompt(profile, extraInstructions, metadata.duration, metadata.bpm, metadata.structure);
|
||||
let raw = await provider.call(cacheBustPrompt(GENERATION_SYSTEM_PROMPT), userPrompt, effectiveModel, onChunk, callOptions);
|
||||
|
||||
raw = stripThinkingBlocks(raw);
|
||||
raw = raw.replace(/<\|[a-z_]+\|>/g, '');
|
||||
raw = raw.replace(/\[?(System|User|Assistant)\]?:.*/gi, '');
|
||||
raw = raw.replace(/\s*\((?:Hook|You|Repeat|x\d|Refrain|Spoken|Whispered|Ad[- ]?lib|Echo)\)\s*/gi, '');
|
||||
raw = raw.replace(/ +$/gm, '');
|
||||
|
||||
const rawLines = raw.trim().split('\n');
|
||||
for (let i = 0; i < rawLines.length; i++) {
|
||||
const match = rawLines[i].match(/^(?:Title:\s*|#\s*)(.*)/i);
|
||||
if (match) {
|
||||
const rest = rawLines.slice(i + 1);
|
||||
while (rest.length && !rest[0].trim()) rest.shift();
|
||||
raw = rest.join('\n');
|
||||
break;
|
||||
}
|
||||
if (rawLines[i].trim().startsWith('[') || (rawLines[i].trim() && i > 2)) break;
|
||||
}
|
||||
|
||||
raw = postprocessLyrics(raw);
|
||||
raw = fixSectionLabels(raw);
|
||||
raw = fixAPrefix(raw);
|
||||
raw = enforceLineCounts(raw);
|
||||
|
||||
const slopResult = slopDetector.scanForSlop(raw);
|
||||
if (slopResult.ai_score > 0) {
|
||||
console.warn(`Generation slop scan: score=${slopResult.ai_score} severity=${slopResult.severity}`,
|
||||
'words:', slopResult.layers.blacklisted_words.found,
|
||||
'phrases:', slopResult.layers.blacklisted_phrases.found,
|
||||
'overuse:', slopResult.layers.overuse.found.map((o: any) => `${o.word}(${o.count}x)`).join(', ') || 'none',
|
||||
'hook_formulas:', slopResult.layers.hook_formulas.found.join(', ') || 'none');
|
||||
}
|
||||
|
||||
if (onPhase) onPhase("Choosing title…");
|
||||
let title = '';
|
||||
try {
|
||||
const titleUserPrompt = buildTitlePrompt(raw, profile.artist, profile.album, usedTitles);
|
||||
let titleRaw = await provider.call(cacheBustPrompt(TITLE_DERIVATION_PROMPT), titleUserPrompt, effectiveModel, onChunk, callOptions);
|
||||
titleRaw = stripThinkingBlocks(titleRaw).trim();
|
||||
titleRaw = titleRaw.replace(/^(?:Title:\s*|#\s*)/i, '').replace(/^["'`]|["'`]$/g, '').trim();
|
||||
title = titleRaw.split('\n')[0].trim();
|
||||
console.log('[LLM] Derived title:', title);
|
||||
|
||||
// Validate title against banned words/patterns
|
||||
const titleIssues = validateTitle(title);
|
||||
if (titleIssues.length) {
|
||||
console.warn(`[LLM] Title "${title}" failed validation: ${titleIssues.join(', ')}. Requesting re-derivation.`);
|
||||
// Re-derive with explicit rejection guidance
|
||||
const retryPrompt = [
|
||||
titleUserPrompt,
|
||||
`\nThe title "${title}" is REJECTED because: ${titleIssues.join(', ')}.`,
|
||||
'Choose a DIFFERENT title that avoids these issues. Return ONLY the new title:',
|
||||
].join('\n');
|
||||
try {
|
||||
let retryRaw = await provider.call(cacheBustPrompt(TITLE_DERIVATION_PROMPT), retryPrompt, effectiveModel, onChunk, callOptions);
|
||||
retryRaw = stripThinkingBlocks(retryRaw).trim();
|
||||
retryRaw = retryRaw.replace(/^(?:Title:\s*|#\s*)/i, '').replace(/^["'`]|["'`]$/g, '').trim();
|
||||
const retryTitle = retryRaw.split('\n')[0].trim();
|
||||
const retryIssues = validateTitle(retryTitle);
|
||||
if (!retryIssues.length) {
|
||||
console.log(`[LLM] Re-derived title: "${retryTitle}" (was: "${title}")`);
|
||||
title = retryTitle;
|
||||
} else {
|
||||
console.warn(`[LLM] Re-derived title "${retryTitle}" still failed: ${retryIssues.join(', ')}. Keeping original.`);
|
||||
}
|
||||
} catch (retryErr) {
|
||||
console.warn('[LLM] Title re-derivation failed:', retryErr);
|
||||
}
|
||||
}
|
||||
} catch (err) { console.warn('[LLM] Title derivation failed, falling back to empty:', err); }
|
||||
|
||||
// Tag the title with the model that wrote it, same as the MCP path does
|
||||
title = withModelSuffix(title, effectiveModel);
|
||||
|
||||
let duration = metadata.duration || 0;
|
||||
if (metadata.bpm > 0 && !duration) duration = estimateDuration(raw, metadata.bpm);
|
||||
|
||||
return {
|
||||
lyrics: raw, provider: providerName, model: effectiveModel, title,
|
||||
subject: metadata.subject, bpm: metadata.bpm, key: metadata.key,
|
||||
caption: metadata.caption, duration,
|
||||
system_prompt: GENERATION_SYSTEM_PROMPT, user_prompt: userPrompt
|
||||
};
|
||||
}
|
||||
|
||||
export async function refineLyricsStreaming(
|
||||
originalLyrics: string, artistName: string, title: string,
|
||||
providerName: string, model?: string, profile?: LyricsProfile,
|
||||
onChunk?: ChunkCallback
|
||||
): Promise<GenerationResponse> {
|
||||
const provider = getProvider(providerName);
|
||||
const effectiveModel = model || provider.defaultModel;
|
||||
const slopScan = slopDetector.scanForSlop(originalLyrics);
|
||||
const foundSlop = [...slopScan.layers.blacklisted_words.found, ...slopScan.layers.blacklisted_phrases.found];
|
||||
const userPrompt = buildRefinementPrompt(originalLyrics, artistName, title, profile, foundSlop);
|
||||
let raw = await provider.call(cacheBustPrompt(REFINEMENT_SYSTEM_PROMPT), userPrompt, effectiveModel, onChunk);
|
||||
|
||||
raw = stripThinkingBlocks(raw);
|
||||
raw = raw.replace(/<\|[a-z_]+\|>/g, '');
|
||||
raw = raw.replace(/\s*\((?:Hook|You|Repeat|x\d|Refrain|Spoken|Whispered|Ad[- ]?lib|Echo)\)\s*/gi, '');
|
||||
raw = raw.replace(/ +$/gm, '');
|
||||
|
||||
let refinedTitle = title;
|
||||
const rLines = raw.trim().split('\n');
|
||||
for (let i = 0; i < rLines.length; i++) {
|
||||
const match = rLines[i].match(/^(?:Title:\s*|#\s*)(.*)/i);
|
||||
if (match) {
|
||||
refinedTitle = match[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
const rest = rLines.slice(i + 1);
|
||||
while (rest.length && !rest[0].trim()) rest.shift();
|
||||
raw = rest.join('\n');
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
raw = postprocessLyrics(raw);
|
||||
raw = fixSectionLabels(raw);
|
||||
raw = fixAPrefix(raw);
|
||||
raw = enforceLineCounts(raw);
|
||||
|
||||
const slopResult = slopDetector.scanForSlop(raw);
|
||||
if (slopResult.ai_score > 0) {
|
||||
console.warn(`Refinement slop scan: score=${slopResult.ai_score} severity=${slopResult.severity}`,
|
||||
'words:', slopResult.layers.blacklisted_words.found,
|
||||
'phrases:', slopResult.layers.blacklisted_phrases.found,
|
||||
'overuse:', slopResult.layers.overuse.found.map((o: any) => `${o.word}(${o.count}x)`).join(', ') || 'none',
|
||||
'hook_formulas:', slopResult.layers.hook_formulas.found.join(', ') || 'none');
|
||||
}
|
||||
|
||||
refinedTitle = withModelSuffix(refinedTitle, effectiveModel);
|
||||
|
||||
return {
|
||||
lyrics: raw, provider: providerName, model: effectiveModel, title: refinedTitle,
|
||||
subject: '', bpm: 0, key: '', caption: '', duration: 0,
|
||||
system_prompt: REFINEMENT_SYSTEM_PROMPT, user_prompt: userPrompt
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
// llm/postprocess.ts — Lyric post-processing pipeline (ported from HOT-Step 9000)
|
||||
|
||||
/** Strip thinking/reasoning blocks from LLM output */
|
||||
export function stripThinkingBlocks(text: string): string {
|
||||
let result = text.replace(/<think>[\s\S]*?<\/think>/g, '');
|
||||
result = result.replace(/<analysis>[\s\S]*?<\/analysis>/g, '');
|
||||
result = result.replace(/<reasoning>[\s\S]*?<\/reasoning>/g, '');
|
||||
result = result.replace(/<reflection>[\s\S]*?<\/reflection>/g, '');
|
||||
result = result.replace(/<thought>[\s\S]*?<\/thought>/g, '');
|
||||
result = result.replace(/<\|channel>thought[\s\S]*?<channel\|>/g, '');
|
||||
result = result.replace(/<(?:think|analysis|reasoning|reflection|thought)>[\s\S]*/g, '');
|
||||
result = result.replace(/<\|channel>thought[\s\S]*/g, '');
|
||||
const cotMatch = result.match(/^(?:\s*\*+\s*)?(?:Thinking Process|Thought Process|Thinking|Reasoning):\s*[\s\S]*?(?:---|[*]{3,}|={3,})\s*/i);
|
||||
if (cotMatch) result = result.slice(cotMatch[0].length);
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
const SECTION_KEYWORDS = [
|
||||
'Intro', 'Verse', 'Pre-Chorus', 'Chorus', 'Post-Chorus',
|
||||
'Bridge', 'Interlude', 'Outro', 'Hook', 'Refrain',
|
||||
];
|
||||
|
||||
export const SECTION_LINE_RE = new RegExp(
|
||||
'^\\[?(' + SECTION_KEYWORDS.map(k => k.replace(/[-/]/g, '\\$&')).join('|') + ')\\s*(\\d*)\\]?\\s*$',
|
||||
'i'
|
||||
);
|
||||
|
||||
const PUNCTUATION_ENDINGS = new Set('.,!?;:-…)"\'');
|
||||
|
||||
export function postprocessLyrics(text: string): string {
|
||||
const resultLines: string[] = [];
|
||||
for (const line of text.split('\n')) {
|
||||
const stripped = line.trim();
|
||||
if (!stripped) { resultLines.push(''); continue; }
|
||||
const m = SECTION_LINE_RE.exec(stripped);
|
||||
if (m) {
|
||||
const sectionName = m[1].charAt(0).toUpperCase() + m[1].slice(1).toLowerCase();
|
||||
const sectionNum = m[2];
|
||||
resultLines.push(sectionNum ? `[${sectionName} ${sectionNum}]` : `[${sectionName}]`);
|
||||
continue;
|
||||
}
|
||||
if (/^\[.+\]$/.test(stripped)) { resultLines.push(stripped); continue; }
|
||||
if (stripped && !PUNCTUATION_ENDINGS.has(stripped[stripped.length - 1])) {
|
||||
resultLines.push(stripped + ',');
|
||||
} else {
|
||||
resultLines.push(stripped);
|
||||
}
|
||||
}
|
||||
return resultLines.join('\n');
|
||||
}
|
||||
|
||||
export function fixSectionLabels(text: string): string {
|
||||
const INVALID_TO_VALID: Record<string, string> = {
|
||||
'x': 'Interlude', 'breakdown': 'Bridge', 'drop': 'Chorus',
|
||||
'solo': 'Interlude', 'hook': 'Chorus', 'rap': 'Verse', 'spoken': 'Verse',
|
||||
};
|
||||
const lines = text.split('\n');
|
||||
const result: string[] = [];
|
||||
const sectionHeaders: { lineIdx: number; header: string }[] = [];
|
||||
for (const line of lines) {
|
||||
const stripped = line.trim();
|
||||
const m = stripped.match(/^\[(.+?)(?:\s+\d+)?\]$/);
|
||||
if (m) {
|
||||
let label = m[1].trim().toLowerCase();
|
||||
let newStripped = stripped;
|
||||
if (INVALID_TO_VALID[label]) {
|
||||
const newLabel = INVALID_TO_VALID[label];
|
||||
const numMatch = stripped.match(/\d+/);
|
||||
newStripped = numMatch ? `[${newLabel} ${numMatch[0]}]` : `[${newLabel}]`;
|
||||
}
|
||||
sectionHeaders.push({ lineIdx: result.length, header: newStripped });
|
||||
result.push(newStripped);
|
||||
} else {
|
||||
result.push(stripped.startsWith('[') && stripped.endsWith(']') ? stripped : line);
|
||||
}
|
||||
}
|
||||
const bridgeIndices = sectionHeaders.map((h, i) => ({ i, h })).filter(x => x.h.header.toLowerCase().includes('bridge'));
|
||||
const chorusExists = sectionHeaders.some(h => h.header.toLowerCase().includes('chorus'));
|
||||
if (!chorusExists && bridgeIndices.length >= 2) {
|
||||
for (const bi of bridgeIndices.slice(0, -1)) {
|
||||
result[sectionHeaders[bi.i].lineIdx] = '[Chorus]';
|
||||
}
|
||||
}
|
||||
return result.join('\n');
|
||||
}
|
||||
|
||||
// Musical phrases resolve in even numbers of lines, so odd-length verses and
|
||||
// choruses (5, 7, 9 lines) clash with the music model's phrasing. Rather than
|
||||
// forcing every section to exactly 4/8 lines (the old behaviour, which deleted
|
||||
// up to 3 lines of content), only trim ONE line when the count is odd.
|
||||
// Even counts of any reasonable size pass through untouched.
|
||||
export function enforceLineCounts(text: string): string {
|
||||
const sections: { header: string; lines: string[] }[] = [];
|
||||
let currentHeader = '';
|
||||
let currentLines: string[] = [];
|
||||
for (const line of text.split('\n')) {
|
||||
const stripped = line.trim();
|
||||
if (/^\[.+\]$/.test(stripped)) {
|
||||
if (currentHeader || currentLines.length) sections.push({ header: currentHeader, lines: currentLines });
|
||||
currentHeader = stripped;
|
||||
currentLines = [];
|
||||
} else { currentLines.push(line); }
|
||||
}
|
||||
if (currentHeader || currentLines.length) sections.push({ header: currentHeader, lines: currentLines });
|
||||
const resultParts: string[] = [];
|
||||
for (const { header, lines } of sections) {
|
||||
const lyricLines = lines.filter(l => l.trim());
|
||||
const count = lyricLines.length;
|
||||
const headerLower = header.toLowerCase();
|
||||
const isVerse = headerLower.includes('verse');
|
||||
const isChorus = headerLower.includes('chorus') || headerLower.includes('hook');
|
||||
let target: number | null = null;
|
||||
if ((isVerse || isChorus) && count >= 3 && count % 2 !== 0) target = count - 1;
|
||||
let finalLines = lines;
|
||||
if (target !== null && target < count) {
|
||||
let kept = 0; finalLines = [];
|
||||
for (const l of lines) {
|
||||
if (l.trim()) { if (kept < target) { finalLines.push(l); kept++; } }
|
||||
else { if (kept < target) finalLines.push(l); }
|
||||
}
|
||||
}
|
||||
if (header) resultParts.push(header);
|
||||
resultParts.push(...finalLines);
|
||||
}
|
||||
return resultParts.join('\n');
|
||||
}
|
||||
|
||||
const BAD_A_PREFIX_RE = /\ba-(?!\w+ing\b)(?!\w+in'\b)/gi;
|
||||
export function fixAPrefix(text: string): string { return text.replace(BAD_A_PREFIX_RE, ''); }
|
||||
|
||||
export function stripLyricQuotes(text: string): string { return text.replace(/'[^']{4,}'/g, '[quote removed]'); }
|
||||
|
||||
export function estimateDuration(lyrics: string, bpm: number): number {
|
||||
if (!lyrics.trim() || bpm <= 0) return 0;
|
||||
const barDuration = 240.0 / Math.max(bpm, 40);
|
||||
const lines = lyrics.trim().split('\n');
|
||||
let sectionCount = 0, lyricLineCount = 0;
|
||||
for (const line of lines) {
|
||||
const stripped = line.trim();
|
||||
if (!stripped) continue;
|
||||
if (SECTION_LINE_RE.test(stripped) || (stripped.startsWith('[') && stripped.endsWith(']'))) sectionCount++;
|
||||
else lyricLineCount++;
|
||||
}
|
||||
return Math.max(90, Math.min(Math.floor(lyricLineCount * 3.5 + Math.max(sectionCount - 1, 0) * 4 * barDuration), 360));
|
||||
}
|
||||
|
||||
// selectBestBlueprint was removed — structure selection now lives in
|
||||
// ../prompts.ts (pickBlueprint), which samples from the artist's observed
|
||||
// blueprints instead of deterministically picking the same one every time.
|
||||
@@ -0,0 +1,59 @@
|
||||
// llm/registry.ts — Provider registry and lookup
|
||||
|
||||
import { LLMProvider } from './base.js';
|
||||
import type { ProviderInfo } from './types.js';
|
||||
import { GeminiProvider } from './gemini.js';
|
||||
import { OpenAIProvider } from './openai.js';
|
||||
import { AnthropicProvider } from './anthropic.js';
|
||||
import { OllamaProvider } from './ollama.js';
|
||||
import { LMStudioProvider } from './lmstudio.js';
|
||||
import { UnslothProvider } from './unsloth.js';
|
||||
import { OpenAICompatProvider } from './openai-compat.js';
|
||||
import { LlamaCppProvider } from './llamacpp.js';
|
||||
|
||||
const providers: Record<string, LLMProvider> = {
|
||||
gemini: new GeminiProvider(),
|
||||
openai: new OpenAIProvider(),
|
||||
anthropic: new AnthropicProvider(),
|
||||
ollama: new OllamaProvider(),
|
||||
lmstudio: new LMStudioProvider(),
|
||||
unsloth: new UnslothProvider(),
|
||||
llamacpp: new LlamaCppProvider(),
|
||||
'openai-compat': new OpenAICompatProvider(),
|
||||
};
|
||||
|
||||
export function getProvider(name: string): LLMProvider {
|
||||
const provider = providers[name];
|
||||
if (!provider) throw new Error(`Unknown LLM provider: ${name}`);
|
||||
return provider;
|
||||
}
|
||||
|
||||
export async function listProviders(): Promise<ProviderInfo[]> {
|
||||
const PROVIDER_TIMEOUT_MS = 5000;
|
||||
|
||||
const promises = Object.values(providers).map(async (p): Promise<ProviderInfo> => {
|
||||
try {
|
||||
if (p instanceof GeminiProvider || p instanceof OllamaProvider || p instanceof LMStudioProvider || p instanceof UnslothProvider || p instanceof LlamaCppProvider || p instanceof OpenAICompatProvider) {
|
||||
// Race against a timeout so one dead provider can't block the rest
|
||||
const info = await Promise.race([
|
||||
p.toInfoAsync(),
|
||||
new Promise<ProviderInfo>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`${p.name} timed out`)), PROVIDER_TIMEOUT_MS)
|
||||
),
|
||||
]);
|
||||
return info;
|
||||
} else {
|
||||
return p.toInfo();
|
||||
}
|
||||
} catch (err: any) {
|
||||
console.warn(`[LLM Registry] Provider ${p.name} failed: ${err.message}`);
|
||||
// Return the provider as unavailable rather than dropping it
|
||||
return { ...p.toInfo(), available: false, models: p.defaultModel ? [p.defaultModel] : [] };
|
||||
}
|
||||
});
|
||||
|
||||
const settled = await Promise.allSettled(promises);
|
||||
return settled
|
||||
.filter((r): r is PromiseFulfilledResult<ProviderInfo> => r.status === 'fulfilled')
|
||||
.map(r => r.value);
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// llm/types.ts — Shared types for the LLM provider system
|
||||
|
||||
export interface ProviderInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
available: boolean;
|
||||
models: string[];
|
||||
default_model: string;
|
||||
}
|
||||
|
||||
export interface GenerationResponse {
|
||||
lyrics: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
title: string;
|
||||
subject: string;
|
||||
bpm: number;
|
||||
key: string;
|
||||
caption: string;
|
||||
duration: number;
|
||||
system_prompt: string;
|
||||
user_prompt: string;
|
||||
}
|
||||
|
||||
export type ChunkCallback = (chunk: string) => void;
|
||||
|
||||
export interface CallOptions {
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
/**
|
||||
* Best-effort "answer without reasoning" for local thinking models.
|
||||
* There is NO universal off-switch across runtimes, so providers layer
|
||||
* every mechanism that is harmless where unsupported:
|
||||
* - `reasoning_effort: 'none'` (LM Studio — EMPIRICALLY VERIFIED 2026-07-09
|
||||
* on Qwen3.6: reasoning drops to zero; gemma accepts it harmlessly)
|
||||
* - `/no_think` soft switch appended to the system prompt (older Qwen3;
|
||||
* Qwen3.5+ dropped it — verified ignored on Qwen3.6)
|
||||
* - `chat_template_kwargs: { enable_thinking: false }` (llama.cpp server;
|
||||
* LM Studio ignores it — verified)
|
||||
* - `think: false` (Ollama native)
|
||||
* Models with no supported mechanism still think; stripThinkingBlocks()
|
||||
* downstream keeps the output clean either way.
|
||||
*/
|
||||
noThink?: boolean;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
// Global skip thinking signal
|
||||
export let skipThinkingSignal = false;
|
||||
export function setSkipThinking() {
|
||||
skipThinkingSignal = true;
|
||||
console.log('[LLM] Skip-thinking signal received');
|
||||
}
|
||||
export function resetSkipThinking() {
|
||||
skipThinkingSignal = false;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// llm/unsloth.ts — Unsloth Studio provider
|
||||
|
||||
import { config } from '../../../config.js';
|
||||
import { LLMProvider, readSSE } from './base.js';
|
||||
import type { ProviderInfo, ChunkCallback } from './types.js';
|
||||
|
||||
export class UnslothProvider extends LLMProvider {
|
||||
id = 'unsloth';
|
||||
name = 'Unsloth Studio';
|
||||
get defaultModel() { return config.lireek.unslothModel; }
|
||||
|
||||
private cachedToken = '';
|
||||
private tokenExpiry = 0;
|
||||
|
||||
isAvailable() { return !!config.lireek.unslothUsername && !!config.lireek.unslothPassword; }
|
||||
|
||||
private async authenticate(): Promise<string> {
|
||||
const now = Date.now() / 1000;
|
||||
if (this.cachedToken && now < this.tokenExpiry - 60) return this.cachedToken;
|
||||
|
||||
const payload = { email: config.lireek.unslothUsername, password: config.lireek.unslothPassword };
|
||||
const resp = await fetch(`${config.lireek.unslothBaseUrl}/api/auth/login`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
|
||||
// Unsloth also sometimes takes username instead of email, trying generic handling if this fails
|
||||
if (!resp.ok) {
|
||||
const payload2 = { username: config.lireek.unslothUsername, password: config.lireek.unslothPassword };
|
||||
const resp2 = await fetch(`${config.lireek.unslothBaseUrl}/api/auth/login`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload2)
|
||||
});
|
||||
if (!resp2.ok) throw new Error("Failed to authenticate with Unsloth");
|
||||
const data = await resp2.json();
|
||||
this.cachedToken = data.access_token || data.token;
|
||||
} else {
|
||||
const data = await resp.json();
|
||||
this.cachedToken = data.access_token || data.token;
|
||||
}
|
||||
|
||||
try {
|
||||
const payloadB64 = this.cachedToken.split('.')[1];
|
||||
const decoded = JSON.parse(Buffer.from(payloadB64, 'base64').toString());
|
||||
this.tokenExpiry = decoded.exp || (now + 3600);
|
||||
} catch {
|
||||
this.tokenExpiry = now + 3600;
|
||||
}
|
||||
return this.cachedToken;
|
||||
}
|
||||
|
||||
private async getLocalModels(): Promise<string[]> {
|
||||
try {
|
||||
const token = await this.authenticate();
|
||||
const resp = await fetch(`${config.lireek.unslothBaseUrl}/v1/models`, {
|
||||
headers: { 'Authorization': `Bearer ${token}` },
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
if (!resp.ok) return [];
|
||||
const data = await resp.json();
|
||||
this.availableModels = data.data?.map((m: any) => m.id).sort().reverse() || [];
|
||||
return this.availableModels;
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
async toInfoAsync(): Promise<ProviderInfo> {
|
||||
const models = await this.getLocalModels();
|
||||
return {
|
||||
...this.toInfo(),
|
||||
models: models.length ? models : (this.defaultModel ? [this.defaultModel] : []),
|
||||
default_model: models.length ? models[0] : this.defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
async call(systemPrompt: string, userPrompt: string, model?: string, onChunk?: ChunkCallback): Promise<string> {
|
||||
const token = await this.authenticate();
|
||||
const modelName = model || (await this.getLocalModels())[0] || this.defaultModel;
|
||||
if (!modelName) throw new Error("No models loaded in Unsloth Studio");
|
||||
|
||||
const url = `${config.lireek.unslothBaseUrl}/v1/chat/completions`;
|
||||
const payload = {
|
||||
model: modelName,
|
||||
messages: [{ role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }],
|
||||
stream: true, // Unsloth often requires stream: true
|
||||
};
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${token}`
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(300_000),
|
||||
});
|
||||
|
||||
if (!resp.ok) throw new Error(`Unsloth error: ${resp.status} ${await resp.text()}`);
|
||||
|
||||
return await readSSE(resp, onChunk || (() => {}), (data) => data.choices?.[0]?.delta?.content || null, (data) => data.choices?.[0]?.delta?.reasoning_content || null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// llmService.ts — Barrel re-export from the modular llm/ directory
|
||||
//
|
||||
// This file preserves the original import path (./llmService) for existing
|
||||
// consumers. All implementation has been decomposed into llm/*.ts modules.
|
||||
//
|
||||
// New code should import from './llm/index.js' directly.
|
||||
|
||||
export {
|
||||
type ProviderInfo,
|
||||
type GenerationResponse,
|
||||
type ChunkCallback,
|
||||
type CallOptions,
|
||||
skipThinkingSignal,
|
||||
setSkipThinking,
|
||||
resetSkipThinking,
|
||||
LLMProvider,
|
||||
getProvider,
|
||||
listProviders,
|
||||
stripThinkingBlocks,
|
||||
postprocessLyrics,
|
||||
fixSectionLabels,
|
||||
enforceLineCounts,
|
||||
fixAPrefix,
|
||||
stripLyricQuotes,
|
||||
estimateDuration,
|
||||
generateLyricsStreaming,
|
||||
refineLyricsStreaming,
|
||||
} from './llm/index.js';
|
||||
@@ -0,0 +1,47 @@
|
||||
// modelName.ts — model-name prettifying + title suffixing ("Song Name - Fable 5")
|
||||
//
|
||||
// CANONICAL single source, shared with tools/mcp-lyricstudio (which imports it
|
||||
// directly — it runs from TS source via tsx). The suffix append is deterministic
|
||||
// code, NOT part of any LLM prompt, so titles stay clean for validation and the
|
||||
// suffix can't be hallucinated.
|
||||
|
||||
/** Turn a model id like "claude-fable-5" / "claude-opus-4-8" into "Fable 5" / "Opus 4.8",
|
||||
* or an OpenAI-compatible / LM Studio id like
|
||||
* "lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF" into "Meta Llama 3.1 8B Instruct".
|
||||
* Friendly names ("Fable 5", "Gemini 3 Pro") pass through unchanged. */
|
||||
export function prettifyModel(raw: string): string {
|
||||
let s = raw.trim();
|
||||
if (!s) return s;
|
||||
if (/\s/.test(s) && !/[-_/]/.test(s)) return s; // already a friendly name
|
||||
const slash = s.lastIndexOf('/'); // org/repo paths → repo part
|
||||
if (slash >= 0) s = s.slice(slash + 1);
|
||||
s = s.replace(/^(us\.)?(anthropic[./])?(claude-)?/i, '');
|
||||
s = s.replace(/\.gguf$/i, ''); // file-style ids
|
||||
s = s.replace(/[-_.]gguf$/i, ''); // "...-GGUF" repo suffix
|
||||
s = s.replace(/[-_.](i?q\d+(?:_[a-z0-9]+)*|f16|f32|bf16|fp16)$/i, ''); // quant tag e.g. -Q4_K_M
|
||||
s = s.replace(/[-.]?\d{8}$/, ''); // date suffix e.g. -20251001
|
||||
s = s.replace(/[-.]?v\d+(?:[.:]\d+)*$/i, ''); // version suffix: v1, v0.3, v1:0
|
||||
const parts = s.split(/[-_]/).filter(Boolean);
|
||||
const out: string[] = [];
|
||||
for (const part of parts) {
|
||||
// Join consecutive single-digit segments as a version: opus 4 8 → opus 4.8
|
||||
if (/^\d$/.test(part) && out.length && /^\d+(\.\d+)*$/.test(out[out.length - 1])) {
|
||||
out[out.length - 1] += `.${part}`;
|
||||
} else {
|
||||
out.push(/^\d/.test(part) ? part : part.charAt(0).toUpperCase() + part.slice(1));
|
||||
}
|
||||
}
|
||||
// Community model names can be absurdly long
|
||||
// ("Qwen3.6-27B-Fable-Fusion-711-Uncensored-Heretic-NM-DAU-NEO-MAX-MTP-GGUF");
|
||||
// the first few tokens identify the model, the rest is noise in a title suffix.
|
||||
return out.slice(0, 4).join(' ') || raw.trim();
|
||||
}
|
||||
|
||||
/** Append " - <Model>" to a title unless it already carries that suffix (or is empty). */
|
||||
export function withModelSuffix(title: string, model?: string): string {
|
||||
const t = title.trim();
|
||||
if (!t || !model) return t;
|
||||
const pretty = prettifyModel(model);
|
||||
if (!pretty || t.toLowerCase().endsWith(`- ${pretty.toLowerCase()}`)) return t;
|
||||
return `${t} - ${pretty}`;
|
||||
}
|
||||
@@ -0,0 +1,838 @@
|
||||
import * as cmuDictRaw from 'cmu-pronouncing-dictionary';
|
||||
import type { ChunkCallback } from './llmService.js';
|
||||
|
||||
export interface SongLyrics {
|
||||
title: string;
|
||||
album?: string;
|
||||
lyrics: string;
|
||||
}
|
||||
|
||||
export interface LyricsProfile {
|
||||
artist_id?: number;
|
||||
artist: string;
|
||||
album?: string;
|
||||
/** Measured audio facts from a Training Studio export (bpm range, keys,
|
||||
* genre, caption examples) — null/absent for plain Genius-fetched sets. */
|
||||
audio_enrichment?: AlbumEnrichment | null;
|
||||
themes: string[];
|
||||
common_subjects: string[];
|
||||
rhyme_schemes: string[];
|
||||
avg_verse_lines: number;
|
||||
avg_chorus_lines: number;
|
||||
vocabulary_notes?: string;
|
||||
tone_and_mood?: string;
|
||||
structural_patterns?: string;
|
||||
additional_notes?: string;
|
||||
raw_summary?: string;
|
||||
song_subjects?: Record<string, string>;
|
||||
subject_categories?: string[];
|
||||
repetition_stats?: {
|
||||
chorus_repetition_pct?: number;
|
||||
verse_repetition_pct?: number;
|
||||
cross_section_repeats?: number;
|
||||
pattern?: string;
|
||||
hook_examples?: string[];
|
||||
};
|
||||
structure_blueprints?: string[];
|
||||
perspective?: string;
|
||||
meter_stats?: {
|
||||
avg_syllables_per_line?: number;
|
||||
syllable_std_dev?: number;
|
||||
avg_words_per_line?: number;
|
||||
line_length_range?: string;
|
||||
line_length_variation?: {
|
||||
histogram?: Record<string, number>;
|
||||
per_section?: Record<string, any>;
|
||||
short_line_examples?: string[];
|
||||
long_line_examples?: string[];
|
||||
};
|
||||
};
|
||||
vocabulary_stats?: {
|
||||
type_token_ratio?: number;
|
||||
total_words?: number;
|
||||
unique_words?: number;
|
||||
contraction_pct?: number;
|
||||
profanity_pct?: number;
|
||||
distinctive_words?: string[];
|
||||
};
|
||||
representative_excerpts?: string[];
|
||||
narrative_techniques?: string;
|
||||
imagery_patterns?: string;
|
||||
signature_devices?: string;
|
||||
emotional_arc?: string;
|
||||
rhyme_quality?: Record<string, number>;
|
||||
examples?: any[];
|
||||
style_caption?: string;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
import * as llmService from './llmService.js';
|
||||
import {
|
||||
PROFILE_PROMPT_1,
|
||||
PROFILE_PROMPT_2,
|
||||
PROFILE_PROMPT_3,
|
||||
STYLE_CAPTION_PROMPT,
|
||||
SUBJECT_ANALYSIS_PROMPT,
|
||||
buildProfilePrompt,
|
||||
buildSubjectAnalysisPrompt,
|
||||
computeAlbumEnrichment,
|
||||
type AlbumEnrichment,
|
||||
} from './prompts.js';
|
||||
|
||||
// The imported dict is a default export depending on interop.
|
||||
const CMU_DICT: Record<string, string> = (cmuDictRaw as any).default || cmuDictRaw;
|
||||
|
||||
// ── Robust JSON extraction ────────────────────────────────────────────────────
|
||||
|
||||
function repairJson(text: string): string {
|
||||
// Fix missing commas between string array elements
|
||||
let fixed = text.replace(/"\s*\n(\s*")/g, '",\n$1');
|
||||
// Fix missing commas after ] when followed by " (next key)
|
||||
fixed = fixed.replace(/\]\s*\n(\s*")/g, '],\n$1');
|
||||
// Fix stray } after ]
|
||||
fixed = fixed.replace(/\]\s*\n\s*},?\s*\n(\s*")/g, '],\n$1');
|
||||
// Fix missing commas between } and " or {
|
||||
fixed = fixed.replace(/}\s*\n(\s*["{])/g, '},\n$1');
|
||||
// Fix trailing commas before ] or }
|
||||
fixed = fixed.replace(/,(\s*[}\]])/g, '$1');
|
||||
return fixed;
|
||||
}
|
||||
|
||||
function extractJson(text: string): Record<string, any> | null {
|
||||
// Strategy 0: strip reasoning model <think> blocks
|
||||
let stripped = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
|
||||
if (stripped.includes('<think>')) {
|
||||
stripped = stripped.replace(/<think>[\s\S]*/g, '').trim();
|
||||
}
|
||||
|
||||
// Strategy 1: direct parse
|
||||
try { return JSON.parse(stripped); } catch (e) {}
|
||||
|
||||
// Strategy 2: strip code fences
|
||||
let clean = stripped.replace(/^```(?:json)?\s*/m, '').replace(/\s*```$/m, '').trim();
|
||||
try { return JSON.parse(clean); } catch (e) {}
|
||||
|
||||
// Strategy 3: find outermost { ... }
|
||||
const firstBrace = clean.indexOf('{');
|
||||
const lastBrace = clean.lastIndexOf('}');
|
||||
let candidate: string | null = null;
|
||||
if (firstBrace !== -1 && lastBrace > firstBrace) {
|
||||
candidate = clean.slice(firstBrace, lastBrace + 1);
|
||||
try { return JSON.parse(candidate); } catch (e) {}
|
||||
}
|
||||
|
||||
// Strategy 4: repair
|
||||
const toRepair = candidate || clean;
|
||||
const repaired = repairJson(toRepair);
|
||||
try { return JSON.parse(repaired); } catch (e) {}
|
||||
|
||||
// Strategy 5: brute-force
|
||||
let lines = repaired.split('\n');
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
let found = false;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const s = lines[i].trim();
|
||||
if ((s === '}' || s === '},') && i > 0 && i < lines.length - 1) {
|
||||
let trialLines = [...lines];
|
||||
trialLines.splice(i, 1);
|
||||
try {
|
||||
return JSON.parse(trialLines.join('\n'));
|
||||
} catch (e) {
|
||||
lines = trialLines;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found) break;
|
||||
}
|
||||
|
||||
console.warn("All JSON extraction strategies failed for response");
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── CMU Pronouncing Dictionary ────────────────────────────────────────────────
|
||||
|
||||
function getPhones(word: string): string[] | null {
|
||||
const cleanWord = word.toLowerCase().replace(/['".,!?;:-]/g, '');
|
||||
const entry = CMU_DICT[cleanWord];
|
||||
if (entry) return entry.split(' ');
|
||||
return null;
|
||||
}
|
||||
|
||||
function getVowelTail(phones: string[], n: int = 3): string[] {
|
||||
let result: string[] = [];
|
||||
for (let i = phones.length - 1; i >= 0; i--) {
|
||||
let clean = phones[i].replace(/\d/g, ''); // strip stress marker
|
||||
result.push(clean);
|
||||
if (result.length >= n) break;
|
||||
}
|
||||
return result.reverse();
|
||||
}
|
||||
|
||||
function rhymeQuality(wordA: string, wordB: string): string {
|
||||
if (wordA === wordB) return 'perfect';
|
||||
|
||||
const phonesA = getPhones(wordA);
|
||||
const phonesB = getPhones(wordB);
|
||||
|
||||
if (!phonesA || !phonesB) {
|
||||
const a = wordA.toLowerCase(), b = wordB.toLowerCase();
|
||||
if (a.length >= 2 && b.length >= 2 && a.slice(-2) === b.slice(-2)) return 'slant';
|
||||
return 'none';
|
||||
}
|
||||
|
||||
const tailA = getVowelTail(phonesA, 3);
|
||||
const tailB = getVowelTail(phonesB, 3);
|
||||
|
||||
if (tailA.join('-') === tailB.join('-')) return 'perfect';
|
||||
|
||||
const tailA2 = getVowelTail(phonesA, 2);
|
||||
const tailB2 = getVowelTail(phonesB, 2);
|
||||
if (tailA2.join('-') === tailB2.join('-')) return 'perfect';
|
||||
|
||||
const vowelsA = phonesA.filter(p => /\d/.test(p)).map(p => p.replace(/\d/g, ''));
|
||||
const vowelsB = phonesB.filter(p => /\d/.test(p)).map(p => p.replace(/\d/g, ''));
|
||||
|
||||
if (vowelsA.length && vowelsB.length && vowelsA[vowelsA.length - 1] === vowelsB[vowelsB.length - 1]) {
|
||||
if (tailA2[0] === tailB2[0] || tailA.some(p => tailB.includes(p))) return 'slant';
|
||||
return 'assonance';
|
||||
}
|
||||
|
||||
if (tailA.filter(p => tailB.includes(p)).length >= 2) return 'slant';
|
||||
return 'none';
|
||||
}
|
||||
|
||||
function countSyllablesHeuristic(word: string): int {
|
||||
let w = word.toLowerCase().replace(/['".,!?;:-]/g, '');
|
||||
if (!w) return 0;
|
||||
const matches = w.match(/[aeiouy]+/g);
|
||||
let count = matches ? matches.length : 0;
|
||||
if (w.endsWith('e') && count > 1) count--;
|
||||
return Math.max(count, 1);
|
||||
}
|
||||
|
||||
function countSyllables(word: string): int {
|
||||
const phones = getPhones(word);
|
||||
if (phones) {
|
||||
const cmu = phones.filter(p => /\d/.test(p)).length;
|
||||
if (cmu > 0) return cmu;
|
||||
}
|
||||
return countSyllablesHeuristic(word);
|
||||
}
|
||||
|
||||
// ── Section parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
const SECTION_HEADER_RE = /^\[(.+?)\]$/i;
|
||||
|
||||
// ORDER IS SIGNIFICANT. Matching is substring-based, so the most specific patterns
|
||||
// must come first: 'Pre-Chorus' and 'Post-Chorus' both contain 'chorus', and were
|
||||
// previously swallowed by the generic 'chorus' entry — meaning PC and POC could
|
||||
// never be produced by the analyser at all.
|
||||
//
|
||||
// The instrumental catch-alls sit last on purpose, so labels that name a musical
|
||||
// role ('Instrumental Bridge', 'Instrumental Outro') keep that role, while a bare
|
||||
// 'Instrumental' or 'Instrumental Break' falls through to Interlude.
|
||||
const SECTION_LABEL_PATTERNS: [string, string][] = [
|
||||
['pre-chorus', 'PC'], ['pre chorus', 'PC'], ['prechorus', 'PC'],
|
||||
['post-chorus', 'POC'], ['post chorus', 'POC'], ['postchorus', 'POC'],
|
||||
['verse', 'V'],
|
||||
['chorus', 'C'], ['hook', 'C'], ['refrain', 'C'],
|
||||
['bridge', 'B'],
|
||||
['intro', 'I'],
|
||||
['outro', 'O'],
|
||||
['interlude', 'IL'],
|
||||
['break', 'IL'],
|
||||
['instrumental', 'IL'],
|
||||
['solo', 'IL'],
|
||||
];
|
||||
|
||||
function normaliseSectionLabel(rawLabel: string): string {
|
||||
const lower = rawLabel.toLowerCase().trim();
|
||||
for (const [key, code] of SECTION_LABEL_PATTERNS) {
|
||||
if (lower.includes(key)) return code;
|
||||
}
|
||||
return 'X';
|
||||
}
|
||||
|
||||
function splitIntoSections(lyrics: string): { label: string, lines: string[] }[] {
|
||||
const sections: { label: string, lines: string[] }[] = [];
|
||||
let currentLabel = 'X';
|
||||
let currentLines: string[] = [];
|
||||
|
||||
const lines = lyrics.split('\n');
|
||||
for (const line of lines) {
|
||||
const stripped = line.trim();
|
||||
const match = SECTION_HEADER_RE.exec(stripped);
|
||||
if (match) {
|
||||
if (currentLines.length) sections.push({ label: currentLabel, lines: currentLines });
|
||||
currentLabel = normaliseSectionLabel(match[1]);
|
||||
currentLines = [];
|
||||
} else if (stripped === '') {
|
||||
if (currentLines.length) {
|
||||
sections.push({ label: currentLabel, lines: currentLines });
|
||||
currentLines = [];
|
||||
}
|
||||
} else {
|
||||
currentLines.push(stripped);
|
||||
}
|
||||
}
|
||||
if (currentLines.length) sections.push({ label: currentLabel, lines: currentLines });
|
||||
return sections;
|
||||
}
|
||||
|
||||
// ── Analysis functions ────────────────────────────────────────────────────────
|
||||
|
||||
function getLastWord(line: string): string {
|
||||
const words = line.match(/[a-zA-Z']+/g);
|
||||
return words ? words[words.length - 1].toLowerCase() : '';
|
||||
}
|
||||
|
||||
function detectRhymeScheme(sectionLines: string[]): { scheme: string, quality: Record<string, int> } {
|
||||
const lines = sectionLines.slice(0, 8);
|
||||
const endings = lines.map(getLastWord);
|
||||
|
||||
const mapping: Record<string, string> = {};
|
||||
let letterIdx = 0;
|
||||
const scheme: string[] = [];
|
||||
const qualityCounts: Record<string, int> = { perfect: 0, slant: 0, assonance: 0 };
|
||||
|
||||
for (const word of endings) {
|
||||
if (!word) {
|
||||
scheme.push('X');
|
||||
continue;
|
||||
}
|
||||
let matched: string | null = null;
|
||||
let bestQuality = 'none';
|
||||
|
||||
for (const [existingWord, letter] of Object.entries(mapping)) {
|
||||
const q = rhymeQuality(word, existingWord);
|
||||
if (q === 'perfect' || q === 'slant' || q === 'assonance') {
|
||||
if (!matched || q === 'perfect') {
|
||||
matched = letter;
|
||||
bestQuality = q;
|
||||
if (q === 'perfect') break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (matched) {
|
||||
scheme.push(matched);
|
||||
if (qualityCounts[bestQuality] !== undefined) {
|
||||
qualityCounts[bestQuality]++;
|
||||
}
|
||||
} else {
|
||||
const newLetter = String.fromCharCode(65 + Math.min(letterIdx, 25));
|
||||
mapping[word] = newLetter;
|
||||
scheme.push(newLetter);
|
||||
letterIdx++;
|
||||
}
|
||||
}
|
||||
|
||||
return { scheme: scheme.join(''), quality: qualityCounts };
|
||||
}
|
||||
|
||||
function analyseRhymes(allSongs: SongLyrics[]): { schemes: string[], quality: Record<string, int> } {
|
||||
const schemeFreq: Record<string, int> = {};
|
||||
const totalQuality: Record<string, int> = { perfect: 0, slant: 0, assonance: 0 };
|
||||
|
||||
for (const song of allSongs) {
|
||||
const sections = splitIntoSections(song.lyrics);
|
||||
for (const sec of sections) {
|
||||
if ((sec.label === 'V' || sec.label === 'C') && sec.lines.length >= 2) {
|
||||
const res = detectRhymeScheme(sec.lines);
|
||||
schemeFreq[res.scheme] = (schemeFreq[res.scheme] || 0) + 1;
|
||||
totalQuality.perfect += res.quality.perfect;
|
||||
totalQuality.slant += res.quality.slant;
|
||||
totalQuality.assonance += res.quality.assonance;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const topSchemes = Object.entries(schemeFreq)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 5)
|
||||
.map(x => x[0]);
|
||||
|
||||
return { schemes: topSchemes, quality: totalQuality };
|
||||
}
|
||||
|
||||
function analyseStructure(allSongs: SongLyrics[]): { v: number, c: number, blueprints: string[] } {
|
||||
let vCount = 0, vSum = 0;
|
||||
let cCount = 0, cSum = 0;
|
||||
const blueprintsFreq: Record<string, int> = {};
|
||||
|
||||
for (const song of allSongs) {
|
||||
const sections = splitIntoSections(song.lyrics);
|
||||
const labels: string[] = [];
|
||||
|
||||
for (const sec of sections) {
|
||||
if (sec.label === 'V') { vCount++; vSum += sec.lines.length; }
|
||||
else if (sec.label === 'C') { cCount++; cSum += sec.lines.length; }
|
||||
|
||||
if (!labels.length || labels[labels.length - 1] !== sec.label) {
|
||||
labels.push(sec.label);
|
||||
}
|
||||
}
|
||||
// A song with no recognised section headers parses to a single 'X' and has no
|
||||
// observable structure. Counting those produced a bare 'X' blueprint that, being
|
||||
// identical across every unheadered song, outranked every genuine structure.
|
||||
if (!labels.some(l => l !== 'X')) continue;
|
||||
|
||||
const bp = labels.join('-');
|
||||
if (bp) blueprintsFreq[bp] = (blueprintsFreq[bp] || 0) + 1;
|
||||
}
|
||||
|
||||
const topBps = Object.entries(blueprintsFreq)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(x => x[0])
|
||||
// Sanitise medley/multi-part songs: truncate at Outro
|
||||
.map(bp => {
|
||||
const parts = bp.split('-');
|
||||
const outroIdx = parts.indexOf('O');
|
||||
return outroIdx >= 0 ? parts.slice(0, outroIdx + 1).join('-') : bp;
|
||||
})
|
||||
// Drop unparseable stretches so the generator is never handed a bare 'X'
|
||||
.map(bp => bp.split('-').filter(p => p !== 'X').join('-'))
|
||||
.filter(bp => bp.length > 0)
|
||||
// Dedupe after truncation, then take the top 3
|
||||
.filter((bp, i, arr) => arr.indexOf(bp) === i)
|
||||
.slice(0, 3);
|
||||
|
||||
return {
|
||||
v: vCount ? parseFloat((vSum / vCount).toFixed(1)) : 0,
|
||||
c: cCount ? parseFloat((cSum / cCount).toFixed(1)) : 0,
|
||||
blueprints: topBps
|
||||
};
|
||||
}
|
||||
|
||||
function analysePerspective(allSongs: SongLyrics[]): string {
|
||||
let p1 = 0, p2 = 0, p3 = 0;
|
||||
const fw = new Set(['i','me','my','mine','myself',"i'm","i've","i'll","i'd","im"]);
|
||||
const sw = new Set(['you','your','yours','yourself',"you're","you've","you'll",'ya']);
|
||||
const tw = new Set(['he','she','they','him','her','them','his','their','hers','theirs']);
|
||||
|
||||
for (const song of allSongs) {
|
||||
const words = song.lyrics.toLowerCase().match(/[a-zA-Z']+/g) || [];
|
||||
for (const w of words) {
|
||||
if (fw.has(w)) p1++;
|
||||
else if (sw.has(w)) p2++;
|
||||
else if (tw.has(w)) p3++;
|
||||
}
|
||||
}
|
||||
|
||||
const total = p1 + p2 + p3;
|
||||
if (!total) return "Indeterminate (no clear pronoun pattern)";
|
||||
|
||||
const pct1 = Math.round(100 * p1 / total);
|
||||
const pct2 = Math.round(100 * p2 / total);
|
||||
const pct3 = Math.round(100 * p3 / total);
|
||||
|
||||
const parts: string[] = [];
|
||||
if (pct1 >= 50) parts.push(`First-person dominant (${pct1}% I/me/my)`);
|
||||
if (pct2 >= 30) parts.push(`Second-person address (${pct2}% you/your)`);
|
||||
if (pct3 >= 30) parts.push(`Third-person narrative (${pct3}% he/she/they)`);
|
||||
|
||||
if (!parts.length) parts.push(`Mixed voice (${pct1}% first / ${pct2}% second / ${pct3}% third)`);
|
||||
|
||||
if (pct1 >= 70) parts.push("— confessional / introspective style");
|
||||
else if (pct1 >= 50 && pct2 >= 20) parts.push("— conversational / direct address style");
|
||||
else if (pct2 >= 50) parts.push("— confrontational / accusatory style");
|
||||
else if (pct3 >= 40) parts.push("— storytelling / observational style");
|
||||
|
||||
return parts.join(' ');
|
||||
}
|
||||
|
||||
function analyseMeter(allSongs: SongLyrics[]): Record<string, any> {
|
||||
const sylCounts: int[] = [];
|
||||
const wordCounts: int[] = [];
|
||||
const charCounts: int[] = [];
|
||||
|
||||
for (const song of allSongs) {
|
||||
const lines = song.lyrics.split('\n');
|
||||
for (let line of lines) {
|
||||
line = line.trim();
|
||||
if (!line || SECTION_HEADER_RE.test(line)) continue;
|
||||
const words = line.match(/[a-zA-Z']+/g);
|
||||
if (!words) continue;
|
||||
|
||||
const syl = words.reduce((acc: number, w: string) => acc + countSyllables(w), 0);
|
||||
sylCounts.push(syl);
|
||||
wordCounts.push(words.length);
|
||||
charCounts.push(line.length);
|
||||
}
|
||||
}
|
||||
|
||||
if (!sylCounts.length) {
|
||||
return { avg_syllables_per_line: 0, syllable_std_dev: 0, avg_words_per_line: 0, line_length_range: "0-0" };
|
||||
}
|
||||
|
||||
const avgSyl = sylCounts.reduce((a, b) => a + b, 0) / sylCounts.length;
|
||||
const variance = sylCounts.reduce((a, b) => a + Math.pow(b - avgSyl, 2), 0) / sylCounts.length;
|
||||
const avgWords = wordCounts.reduce((a, b) => a + b, 0) / wordCounts.length;
|
||||
|
||||
return {
|
||||
avg_syllables_per_line: parseFloat(avgSyl.toFixed(1)),
|
||||
syllable_std_dev: parseFloat(Math.sqrt(variance).toFixed(1)),
|
||||
avg_words_per_line: parseFloat(avgWords.toFixed(1)),
|
||||
line_length_range: `${Math.min(...charCounts)}-${Math.max(...charCounts)} chars`
|
||||
};
|
||||
}
|
||||
|
||||
const COMMON_WORDS = new Set("the a an and or but if in on at to for of is it its that this with from by as are was were be been being have has had do does did will would shall should can could may might must not no nor so than too very just all each every both few more most other some such any only same also how when where why what which who whom i me my mine we us our they them their he him his she her you your about after again against between into through during before up down out off over under there here then now get got like go going know want need make take come think say tell give see feel find keep let put seem still try call ask look show turn move live help start run write set play hold bring happen begin walk talk love well back even new way day man right old big long little much good great first last time thing part work world life hand oh yeah hey ah oh uh ooh la da na hoo hey".split(' '));
|
||||
|
||||
function analyseVocabulary(allSongs: SongLyrics[]): Record<string, any> {
|
||||
const allWords: string[] = [];
|
||||
let contractions = 0;
|
||||
let profanity = 0;
|
||||
|
||||
const contRe = /\b(?:i[''']m|i[''']ve|i[''']ll|i[''']d|don[''']t|doesn[''']t|didn[''']t|won[''']t|wouldn[''']t|can[''']t|couldn[''']t|shouldn[''']t|isn[''']t|aren[''']t|wasn[''']t|weren[''']t|haven[''']t|hasn[''']t|hadn[''']t|ain[''']t|it[''']s|that[''']s|what[''']s|there[''']s|here[''']s|who[''']s|let[''']s|you[''']re|you[''']ve|you[''']ll|you[''']d|we[''']re|we[''']ve|we[''']ll|we[''']d|they[''']re|they[''']ve|they[''']ll|they[''']d|he[''']s|she[''']s|gonna|wanna|gotta|kinda|sorta|nothin[''']|somethin[''']|burnin[''']|growin[''']|draggin[''']|shaggin[''']|feelin[''']|whinin['''])\b/gi;
|
||||
const profRe = new Set("shit fuck fucking fucked damn damn ass hell bitch bastard crap piss dick".split(' '));
|
||||
|
||||
for (const song of allSongs) {
|
||||
const text = song.lyrics.toLowerCase();
|
||||
const words = text.match(/[a-zA-Z']+/g) || [];
|
||||
allWords.push(...words);
|
||||
|
||||
const contMatches = text.match(contRe);
|
||||
if (contMatches) contractions += contMatches.length;
|
||||
|
||||
for (const w of words) {
|
||||
if (profRe.has(w)) profanity++;
|
||||
}
|
||||
}
|
||||
|
||||
const total = allWords.length;
|
||||
if (!total) return { type_token_ratio: 0, total_words: 0, unique_words: 0, contraction_pct: 0, profanity_pct: 0, distinctive_words: [] };
|
||||
|
||||
const unique = new Set(allWords);
|
||||
const ttr = unique.size / total;
|
||||
|
||||
const wordFreq: Record<string, int> = {};
|
||||
for (const w of allWords) wordFreq[w] = (wordFreq[w] || 0) + 1;
|
||||
|
||||
const distinctive = Object.entries(wordFreq)
|
||||
.filter(([w]) => !COMMON_WORDS.has(w) && w.length > 2)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 15)
|
||||
.map(x => x[0]);
|
||||
|
||||
return {
|
||||
type_token_ratio: parseFloat(ttr.toFixed(3)),
|
||||
total_words: total,
|
||||
unique_words: unique.size,
|
||||
contraction_pct: parseFloat((100 * contractions / total).toFixed(1)),
|
||||
profanity_pct: parseFloat((100 * profanity / total).toFixed(1)),
|
||||
distinctive_words: distinctive
|
||||
};
|
||||
}
|
||||
|
||||
function analyseLineLengthVariation(allSongs: SongLyrics[]): Record<string, any> {
|
||||
const sectionSyl: Record<string, int[]> = { V: [], C: [], B: [] };
|
||||
const allSyl: int[] = [];
|
||||
const examples: { short: {s:int, l:string}[], long: {s:int, l:string}[] } = { short: [], long: [] };
|
||||
|
||||
for (const song of allSongs) {
|
||||
const sections = splitIntoSections(song.lyrics);
|
||||
for (const sec of sections) {
|
||||
for (const line of sec.lines) {
|
||||
const words = line.match(/[a-zA-Z']+/g);
|
||||
if (!words) continue;
|
||||
const syl = words.reduce((acc: number, w: string) => acc + countSyllables(w), 0);
|
||||
allSyl.push(syl);
|
||||
if (sectionSyl[sec.label]) sectionSyl[sec.label].push(syl);
|
||||
|
||||
if (syl <= 4) examples.short.push({ s: syl, l: line.trim() });
|
||||
else examples.long.push({ s: syl, l: line.trim() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!allSyl.length) return {};
|
||||
|
||||
const buckets = { '1-4': 0, '5-7': 0, '8-10': 0, '11-14': 0, '15+': 0 };
|
||||
for (const s of allSyl) {
|
||||
if (s <= 4) buckets['1-4']++;
|
||||
else if (s <= 7) buckets['5-7']++;
|
||||
else if (s <= 10) buckets['8-10']++;
|
||||
else if (s <= 14) buckets['11-14']++;
|
||||
else buckets['15+']++;
|
||||
}
|
||||
|
||||
const hist: Record<string, int> = {};
|
||||
for (const [k, v] of Object.entries(buckets)) hist[k] = Math.round(100 * v / allSyl.length);
|
||||
|
||||
const perSection: Record<string, any> = {};
|
||||
for (const [lbl, counts] of Object.entries(sectionSyl)) {
|
||||
if (counts.length) {
|
||||
const avg = counts.reduce((a,b)=>a+b,0)/counts.length;
|
||||
const std = Math.sqrt(counts.reduce((a,b)=>a+Math.pow(b-avg,2),0)/counts.length);
|
||||
perSection[lbl] = {
|
||||
min: Math.min(...counts), max: Math.max(...counts),
|
||||
avg: parseFloat(avg.toFixed(1)), std: parseFloat(std.toFixed(1))
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const shortEx = examples.short.sort((a,b) => a.s - b.s).slice(0,3).map(x => `(${x.s} syl) ${x.l}`);
|
||||
const longEx = examples.long.sort((a,b) => b.s - a.s).slice(0,3).map(x => `(${x.s} syl) ${x.l}`);
|
||||
|
||||
return { histogram: hist, per_section: perSection, short_line_examples: shortEx, long_line_examples: longEx };
|
||||
}
|
||||
|
||||
function analyseRepetition(allSongs: SongLyrics[]): Record<string, any> {
|
||||
let cTotal = 0, cRepeat = 0;
|
||||
let vTotal = 0, vRepeat = 0;
|
||||
const hookEx: string[] = [];
|
||||
const globalLines: string[] = [];
|
||||
|
||||
for (const song of allSongs) {
|
||||
const sections = splitIntoSections(song.lyrics);
|
||||
for (const sec of sections) {
|
||||
if (sec.label === 'C' && sec.lines.length >= 2) {
|
||||
cTotal += sec.lines.length;
|
||||
const counts: Record<string, int> = {};
|
||||
sec.lines.forEach((l: string) => { const s = l.trim().toLowerCase(); counts[s] = (counts[s]||0)+1; });
|
||||
for (const [l, c] of Object.entries(counts)) {
|
||||
if (c > 1) {
|
||||
cRepeat += c;
|
||||
if (hookEx.length < 5 && l) hookEx.push(l);
|
||||
}
|
||||
}
|
||||
} else if (sec.label === 'V' && sec.lines.length >= 2) {
|
||||
vTotal += sec.lines.length;
|
||||
const counts: Record<string, int> = {};
|
||||
sec.lines.forEach((l: string) => { const s = l.trim().toLowerCase(); counts[s] = (counts[s]||0)+1; });
|
||||
for (const c of Object.values(counts)) if (c > 1) vRepeat += c;
|
||||
}
|
||||
}
|
||||
song.lyrics.split('\n').filter((l: string) => l.trim() && !SECTION_HEADER_RE.test(l)).forEach((l: string) => globalLines.push(l.trim().toLowerCase()));
|
||||
}
|
||||
|
||||
const gCounts: Record<string, int> = {};
|
||||
globalLines.forEach(l => gCounts[l] = (gCounts[l]||0)+1);
|
||||
const crossRepeat = Object.values(gCounts).filter(c => c >= 3).length;
|
||||
|
||||
const cPct = Math.round(100 * cRepeat / Math.max(cTotal, 1));
|
||||
const vPct = Math.round(100 * vRepeat / Math.max(vTotal, 1));
|
||||
|
||||
let pattern = "low-repetition: choruses mostly unique lines";
|
||||
if (cPct >= 50) pattern = "heavy-hook: choruses built around repeated lines";
|
||||
else if (cPct >= 20) pattern = "moderate-hook: choruses use some repeated lines";
|
||||
|
||||
return { chorus_repetition_pct: cPct, verse_repetition_pct: vPct, cross_section_repeats: crossRepeat, pattern, hook_examples: hookEx.slice(0,5) };
|
||||
}
|
||||
|
||||
function selectRepresentativeExcerpts(allSongs: SongLyrics[], maxExcerpts = 5): string[] {
|
||||
const candidates: { score: number, title: string, text: string }[] = [];
|
||||
|
||||
for (const song of allSongs) {
|
||||
const sections = splitIntoSections(song.lyrics);
|
||||
for (const sec of sections) {
|
||||
if ((sec.label !== 'V' && sec.label !== 'C') || sec.lines.length < 2) continue;
|
||||
const text = sec.lines.join('\n');
|
||||
const lengthScore = 1.0 - Math.abs(sec.lines.length - 6) * 0.15;
|
||||
const res = detectRhymeScheme(sec.lines);
|
||||
const rhymeScore = res.quality.perfect * 1.0 + res.quality.slant * 0.6 + res.quality.assonance * 0.3;
|
||||
const sectionName = sec.label === 'V' ? 'Verse' : 'Chorus';
|
||||
candidates.push({ score: lengthScore + rhymeScore, title: `${song.title} (${sectionName})`, text });
|
||||
}
|
||||
}
|
||||
|
||||
candidates.sort((a,b) => b.score - a.score);
|
||||
const excerpts: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const c of candidates) {
|
||||
if (!seen.has(c.text)) {
|
||||
seen.add(c.text);
|
||||
excerpts.push(`[${c.title}]\n${c.text}`);
|
||||
if (excerpts.length >= maxExcerpts) break;
|
||||
}
|
||||
}
|
||||
return excerpts;
|
||||
}
|
||||
|
||||
// ── LLM Caller ────────────────────────────────────────────────────────────────
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
async function llmCallWithRetry(providerName: string, modelName: string, sysPrompt: string, usrPrompt: string, label: string, onPhase?: (p:string)=>void, onChunk?: ChunkCallback): Promise<{ raw: string, data: any }> {
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
if (attempt > 1 && onPhase) onPhase(`${label} (retry ${attempt}/${MAX_RETRIES})…`);
|
||||
try {
|
||||
const provider = llmService.getProvider(providerName);
|
||||
const raw = await provider.call(sysPrompt, usrPrompt, modelName, onChunk);
|
||||
const data = extractJson(raw);
|
||||
if (data) return { raw, data };
|
||||
} catch (e) {
|
||||
console.warn(`${label}: Attempt ${attempt} failed`, e);
|
||||
}
|
||||
}
|
||||
return { raw: '', data: {} };
|
||||
}
|
||||
|
||||
// buildProfilePrompt + the subject-analysis system prompt live in prompts.ts
|
||||
// (the canonical prompt source shared with the MCP server).
|
||||
|
||||
async function analyseSongSubjects(songs: SongLyrics[], providerName: string, modelName: string, onChunk?: ChunkCallback): Promise<{song_subjects: Record<string, string>, subject_categories: string[]}> {
|
||||
const usrPrompt = buildSubjectAnalysisPrompt(songs);
|
||||
|
||||
const provider = llmService.getProvider(providerName);
|
||||
try {
|
||||
const raw = await provider.call(SUBJECT_ANALYSIS_PROMPT, usrPrompt, modelName, onChunk);
|
||||
const data = extractJson(raw);
|
||||
if (data) return { song_subjects: data.song_subjects || {}, subject_categories: data.subject_categories || [] };
|
||||
} catch (e) {
|
||||
console.warn("Subject LLM call failed", e);
|
||||
}
|
||||
return { song_subjects: {}, subject_categories: [] };
|
||||
}
|
||||
|
||||
function coerceStr(val: any): string {
|
||||
if (Array.isArray(val)) return val.join('\n');
|
||||
return val ? String(val) : "";
|
||||
}
|
||||
|
||||
type int = number;
|
||||
|
||||
export async function buildProfile(
|
||||
artist: string,
|
||||
album: string | null,
|
||||
songs: SongLyrics[],
|
||||
providerName: string,
|
||||
modelName?: string,
|
||||
onPhase?: (phase: string) => void,
|
||||
onChunk?: ChunkCallback
|
||||
): Promise<Partial<LyricsProfile>> {
|
||||
const effModel = modelName || llmService.getProvider(providerName).defaultModel;
|
||||
|
||||
const struct = analyseStructure(songs);
|
||||
const rhyme = analyseRhymes(songs);
|
||||
const perspective = analysePerspective(songs);
|
||||
const meter = analyseMeter(songs);
|
||||
const vocab = analyseVocabulary(songs);
|
||||
const llv = analyseLineLengthVariation(songs);
|
||||
const rep = analyseRepetition(songs);
|
||||
const excerpts = selectRepresentativeExcerpts(songs);
|
||||
|
||||
meter.line_length_variation = llv;
|
||||
|
||||
const ruleStats = {
|
||||
avg_verse_lines: struct.v,
|
||||
avg_chorus_lines: struct.c,
|
||||
rhyme_schemes: rhyme.schemes,
|
||||
rhyme_quality: rhyme.quality,
|
||||
structure_blueprints: struct.blueprints,
|
||||
perspective,
|
||||
meter_stats: meter,
|
||||
vocabulary_stats: vocab,
|
||||
repetition_stats: rep
|
||||
};
|
||||
|
||||
const usrPrompt = buildProfilePrompt(artist, album, songs, ruleStats);
|
||||
const merged: Record<string, any> = {};
|
||||
const raws: string[] = [];
|
||||
|
||||
if (onPhase) onPhase("Analysing themes & vocabulary… (1/4)");
|
||||
const res1 = await llmCallWithRetry(providerName, effModel, PROFILE_PROMPT_1, usrPrompt, "Call 1/4", onPhase, onChunk);
|
||||
raws.push(res1.raw); Object.assign(merged, res1.data);
|
||||
|
||||
if (onPhase) onPhase("Analysing tone & structure… (2/4)");
|
||||
const res2 = await llmCallWithRetry(providerName, effModel, PROFILE_PROMPT_2, usrPrompt, "Call 2/4", onPhase, onChunk);
|
||||
raws.push(res2.raw); Object.assign(merged, res2.data);
|
||||
|
||||
if (onPhase) onPhase("Analysing imagery & signature… (3/4)");
|
||||
const res3 = await llmCallWithRetry(providerName, effModel, PROFILE_PROMPT_3, usrPrompt, "Call 3/4", onPhase, onChunk);
|
||||
raws.push(res3.raw); Object.assign(merged, res3.data);
|
||||
|
||||
if (onPhase) onPhase("Analysing song subjects… (4/5)");
|
||||
const subjects = await analyseSongSubjects(songs, providerName, effModel, onChunk);
|
||||
|
||||
// Step 5: Generate style caption (non-critical)
|
||||
let style_caption = '';
|
||||
try {
|
||||
if (onPhase) onPhase("Generating style caption… (5/5)");
|
||||
const captionUserPrompt = [
|
||||
`Artist: ${artist}`,
|
||||
album ? `Album: ${album}` : '',
|
||||
merged.themes?.length ? `Themes: ${merged.themes.join(', ')}` : '',
|
||||
merged.tone_and_mood ? `Tone and mood: ${merged.tone_and_mood}` : '',
|
||||
merged.vocabulary_notes ? `Vocabulary: ${merged.vocabulary_notes}` : '',
|
||||
].filter(Boolean).join('\n');
|
||||
const provider = llmService.getProvider(providerName);
|
||||
const captionRaw = await provider.call(STYLE_CAPTION_PROMPT, captionUserPrompt, effModel, onChunk);
|
||||
// Strip any wrapping quotes or whitespace
|
||||
style_caption = captionRaw.replace(/^["'`]+|["'`]+$/g, '').trim();
|
||||
} catch (e) {
|
||||
console.warn('Style caption generation failed (non-critical):', e);
|
||||
}
|
||||
|
||||
const rawCombined = raws.join('\n\n---\n\n');
|
||||
|
||||
return {
|
||||
artist,
|
||||
album: album || undefined,
|
||||
// Deterministic, never LLM-derived — measured facts from the source audio.
|
||||
audio_enrichment: computeAlbumEnrichment(songs),
|
||||
themes: merged.themes || [],
|
||||
common_subjects: merged.common_subjects || [],
|
||||
rhyme_schemes: merged.rhyme_schemes || rhyme.schemes,
|
||||
avg_verse_lines: merged.avg_verse_lines || struct.v,
|
||||
avg_chorus_lines: merged.avg_chorus_lines || struct.c,
|
||||
vocabulary_notes: coerceStr(merged.vocabulary_notes),
|
||||
tone_and_mood: coerceStr(merged.tone_and_mood),
|
||||
structural_patterns: coerceStr(merged.structural_patterns),
|
||||
additional_notes: coerceStr(merged.additional_notes),
|
||||
raw_summary: coerceStr(merged.raw_summary || rawCombined),
|
||||
// Extra parsed data
|
||||
structure_blueprints: struct.blueprints,
|
||||
perspective,
|
||||
meter_stats: meter,
|
||||
vocabulary_stats: vocab,
|
||||
representative_excerpts: excerpts,
|
||||
narrative_techniques: coerceStr(merged.narrative_techniques),
|
||||
imagery_patterns: coerceStr(merged.imagery_patterns),
|
||||
signature_devices: coerceStr(merged.signature_devices),
|
||||
emotional_arc: coerceStr(merged.emotional_arc),
|
||||
rhyme_quality: rhyme.quality as any, // bypassing strict TS checking for complex types
|
||||
song_subjects: subjects.song_subjects as any,
|
||||
subject_categories: subjects.subject_categories,
|
||||
repetition_stats: rep as any,
|
||||
style_caption,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-runs all local (non-LLM) statistical analysis on an existing profile's
|
||||
* lyrics set and patches the profile_data in place. This is fast — no LLM calls.
|
||||
*/
|
||||
export function recalculateProfileStats(songs: SongLyrics[], profileData: any): any {
|
||||
const vocab = analyseVocabulary(songs);
|
||||
const meter = analyseMeter(songs);
|
||||
const rhyme = analyseRhymes(songs);
|
||||
const struct = analyseStructure(songs);
|
||||
const rep = analyseRepetition(songs);
|
||||
const lineVar = analyseLineLengthVariation(songs);
|
||||
const perspective = analysePerspective(songs);
|
||||
const excerpts = selectRepresentativeExcerpts(songs);
|
||||
|
||||
return {
|
||||
...profileData,
|
||||
// Overwrite computed stats
|
||||
audio_enrichment: computeAlbumEnrichment(songs),
|
||||
vocabulary_stats: vocab,
|
||||
meter_stats: meter,
|
||||
rhyme_schemes: rhyme.schemes,
|
||||
rhyme_quality: rhyme.quality as any,
|
||||
avg_verse_lines: struct.v,
|
||||
avg_chorus_lines: struct.c,
|
||||
repetition_stats: rep as any,
|
||||
line_length_variation: lineVar,
|
||||
perspective,
|
||||
// These two are derived here as well as in buildProfile — without them, profiles
|
||||
// created outside the app (e.g. via the MCP) have no structural vocabulary for
|
||||
// the generator to plan a structure from, and no style excerpts to imitate.
|
||||
structure_blueprints: struct.blueprints,
|
||||
representative_excerpts: excerpts,
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,370 @@
|
||||
// slopDetector.ts — AI-Slop Detection and Prevention System
|
||||
//
|
||||
// 7-layer defense to ensure generated lyrics feel authentic.
|
||||
// Direct port from Python slop_detector.py — all patterns preserved.
|
||||
// Layer 7 added based on corpus analysis of 608 generations (2026-05-02).
|
||||
|
||||
// ── Layer 1 — Blacklisted Words ─────────────────────────────────────────────
|
||||
|
||||
export const BLACKLISTED_WORDS = new Set([
|
||||
// Visual clichés
|
||||
'neon', 'streetlights', 'streetlight', 'silhouette', 'silhouettes',
|
||||
'tapestry', 'mosaic', 'kaleidoscope', 'prism',
|
||||
// Action clichés
|
||||
'yearning', 'beckons', 'beckoning', 'cascading', 'cascade',
|
||||
'unfurling', 'unfurl',
|
||||
// Emotional clichés
|
||||
'bittersweet', 'melancholy', 'wistful', 'poignant', 'ethereal', 'ephemeral',
|
||||
// Abstract concepts
|
||||
'symphony', 'harmonize', 'harmonizing', 'crossroads',
|
||||
// Overused metaphors
|
||||
'phoenix', 'labyrinth', 'soaring',
|
||||
// Generic intensity
|
||||
'pulsing', 'pulsating', 'throbbing', 'vibrant', 'vivid', 'luminous',
|
||||
'radiant', 'shimmering',
|
||||
// Time clichés
|
||||
'hourglass', 'timeless',
|
||||
// Nature clichés
|
||||
'tempest',
|
||||
// Existential clichés
|
||||
'essence', 'consciousness', 'realm', 'dimension',
|
||||
// Modern AI clichés (2024-2026 patterns)
|
||||
'unraveling', 'unravel', 'ember', 'embers', 'ignite', 'ignites',
|
||||
'resonate', 'resonates', 'reverberate', 'reverberates',
|
||||
// Faux-poetic
|
||||
'amidst', 'entwined', 'intertwined', 'ablaze',
|
||||
// Cosmic clichés
|
||||
'constellation', 'constellations', 'cosmos', 'infinite', 'infinity', 'void',
|
||||
// Over-emotional constructions
|
||||
'shattering', 'hollowed',
|
||||
// Synesthesia clichés
|
||||
'crimson sky', 'velvet night',
|
||||
// Generic AI title/filler words
|
||||
'static', 'catalyst', 'paradox', 'paradigm', 'mantra', 'epitome',
|
||||
'chronicles', 'solace', 'juxtaposition', 'serenity', 'resilience',
|
||||
'dichotomy', 'transcend', 'transcendence', 'metamorphosis', 'pinnacle',
|
||||
// Lighting clichés
|
||||
'fluorescent', 'halogen',
|
||||
// Corpus-analysis additions (2026-05-02) — overused across 608 generations
|
||||
'wreckage', 'jagged', 'bitter', 'hollow',
|
||||
'steel', 'metal', 'transmission', 'dashboard', 'gears', 'gloom',
|
||||
// Tech-slop additions (2026-06-05) — user-flagged + corpus analysis
|
||||
// These get slathered across all genres regardless of artist fit
|
||||
'digital', 'algorithm', 'algorithms', 'chrome',
|
||||
'code', 'circuit', 'circuits', 'grid', 'data',
|
||||
'wire', 'wires', 'wired',
|
||||
]);
|
||||
|
||||
// ── Layer 1b — Overused Words (soft-ban: penalized per-occurrence, not banned) ──
|
||||
// These words aren't inherently bad but the model leans on them like a crutch.
|
||||
// "heavy" alone appeared 811 times across 64.3% of 608 songs.
|
||||
|
||||
export const OVERUSED_WORDS = new Set([
|
||||
'heavy', 'broken', 'cold', 'dust', 'ghost', 'machine',
|
||||
'nothing', 'nowhere', 'searching', 'wreckage', 'losing',
|
||||
// Corpus-analysis additions (2026-06-05) — overused in hooks/titles across 992 generations
|
||||
'watch', 'burn', 'fade', 'fading', 'wash', 'sold',
|
||||
'dead', 'blood', 'gold', 'same',
|
||||
]);
|
||||
|
||||
// ── Layer 2 — Blacklisted Phrases ───────────────────────────────────────────
|
||||
|
||||
export const BLACKLISTED_PHRASES = new Set([
|
||||
'reaching up to the sky', 'beneath the streetlights', 'under the streetlights',
|
||||
'neon lights', 'neon glow', 'neon dreams', 'echoes in the night',
|
||||
'whispers in the dark', 'shadows dance', 'dancing shadows',
|
||||
'tapestry of dreams', 'symphony of', 'kaleidoscope of',
|
||||
'mosaic of emotions', 'bittersweet memories', 'fleeting moments',
|
||||
'sands of time', 'rising from the ashes', 'like a phoenix',
|
||||
'tangled web', 'labyrinth of', 'journey begins', 'path ahead',
|
||||
'crossroads of', 'fabric of reality', 'threads of fate',
|
||||
'ocean of tears', 'sea of faces', 'waves of emotion',
|
||||
'storm within', 'tempest raging', 'essence of', 'realm of',
|
||||
'universe within', 'consciousness expands', 'vivid dreams',
|
||||
'radiant light', 'pulsing with', 'cascading down',
|
||||
'ethereal beauty', 'melancholy mood', 'beckons me', 'yearning for',
|
||||
// Expanded set
|
||||
'paint the sky', 'written in the stars', 'dance with the devil',
|
||||
'scream into the void', 'drown in your eyes', 'heart on my sleeve',
|
||||
'break the chains', 'find my voice', 'lost in the moment',
|
||||
'through the fire', 'edge of forever', 'weight of the world',
|
||||
'paint a picture', 'piece by piece', 'shattered glass',
|
||||
'hollow eyes', 'burning bridges', 'chase the sun',
|
||||
'bleeding heart', 'silent scream', 'torn apart',
|
||||
'crumbling walls', 'whisper your name', 'dust settles',
|
||||
'ghost of you', 'ashes to ashes', 'taste of freedom',
|
||||
'colors of the wind', 'sound of silence',
|
||||
'in this moment', 'against the tide', 'into the unknown',
|
||||
'carry the weight', 'unravel the truth', 'embers glow',
|
||||
'spark ignites', 'constellations align', 'resonates within',
|
||||
'let it all go', 'rise above it all',
|
||||
// Corpus-analysis additions (2026-05-02)
|
||||
'nothing left', 'nowhere left', 'nothing left to',
|
||||
'the weight of', 'the wreckage', 'same old',
|
||||
'every single', 'cold and heavy', 'cold and dark',
|
||||
'cold and empty', 'heavy and cold', 'heavy and dark',
|
||||
'pulling me down', 'dragging me down',
|
||||
]);
|
||||
|
||||
// ── Layer 8 — Overused Hook Formulas ────────────────────────────────────────
|
||||
// These structural patterns produce identical-sounding hooks across genres.
|
||||
// Each matches a common LLM comfort pattern for chorus writing.
|
||||
// Penalty: +5 per match in a chorus section.
|
||||
|
||||
export const OVERUSED_HOOK_PATTERNS: RegExp[] = [
|
||||
// "[Verb] it [all/down/away/out/off/back/up]" — 46 titles used this
|
||||
/\b(watch|burn|tear|wash|break|crush|push|pull|let|cut|rip|smash|turn)\s+it\s+(all|down|away|out|off|back|up)\b/gi,
|
||||
// "Watch [me/it/them/us] [verb]" — 25 titles
|
||||
/\bwatch\s+(me|it|them|us|him|her)\s+\w+/gi,
|
||||
// "Don't let them [verb]" — 7 titles
|
||||
/\bdon'?t\s+let\s+(them|him|her|it)\s+\w+/gi,
|
||||
// "Nothing left to [verb]" / "Nowhere left to [verb]"
|
||||
/\b(nothing|nowhere)\s+left\s+to\s+\w+/gi,
|
||||
// "Let it [verb/adjective]" — generic release formula
|
||||
/\blet\s+it\s+(burn|fade|go|fall|bleed|break|rot|die|end|crash|crumble|drown)\b/gi,
|
||||
// "[Verb]ing it all [away/down]"
|
||||
/\b\w+ing\s+it\s+all\s+(away|down)\b/gi,
|
||||
];
|
||||
|
||||
// ── Layer 3 — Regex Patterns ────────────────────────────────────────────────
|
||||
|
||||
const AI_PATTERNS = [
|
||||
/\b(tapestry|fabric|symphony|kaleidoscope|mosaic|labyrinth|maze|void)\s+of\s+\w+\b/gi,
|
||||
/\blike\s+a\s+(phoenix|symphony|kaleidoscope|constellation|ember)\b/gi,
|
||||
/\b\w+ing\s+\w+ing\b/gi,
|
||||
/\bthe\s+\w+\s+of\s+my\s+\w+\b/gi,
|
||||
/^In\s+the\s+(darkness|silence|shadows|distance|stillness|emptiness)\b/gim,
|
||||
// Corpus-analysis additions — "cold and [X]" / "heavy [noun]" overuse patterns
|
||||
/\bcold\s+and\s+\w+\b/gi,
|
||||
/\bheavy\s+(weight|hand|heart|air|load|chain|sky|dust|night|crown|veil|rain|door|stone|iron|fog|clouds?)\b/gi,
|
||||
];
|
||||
|
||||
// ── Layer 4 — Structural Analysis ───────────────────────────────────────────
|
||||
|
||||
const FUNCTION_WORDS = new Set([
|
||||
'the', 'a', 'an', 'of', 'in', 'to', 'and', 'is', 'it', 'for', 'on',
|
||||
'with', 'as', 'at', 'by', 'from', 'or', 'but', 'not', 'be', 'are',
|
||||
'was', 'were', 'been', 'this', 'that', 'which', 'who', 'what', 'if',
|
||||
'so', 'my', 'your', 'we', 'they', 'i', 'you', 'he', 'she', 'me', 'us',
|
||||
]);
|
||||
|
||||
function extractWords(text: string): string[] {
|
||||
return (text.toLowerCase().match(/\b[a-zA-Z]+(?:'[a-zA-Z]+)?\b/g) ?? []);
|
||||
}
|
||||
|
||||
function analyzeStructure(lines: string[]): { issues: string[]; score: number } {
|
||||
const issues: string[] = [];
|
||||
let score = 0;
|
||||
if (lines.length < 4) return { issues, score };
|
||||
|
||||
const wordCounts = lines.filter(l => l.trim()).map(l => l.split(/\s+/).length);
|
||||
if (!wordCounts.length) return { issues, score };
|
||||
|
||||
const mean = wordCounts.reduce((a, b) => a + b, 0) / wordCounts.length;
|
||||
const variance = wordCounts.reduce((a, wc) => a + (wc - mean) ** 2, 0) / wordCounts.length;
|
||||
const stddev = Math.sqrt(variance);
|
||||
|
||||
if (wordCounts.length >= 6 && stddev < 1.0) {
|
||||
issues.push(`Line lengths are suspiciously uniform (stddev=${stddev.toFixed(2)})`);
|
||||
score += 10;
|
||||
}
|
||||
|
||||
// First word repetition
|
||||
const firstWords = lines.filter(l => l.trim()).map(l => l.trim().split(/\s+/)[0]?.toLowerCase()).filter(Boolean);
|
||||
if (firstWords.length) {
|
||||
const freq = new Map<string, number>();
|
||||
for (const w of firstWords) freq.set(w, (freq.get(w) ?? 0) + 1);
|
||||
let maxWord = '', maxCount = 0;
|
||||
for (const [w, c] of freq) { if (c > maxCount) { maxWord = w; maxCount = c; } }
|
||||
const ratio = maxCount / firstWords.length;
|
||||
if (ratio > 0.5 && maxCount >= 3) {
|
||||
issues.push(`Over-repetitive line starter '${maxWord}' (${maxCount}/${firstWords.length} = ${(ratio * 100).toFixed(0)}%)`);
|
||||
score += 8;
|
||||
}
|
||||
}
|
||||
|
||||
return { issues, score };
|
||||
}
|
||||
|
||||
function detectAnomalies(text: string): { issues: string[]; score: number } {
|
||||
const issues: string[] = [];
|
||||
let score = 0;
|
||||
const words = extractWords(text);
|
||||
if (words.length < 20) return { issues, score };
|
||||
|
||||
const freq = new Map<string, number>();
|
||||
for (const w of words) freq.set(w, (freq.get(w) ?? 0) + 1);
|
||||
const unique = freq.size;
|
||||
|
||||
// Hapax ratio
|
||||
let hapax = 0;
|
||||
for (const c of freq.values()) if (c === 1) hapax++;
|
||||
const hapaxRatio = unique ? hapax / unique : 0;
|
||||
if (hapaxRatio > 0.85 && unique > 20) {
|
||||
issues.push(`Very high hapax ratio (${hapaxRatio.toFixed(2)}) — too many unique-once words`);
|
||||
score += 6;
|
||||
}
|
||||
|
||||
// Function word density
|
||||
let funcCount = 0;
|
||||
for (const fw of FUNCTION_WORDS) funcCount += freq.get(fw) ?? 0;
|
||||
const funcRatio = words.length ? funcCount / words.length : 0;
|
||||
if (funcRatio > 0.38) {
|
||||
issues.push(`High function word density (${funcRatio.toFixed(2)}) — reads more like prose than lyrics`);
|
||||
score += 5;
|
||||
}
|
||||
|
||||
// Line opener variety
|
||||
const lyricLines = text.split('\n')
|
||||
.map(l => l.trim())
|
||||
.filter(l => l && !l.startsWith('[') && !l.startsWith('('));
|
||||
if (lyricLines.length >= 6) {
|
||||
const openers = lyricLines.map(l => l.split(/\s+/)[0]?.toLowerCase()).filter(Boolean);
|
||||
const openerUnique = new Set(openers).size / openers.length;
|
||||
if (openerUnique < 0.3) {
|
||||
issues.push(`Very low line opener variety (${openerUnique.toFixed(2)})`);
|
||||
score += 5;
|
||||
} else if (openerUnique > 0.95 && openers.length > 8) {
|
||||
issues.push('Suspiciously perfect line opener variety — real lyrics naturally repeat some starters');
|
||||
score += 3;
|
||||
}
|
||||
}
|
||||
|
||||
return { issues, score };
|
||||
}
|
||||
|
||||
// ── Main Scan Function ──────────────────────────────────────────────────────
|
||||
|
||||
export interface SlopScanResult {
|
||||
ai_score: number;
|
||||
severity: 'high' | 'medium' | 'low';
|
||||
is_likely_ai: boolean;
|
||||
layers: {
|
||||
blacklisted_words: { score: number; found: string[] };
|
||||
blacklisted_phrases: { score: number; found: string[] };
|
||||
pattern_matches: { score: number; found: [string, string][] };
|
||||
structural: { score: number; raw_score: number; issues: string[] };
|
||||
fingerprint: { score: number; raw_score: number; issues: string[] };
|
||||
statistical: { score: number; raw_score: number; issues: string[] };
|
||||
overuse: { score: number; found: { word: string; count: number }[] };
|
||||
hook_formulas: { score: number; found: string[] };
|
||||
};
|
||||
}
|
||||
|
||||
export function scanForSlop(
|
||||
text: string,
|
||||
fingerprint?: Record<string, any> | null,
|
||||
statisticalWeight = 1.0,
|
||||
): SlopScanResult {
|
||||
// Strip section tags and performance notes
|
||||
const clean = text.replace(/\[.*?\]/g, '').replace(/\(.*?\)/g, '');
|
||||
const words = extractWords(clean);
|
||||
const lines = clean.split('\n').map(l => l.trim()).filter(l => l.length > 1);
|
||||
|
||||
// Layer 1: Blacklisted words
|
||||
const badWords = words.filter(w => BLACKLISTED_WORDS.has(w));
|
||||
const l1Score = badWords.length * 10;
|
||||
|
||||
// Layer 2: Blacklisted phrases
|
||||
const textLower = clean.toLowerCase();
|
||||
const badPhrases = [...BLACKLISTED_PHRASES].filter(p => textLower.includes(p));
|
||||
const l2Score = badPhrases.length * 20;
|
||||
|
||||
// Layer 3: Regex patterns
|
||||
const badPatterns: [string, string][] = [];
|
||||
for (const pat of AI_PATTERNS) {
|
||||
pat.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = pat.exec(clean)) !== null) {
|
||||
badPatterns.push([pat.source, m[0]]);
|
||||
}
|
||||
}
|
||||
const l3Score = badPatterns.length * 5;
|
||||
|
||||
// Layer 4: Structural
|
||||
const l4 = analyzeStructure(lines);
|
||||
|
||||
// Layer 5: Fingerprint (basic comparison if provided)
|
||||
const l5 = { issues: [] as string[], score: 0 };
|
||||
if (fingerprint) {
|
||||
const safeVocab = new Set<string>(fingerprint.safe_vocabulary ?? []);
|
||||
if (safeVocab.size) {
|
||||
const wordSet = new Set(words);
|
||||
let overlap = 0;
|
||||
for (const w of wordSet) if (safeVocab.has(w)) overlap++;
|
||||
const overlapRatio = wordSet.size ? overlap / wordSet.size : 0;
|
||||
if (overlapRatio < 0.5) {
|
||||
l5.issues.push(`Low vocabulary overlap with source artist (${(overlapRatio * 100).toFixed(0)}% vs expected ≥50%)`);
|
||||
l5.score += 12;
|
||||
}
|
||||
}
|
||||
const artistTtr = fingerprint.type_token_ratio;
|
||||
if (artistTtr != null) {
|
||||
const genTtr = words.length ? new Set(words).size / words.length : 0;
|
||||
const diff = Math.abs(genTtr - artistTtr);
|
||||
if (diff > 0.15) {
|
||||
l5.issues.push(`TTR mismatch: generated=${genTtr.toFixed(3)}, artist=${artistTtr.toFixed(3)}`);
|
||||
l5.score += 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Layer 6: Statistical anomalies
|
||||
const l6 = detectAnomalies(clean);
|
||||
|
||||
// Layer 7: Overused word detection (soft-ban)
|
||||
// Penalize +3 per occurrence after the first — these aren't banned but the model
|
||||
// uses them as a crutch across genres ("heavy" alone: 811x in 608 songs).
|
||||
const overuseFound: { word: string; count: number }[] = [];
|
||||
let l7Score = 0;
|
||||
const wordFreq = new Map<string, number>();
|
||||
for (const w of words) {
|
||||
if (OVERUSED_WORDS.has(w)) wordFreq.set(w, (wordFreq.get(w) ?? 0) + 1);
|
||||
}
|
||||
for (const [word, count] of wordFreq) {
|
||||
if (count > 1) {
|
||||
const penalty = (count - 1) * 3; // +3 per occurrence after the first
|
||||
l7Score += penalty;
|
||||
overuseFound.push({ word, count });
|
||||
}
|
||||
}
|
||||
|
||||
// Layer 8: Overused hook formula detection
|
||||
// Only scans chorus sections to target hook patterns specifically
|
||||
const chorusSections = text.split(/\[(?:Chorus|Hook).*?\]/i).slice(1)
|
||||
.map(s => s.split(/\[/)[0]?.trim()).filter(Boolean);
|
||||
const chorusText = chorusSections.join('\n');
|
||||
const hookFormulaFound: string[] = [];
|
||||
let l8Score = 0;
|
||||
if (chorusText) {
|
||||
for (const pat of OVERUSED_HOOK_PATTERNS) {
|
||||
pat.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = pat.exec(chorusText)) !== null) {
|
||||
hookFormulaFound.push(m[0]);
|
||||
l8Score += 5;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const sw = Math.max(0, Math.min(1, statisticalWeight));
|
||||
const total = l1Score + l2Score + l3Score +
|
||||
Math.floor(l4.score * sw) + Math.floor(l5.score * sw) + Math.floor(l6.score * sw) +
|
||||
l7Score + l8Score;
|
||||
|
||||
return {
|
||||
ai_score: total,
|
||||
severity: total > 30 ? 'high' : total > 15 ? 'medium' : 'low',
|
||||
is_likely_ai: total > 15,
|
||||
layers: {
|
||||
blacklisted_words: { score: l1Score, found: [...new Set(badWords)] },
|
||||
blacklisted_phrases: { score: l2Score, found: badPhrases },
|
||||
pattern_matches: { score: l3Score, found: badPatterns.slice(0, 10) },
|
||||
structural: { score: Math.floor(l4.score * sw), raw_score: l4.score, issues: l4.issues },
|
||||
fingerprint: { score: Math.floor(l5.score * sw), raw_score: l5.score, issues: l5.issues },
|
||||
statistical: { score: Math.floor(l6.score * sw), raw_score: l6.score, issues: l6.issues },
|
||||
overuse: { score: l7Score, found: overuseFound },
|
||||
hook_formulas: { score: l8Score, found: hookFormulaFound },
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user