1 line
224 KiB
JSON
1 line
224 KiB
JSON
{"repo": "blusterous-phonograph/ai-music-generation-system", "files": {"structure_gen/musicgen_api.py": {"486bab9207d6499336bfe192c2ed40eb91b6c4b0": {"t": 1785768919.8991516, "n": 45729, "h": "7b4aeae1961a1f29", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nmusicgen_api.py\n\n封装 MusicGen 无限续写脚本为后端可调用接口。\n支持 Melody 条件生成、增量续写、重试与显存管理。\n\"\"\"\n\nnetworks = False # 如果没有接入国际互联网，就使用 False\n\n\nimport os\n# 如果没有接入国际互联网，就将 Huggingface 的访问设置为【不访问】\nif not networks:\n # 确保 transformers/hub 都用离线模式\n os.environ[\"HF_HUB_OFFLINE\"] = \"1\"\n os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n # 然后 monkey-patch BertTokenizer.from_pretrained，让它只走本地缓存\n from transformers import BertTokenizer\n _original_bp = BertTokenizer.from_pretrained\n def _from_pretrained_local(repo_id, *args, **kwargs):\n # 强制 local_files_only=True\n return _original_bp(repo_id, local_files_only=True, *args, **kwargs)\n BertTokenizer.from_pretrained = _from_pretrained_local\n\n\nimport json\nimport math\nimport warnings\nimport re\nimport time\nimport ollama\nimport torchaudio\nimport torch\nimport requests\nimport pathlib\nimport soundfile as sf\n\nfrom contextlib import nullcontext\nfrom typing import List, Optional, Union, Dict\nfrom urllib.parse import urljoin\nfrom audiocraft.models import MusicGen\nfrom laion_clap import CLAP_Module\n# from structure_gen.music_structure import STYLE_STRUCTURE_LIB # lin\nfrom music_structure import STYLE_STRUCTURE_LIB # yuan\n\n# ========== 默认配置==========\n\nDEFAULT_USE_LOCAL_LLM = False # 默认是否走本地 LLM（Ollama）\nOPENAI_MODEL = 'gpt-4o-mini' # 远程 LLM 模型名\nmax_retries = 6 # 远程 LLM 最大重试次数\n\nOPENAI_API_KEY = 'sk-vWyNj5NxixWjJcjknaxNivrtIkHIjXDBVlk1b6wRW2pZc0hz' # 网络LLM API KEY\n\n\nUSE_BF16 = torch.cuda.is_bf16_supported() # \nDTYPE = torch.bfloat16 if USE_BF16 else torch.float16 \n\nENABLE_COMPILE = True # False 可快速回退\nSTRICT_LOAD = False\n\n\n# =========================================\n\n\n\n\ndef cast_musicgen_dtype(mg, dtype):\n \"\"\"把 MusicGen 内所有子模块转成半精度 / 全精度\"\"\"\n mg.lm = mg.lm.to(dtype=dtype)\n if hasattr(mg, \"decoder\"):\n mg.decoder = mg.decoder.to(dtype=dtype)\n if hasattr(mg, \"condition_provider\"):\n mg.condition_provider = mg.condition_provider.to(dtype=dtype)\n return mg\n\n\nclass MusicGenService:\n def __init__(\n self,\n *,\n use_local_llm: bool = DEFAULT_USE_LOCAL_LLM,\n api_endpoint: Optional[str] = None,\n openai_api_key: str = OPENAI_API_KEY,\n openai_model: str = OPENAI_MODEL,\n verbose: bool = True,\n device: Optional[str] = None,\n checkpoint_path: Optional[str] = None,\n use_fp16: bool = True,\n ):\n \n # —— 与 LLM 相关的配置 ——\n self.use_local_llm = use_local_llm\n if api_endpoint is None:\n # 如果调用方没指定，按 use_local_llm 选默认地址\n self.api_endpoint = (\n \"http://localhost:11434/api/generate\"\n if self.use_local_llm\n else \"https://www.dmxapi.com/v1/chat/completions\"\n )\n else:\n self.api_endpoint = api_endpoint\n # 参数配置\n self.openai_api_key = openai_api_key\n self.openai_model = openai_model\n self.max_retries = max_retries\n self.verbose = verbose\n self.device = device or (\"cuda\" if torch.cuda.is_available() else \"cpu\")\n self.use_fp16 = use_fp16\n self.clap = CLAP_Module(enable_fusion=False, device=self.device).eval()\n self.SR = 32000\n self.mel_transform = torchaudio.transforms.MelSpectrogram(\n sample_rate=self.SR, n_mels=80, hop_length=512\n ).to(self.device)\n self.checkpoint_path = checkpoint_path\n\n\n # 1) 加载 small 版本\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}...\")\n # # 检查audiocraft版本\n # try:\n # import audiocraft\n # print(f\"[Init] audiocraft版本: {audiocraft.__version__}\")\n # except:\n # print(f\"[Init] 无法获取audiocraft版本\")\n \n self.model = MusicGen.get_pretrained(\"facebook/musicgen-small\", self.device)\n\n # # 添加模型信息调试\n # if self.verbose:\n # print(f\"[Init] MusicGen模型信息:\")\n # print(f\" - 模型类型: {type(self.model)}\")\n # print(f\" - sample_rate: {getattr(self.model, 'sample_rate', 'None')}\")\n # print(f\" - 压缩模型sample_rate: {getattr(self.model.compression_model, 'sample_rate', 'None')}\")\n # print(f\" - 默认duration: {getattr(self.model, 'duration', 'None')}\")\n # if hasattr(self.model, 'lm'):\n # print(f\" - LM模型: {type(self.model.lm)}\")\n # if hasattr(self.model, 'compression_model'):\n # print(f\" - 压缩模型: {type(self.model.compression_model)}\")\n\n # 设置参数\n self.model.set_generation_params(\n duration=30,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n \n # # 测试基本generation功能\n # if self.verbose:\n # print(f\"[Init] 测试基本generation功能...\")\n # try:\n # # 测试generate方法\n # test_wav = self.model.generate(['test'], progress=False)\n # if len(test_wav) > 0:\n # print(f\" - generate测试成功，生成长度: {test_wav[0].shape[-1]/32000:.1f}s\")\n # else:\n # print(f\" - ⚠️ generate测试失败，返回空结果\")\n # \n # # 测试generate_continuation方法（使用简单prompt）\n # simple_prompt = torch.zeros(1, 1, 16000).to(self.device) # 0.5秒的空白音频\n # test_cont = self.model.generate_continuation(\n # prompt=simple_prompt,\n # prompt_sample_rate=32000,\n # descriptions=['test'],\n # progress=False\n # )\n # if len(test_cont) > 0:\n # print(f\" - generate_continuation测试成功，生成长度: {test_cont[0].shape[-1]/32000:.1f}s\")\n # print(f\" - prompt长度: {simple_prompt.shape[-1]/32000:.1f}s\")\n # print(f\" - 续写比例: {test_cont[0].shape[-1]/simple_prompt.shape[-1]:.1f}x\")\n # else:\n # print(f\" - ⚠️ generate_continuation测试失败，返回空结果\")\n # except Exception as e:\n # print(f\" - ⚠️ 测试失败: {e}\")\n # import traceback\n # traceback.print_exc()\n\n # 2) 如果指定了 checkpoint，就加载权重\n if checkpoint_path:\n if self.verbose:\n print(f\"[Init] Loading checkpoint from {checkpoint_path}…\")\n ckpt = torch.load(checkpoint_path, map_location=\"cpu\")\n state_dict = ckpt.get(\"model_state_dict\", ckpt)\n missing, unexpected = self.model.lm.load_state_dict(state_dict, strict=True)\n if self.verbose:\n print(f\"[Init] Checkpoint loaded. missing keys: {len(missing)}, unexpected: {len(unexpected)}\")\n\n # 3) 半精度转换 & 编译\n self.model = cast_musicgen_dtype(self.model, DTYPE)\n print(\"Model dtype:\", next(self.model.lm.parameters()).dtype)\n\n # 编译 lm 子模块以减少 Python 调度开销\n try:\n self.model.lm = torch.compile(self.model.lm,mode=\"reduce-overhead\",fullgraph=True)\n # if self.verbose:\n # print(\"[Init] 🚀 torch.compile() 已启用\")\n except Exception as e:\n if self.verbose:\n print(f\"[Init] ⚠️ torch.compile 失败，回退: {e}\")\n\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}... OK\")\n\n\n @torch.no_grad()\n def embed_cpu(self, wav32: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 将输入的多通道高采样率音频数据转换为单通道 16kHz，\n 并用 CLAP 模型提取其在 CPU 上的特征嵌入表示，\n 用于后续相似度计算或音频理解任务。\n \"\"\"\n mono = wav32.to(\"cpu\", torch.float32).mean(0, keepdim=True)\n m16 = torchaudio.functional.resample(mono, self.SR, 16000)\n return self.clap.get_audio_embedding_from_data(m16, use_tensor=True).cpu()\n\n \n def request_segment_plan(\n self, style: str, user_prompt: str, total_sec: int, window_sec: int\n ) -> List[Dict]:\n \"\"\"\n 根据用户提供的风格、提示词和时长设置，调用本地或远程 ChatGPT 接口生成音乐分段规划。\n\n 该方法会构造一个详细的自然语言提示（prompt），并附带 CLAP 支持的 token 列表，提交给语言模型，\n 请求生成一个包含多个段落的 JSON 列表。每个段落包含音乐结构（如 intro、verse 等）、持续时间、\n 生成提示词以及采样参数（temperature、top_p、top_k）等信息。返回结果会自动补齐字段并计算结束时间戳。\n\n 如果调用失败，方法将退回使用内置的默认分段方案。\n\n 参数:\n style (str): 音乐结构风格名，对应 STYLE_STRUCTURE_LIB 中的 key。\n user_prompt (str): 用户自定义的风格描述文本，将传入 LLM 构造提示。\n total_sec (int): 希望生成的总时长（秒），提示中允许略超，但不得不足。\n window_sec (int): 每个段落的时间粒度，必须是段落时长的倍数。\n\n 返回:\n List[Dict]: 每个元素为一个分段字典，包含字段：\n - section: 段名称\n - duration: 时长（秒）\n - prompt: 用于音乐生成的提示词\n - temperature, top_p, top_k: 控制采样的超参数\n - end_at: 当前段结束时刻（累加）\n\n 异常:\n ValueError: 若 style 无法匹配结构库、或响应无法被正确解析时抛出。\n\n 注意:\n - 默认使用远程 OpenAI API，也支持通过 `use_local_llm` 使用 Ollama 模型。\n - 方法内部自动进行重试机制，并打印失败原因。\n - 所有生成内容中的 prompt 字段将仅使用提供的 token 列表，保证下游模型兼容性。\n \"\"\"\n structure = STYLE_STRUCTURE_LIB.get(style.lower())\n if not structure:\n raise ValueError(f\"风格 '{style}' 不支持\")\n structure = \", \".join(structure)\n if self.verbose:\n print(\"structure is:\"+ structure)\n\n\n fluent_prompt = f\"\"\"You are an experienced music producer. \n Please generate a JSON array of song segments where each item has the following keys: \n - section (string) \n - duration (int, seconds) \n - prompt (string) \n - temperature (float) \n - top_p (float) \n - top_k (float) \n \n Return only the raw JSON array without any markdown fences.\n \n Description is: {user_prompt}, Style is: {style}.\n The total duration should be about {total_sec} seconds (you may exceed but not fall short).\n Use these sections in order: {structure}.\n Each segment’s duration must be a multiple of {window_sec} seconds (e.g. if window_sec = 30, valid values are 30, 60, 90, …).\n top_p must be between 0.6 and 0.9, top_k should be about 250, temperature should be higher than 1.0.\n Construct the prompt field for each segment depending on description {user_prompt} and style {style}.\n When constructing the prompt field inside each segment, use ONLY the following tokens, organized by category. \n Ensuring that any Genre / Style token you choose exactly matches the value in {style}:\n \n Structure: \n intro-like, verse-like, pre-chorus-like, chorus-like, bridge-like, build-up-like, drop-like, interlude-like, outro-like.\n \n Mood / Emotion: \n calming, soothing, serene, ethereal, dreamy, happy, joyful, cheerful, gentle, groovy, funky, sad, melancholic, nostalgic, mournful, peaceful, uplifting, hopeful, inspirational, epic, triumphant, energetic, aggressive, angry, dark, mysterious, spooky, tense, suspenseful, anxious, majestic, glorious, romantic, playful, whimsical.\n \n Genre / Style: \n {style}.\n \n Instrument / Voice: \n acoustic guitar, electric guitar, classical guitar, bass guitar, electric bass, acoustic bass, ukulele, banjo, mandolin, piano, electric piano, upright piano, harpsichord, organ, pipe organ, clavinet, synthesizer, bass synth, pad, lead synth, violin, viola, cello, double bass, string ensemble, orchestral strings, flute, piccolo, recorder, pan flute, clarinet, bass clarinet, oboe, english horn, bassoon, contrabassoon, saxophone, soprano saxophone, alto saxophone, tenor saxophone, baritone saxophone, trumpet, piccolo trumpet, cornet, flugelhorn, trombone, bass trombone, tuba, french horn, euphonium, harmonica, accordion, concertina, bagpipe, sitar, shamisen, koto, erhu, pipa, dizi, ocarina, didgeridoo, marimba, xylophone, vibraphone, glockenspiel, celesta, steel drum, drums, acoustic drum kit, electronic drum kit, snare drum, bass drum, cymbals, hi-hat, tom-tom, timpani, congas, bongo, djembe, tabla, triangle, tambourine, shaker, clap, percussion, male vocals, female vocals, mixed choir, children choir, solo voice.\n \n Environment: \n rain, heavy rain, thunder, wind, snowstorm, ocean waves, seashore, underwater ambience, forest birds, rainforest, desert wind, city daytime, city night, traffic noise, street ambience, airport terminal, subway station, train station, bus station, marketplace, stadium crowd, children playground, office background, classroom, library, crowd murmur, cafe background, industrial, construction noise, fireplace crackle, fireworks, police siren, space ambience, windchimes.\n \n Production Style: \n lo-fi, hi-fi, vintage, analog, digital, electronic, acoustic, orchestral, cinematic, soundtrack, psychedelic, experimental, noise, glitch, ambient, warm, bright, dark, harsh, smooth, compressed, bassy, spatial, surround, stereo wide, monophonic, polyphonic, minimal, maximalist, bitcrushed, chiptune, distorted, reverb-heavy, gated reverb, side-chained, auto-tuned.\n \n Energy / Tempo: \n high energy, powerful, relaxed, calm, dynamic, steady, variable, driving, lively, euphoric, hypnotic, pulsing, accelerando, decelerando.\n \n Chord / Harmony: \n C major, C minor, C7, Cmaj7, Cmin7, A major, A minor, A7, Amaj7, Amin7, Bb major, Bb minor, Bb7, Bbmaj7, B major, B minor, B7, Bmaj7, Bmin7, D major, D minor, D7, Dmaj7, Dmin7, E major, E minor, E7, Emaj7, Emin7, F major, F minor, F7, Fmaj7, Fmin7, F# major, F# minor, F#7, F#maj7, F#min7, G major, G minor, G7, Gmaj7, Gmin7, Ab major, Ab minor, Ab7, Abmaj7, Abmin7.\n\n CLAP generate Harmony:\n C, C#, D, D#, E, F, F#, G, G#, A, A#, B\n \n Example of two valid prompt strings for one segment: \n build-up-like, serene, delta blues, acoustic guitar, windchimes, soundtrack, accelerando, C minor, G \n interlude-like, suspenseful, neo soul, orchestral strings, space ambience, soundtrack, dynamic, C minor, G#.\n \n Return only the JSON array.\n \"\"\"\n\n # ambient, dark ambient, ambient techno, new age, chillout, lofi hip hop, trip hop, future bass, synthwave, vaporwave, house, deep house, progressive house, tech house, techno, psytrance, trance, drum and bass, jungle, dubstep, brostep, trap, EDM, jazz, smooth jazz, swing, bebop, latin jazz, bossa nova, samba, fusion, classical, baroque, romantic, modern classical, contemporary classical, orchestral, opera, chamber music, rock, classic rock, hard rock, punk rock, post-punk, indie rock, alternative rock, progressive rock, shoegaze, grunge, metal, heavy metal, death metal, black metal, thrash metal, pop, synth pop, indie pop, electropop, k-pop, j-pop, R&B, neo soul, soul, funk, disco, gospel, hip hop, rap, boom bap, trap rap, blues, delta blues, reggae, dub, ska, folk, bluegrass, country, americana, celtic, afrobeat, salsa, tango, flamenco, balkan, bollywood, world music, soundtrack, cinematic, video game, experimental, noise.\n \n raw = None\n for attempt in range(1, max_retries+1):\n try:\n if self.use_local_llm:\n # ➡️ 调 Ollama Chat —— \n resp = ollama.chat(\n model=\"deepseek-r1:14b\",\n messages=[{\"role\": \"user\", \"content\": fluent_prompt}]\n )\n \n # 直接从响应字典中提取文本\n try:\n text = resp[\"message\"][\"content\"]\n except Exception as e:\n raise ValueError(f\"⚠️ 无法解析 Ollama 返回结构: {e}\\n原始内容:\\n{resp}\")\n \n # 打印内容供 debug\n if self.verbose:\n print(\"the text is : \" + text)\n \n # 尝试提取 JSON 数组（包括 ```json ... ``` 或直接 [...]）\n m = re.search(r\"```json\\s*($begin:math:display$.*?$end:math:display$)\\s*```\", text, re.S)\n if not m:\n m = re.search(r\"\\[.*?\\]\", text, re.S)\n \n if not m:\n raise ValueError(f\"❌ 无法从文本中提取 JSON 数组:\\n{text}\")\n \n # 提取原始 JSON 内容\n raw = m.group(1) if m.lastindex else m.group(0)\n raw = raw.encode('utf-8').decode('unicode_escape')\n raw = raw.replace('\\\\\"', '\"')\n\n else:\n # ➡️ 调 LLM API\n payload = {\n \"model\": self.openai_model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": fluent_prompt}\n ]\n }\n headers = {\n \"Authorization\": f\"Bearer {self.openai_api_key}\",\n \"Content-Type\": \"application/json\"\n }\n\n url = self.api_endpoint\n \n resp = requests.post(\n url, \n headers=headers, \n json=payload\n )\n\n resp.raise_for_status()\n raw = resp.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n \n \n if self.verbose:\n print(\"☀️ the raw json is : \"+raw)\n print(f\"[SegPlan] 成功获得规划 (attempt {attempt})\")\n break\n \n # 异常处理与多次重试 \n except requests.RequestException as e:\n if attempt < max_retries:\n wait = min(2 ** (attempt-1) , 60)\n if self.verbose:\n print(f\"[SegPlan] 请求失败，第{attempt}次重试，{wait}s后再试… ({e})\")\n time.sleep(wait)\n else:\n if self.verbose:\n print(f\"[SegPlan] 重试{max_retries}次失败，使用默认分段（❌没有完成，寄）\")\n # 默认回退\n if not raw:\n # 请求失败，使用固定默认分段列表，并继续后续解析流程\n if self.verbose:\n print(\"[SegPlan] 请求多次失败，使用固定默认分段列表\")\n default_segs = [\n {'section': 'minimal-pulse', 'duration': 30, 'prompt': 'intro-like, peaceful, ambient, bass synth, pad, warm, reverb-heavy, pulsing, Cmaj7, C', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'arpeggio-loop', 'duration': 30, 'prompt': 'verse-like, calming, ambient, synthesizer, pad, warm, smooth, slow, Amin7, A', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 8.0},\n {'section': 'white-noise-bed', 'duration': 30, 'prompt': 'pre-chorus-like, soothing, ambient, pad, space ambience, warm, reverb-heavy, slow, Dmin7, D', 'temperature': 0.65, 'top_p': 0.75, 'cfg': 6.5},\n {'section': 'modulation-bridge', 'duration': 30, 'prompt': 'bridge-like, ethereal, ambient, string ensemble, pad, warm, reverb-heavy, slow, Gmaj7, G', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 7.5},\n {'section': 'repetitive-theme', 'duration': 30, 'prompt': 'chorus-like, dreamy, ambient, string ensemble, pad, warm, smooth, slow, Fmaj7, F', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'soft-ending', 'duration': 30, 'prompt': 'outro-like, serene, ambient, pad, string ensemble, warm, smooth, decelerando, Cmaj7, C', 'temperature': 0.5, 'top_p': 0.6, 'cfg': 6.0},\n ]\n # 用默认列表生成 raw JSON，继续解析流程\n raw = json.dumps(default_segs, ensure_ascii=False)\n \n # —— 解析并补齐字段 —— 如果 LLM 给的内容有缺损，使用默认的参数补齐它\n segs = json.loads(raw)\n acc = 0\n for s in segs:\n s[\"duration\"] = int(s.get(\"duration\", window_sec))\n s[\"prompt\"] = s.get(\"prompt\", user_prompt)\n s[\"temperature\"] = float(s.get(\"temperature\", 0.7))\n s[\"top_p\"] = float(s.get(\"top_p\", 0.9))\n s[\"top_k\"] = float(s.get(\"top_k\", 250))\n acc += s[\"duration\"]\n s[\"end_at\"] = acc\n if self.verbose:\n print(\"[SegPlan] Parsed segments:\")\n for x in segs: print(x)\n\n return segs\n\n def compute_si_sdr(self, ref: torch.Tensor, est: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算 Scale-Invariant Signal-to-Distortion Ratio（SI-SDR），用于评估估计音频与参考音频之间的相似度。\n\n SI-SDR 是一种用于盲音源分离和音频质量评价的指标，具备尺度不变性，\n 可以排除音量（能量）缩放对相似度评分的影响。\n\n 参数:\n ref (torch.Tensor): 参考音频信号，形状为 [C, T] 或 [T]（C 为通道数，T 为时间长度）。\n est (torch.Tensor): 估计音频信号，形状与 ref 相同或兼容。\n eps (float): 避免除零错误的微小常数，默认值为 1e-8。\n\n 返回:\n torch.Tensor: 每个样本/通道的 SI-SDR 得分（以 dB 为单位），形状为 [C] 或标量。\n \"\"\"\n # 对齐 ref 和 est 的时间长度，支持单通道或多通道输入\n if ref.dim() == 2 and est.dim() == 2:\n T = min(ref.shape[1], est.shape[1])\n ref, est = ref[:, :T], est[:, :T]\n else:\n T = min(ref.shape[-1], est.shape[-1])\n ref, est = ref[:T], est[:T]\n\n # 去直流分量（零均值化）\n ref_zm = ref - ref.mean(-1, keepdim=True)\n est_zm = est - est.mean(-1, keepdim=True)\n\n # 计算参考信号能量\n ref_energy = torch.sum(ref_zm ** 2, dim=-1, keepdim=True)\n\n # 计算最佳缩放因子 scale，使得投影后的信号最接近估计信号\n scale = torch.sum(ref_zm * est_zm, dim=-1, keepdim=True) / (ref_energy + eps)\n\n # 将估计信号投影到参考信号方向\n proj = scale * ref_zm\n\n # 残差噪声 = 去中心化估计信号 - 投影部分\n noise = est_zm - proj\n\n # 计算信噪比（使用投影部分作为信号）\n snr = torch.sum(proj ** 2, dim=-1) / (torch.sum(noise ** 2, dim=-1) + eps)\n\n # 返回以 dB 为单位的 SI-SDR 分数\n return 10 * torch.log10(snr + eps)\n\n def rms_db(self, x: torch.Tensor) -> float:\n return 20*torch.log10(x.square().mean().sqrt()+1e-9).item()\n\n def mel_distance(self, x: torch.Tensor, y: torch.Tensor) -> float:\n \"\"\"\n 计算两个音频信号的梅尔谱（Mel Spectrogram）之间的 L2 距离，用于衡量音频在感知上的相似性。\n\n 此方法首先对两个音频信号做对齐裁剪，并通过 `self.mel_transform` 提取梅尔谱特征，\n 再计算其展开后的欧氏距离（L2 norm）。适用于感知相似度评估、音频生成对比等任务。\n\n 参数:\n x (torch.Tensor): 第一个音频信号，形状为 [T] 或 [C, T]。\n y (torch.Tensor): 第二个音频信号，形状与 x 相同或兼容。\n\n 返回:\n float: 梅尔谱之间的 L2 距离，数值越小表示音频越相似。\n \"\"\"\n # 对齐两个音频在时间维度上的长度\n if x.dim()==2 and y.dim()==2:\n T = min(x.shape[1], y.shape[1])\n x, y = x[:,:T], y[:,:T]\n else:\n T = min(x.shape[-1], y.shape[-1])\n x, y = x[:T], y[:T]\n\n # 通过 mel_transform 提取梅尔谱特征（添加 batch 维度）\n Mx = self.mel_transform(x.unsqueeze(0)) # [1, F, T']\n My = self.mel_transform(y.unsqueeze(0)) # [1, F, T']\n\n # 将梅尔谱差值展平后计算 L2 距离\n return torch.norm((Mx-My).reshape(-1), p=2).item()\n\n def xfade(self, a: torch.Tensor, b: torch.Tensor, ov: int) -> torch.Tensor:\n \"\"\"\n 将两段音频 a 和 b 在尾部和头部以交叉渐变方式进行拼接，实现平滑过渡。\n 自动对齐声道：如果 a 是多声道但 b 单声道，自动复制 b 的声道数。\n \"\"\"\n print(\"xfade input shapes:\", a.shape, b.shape)\n # 1) 自动对齐声道\n if a.dim() == 2 and b.dim() == 2 and a.shape[0] != b.shape[0]:\n # 如果 b 只有 1 个通道，但 a 有多个通道，就复制 b\n if b.shape[0] == 1:\n b = b.repeat(a.shape[0], 1)\n # 反之亦然（极少用到）\n elif a.shape[0] == 1:\n a = a.repeat(b.shape[0], 1)\n\n # 2) 计算 overlap 点数\n n = ov * self.SR\n\n # 3) 构造渐变权重\n w = torch.linspace(0, 1, n, device=a.device).sqrt()\n\n # 4) 交叉区混合\n left = a[..., :-n]\n middle = a[..., -n:] * (1 - w) + b[..., :n] * w\n right = b[..., n:]\n\n # 5) 拼接并返回\n return torch.cat([left, middle, right], dim=-1)\n\n # ====================== generate ======================\n @torch.no_grad()\n def generate(\n self,\n seed_path: str,\n style: str,\n user_prompt: str,\n total_sec: int = 120,\n window_sec: int = 20,\n overlap_sec: int = 5,\n prompt_sec: int = 5,\n safety_tail: int = 3,\n n_tracks: int = 1,\n clap_candidates: int = 3,\n sim_thresh: float = 0.5,\n sil_thresh: float = -40,\n melody_path: Optional[str] = None,\n out_path: Optional[str] = None,\n use_fp16: bool = False,\n ) -> Union[str, List[str]]:\n \"\"\"\n 基于种子音频和用户提示，分段、生成并拼接完整音乐轨道。\n\n 整体流程：\n 1. 读取 seed_path 指定的音频文件并重采样、提取 CLAP 嵌入；\n 2. （可选）读取 melody_path 指定的条件音频；\n 3. 调用 request_segment_plan 获取分段规划；\n 4. 对每个段落并行生成 clap_candidates 条候选，并按相似度选出最佳；\n 5. 通过 xfade 平滑拼接到当前音轨；\n 6. 重复直到累计生成达到 total_sec，并保存最终音轨。\n\n 参数:\n seed_path (str):\n 初始音频文件路径，用于续写输入。\n style (str):\n 分段风格名称，对应内部 STYLE_STRUCTURE_LIB 的键。\n user_prompt (str):\n 用户文本提示，用于精细化每段的生成风格。\n total_sec (int, 可选):\n 目标总时长（秒）。默认 120。\n window_sec (int, 可选):\n 每次生成的片段时长（秒）。默认 20。\n overlap_sec (int, 可选):\n 段落重叠长度（秒），用于淡入淡出拼接。默认 5。\n prompt_sec (int, 可选):\n 每次构建生成 prompt 时要截取的尾部时长（秒）。默认 5。\n safety_tail (int, 可选):\n 生成 prompt 前保留的不参与提示的尾巴长度（秒），避免拼接突变。默认 3。\n n_tracks (int, 可选):\n 并行生成的轨道数量；>1 时返回路径列表。默认 1。\n clap_candidates (int, 可选):\n 每段并行生成的候选数，用于多版本筛选。默认 3。\n sim_thresh (float, 可选):\n 最优候选的相似度阈值；低于则触发重试或跳过。默认 0.5。\n sil_thresh (float, 可选):\n 静音检测阈值（dB）；可用于剔除无声片段。默认 -40。\n melody_path (Optional[str], 可选):\n 条件 melody 音频文件路径；不提供则进入纯生成模式。默认 None。\n out_path (Optional[str], 可选):\n 最终输出文件路径；多轨时自动编号。默认 None。\n use_fp16 (bool, 可选):\n 在 CUDA 上是否启用半精度推理（torch.cuda.amp）。默认 False.\n\n 返回:\n Union[str, List[str]]:\n 生成的音频文件路径。若 n_tracks>1 则为路径列表，否则为单个字符串。\n\n 异常:\n ValueError:\n - 当分段规划（request_segment_plan）失败且无法解析时。\n - 当读取音频文件或网络请求出错并超过重试次数时。\n\n 示例:\n >>> out = svc.generate(\n ... seed_path=\"seed.wav\",\n ... style=\"ambient\",\n ... user_prompt=\"dreamy pad with rain\",\n ... total_sec=180,\n ... n_tracks=2,\n ... clap_candidates=5,\n ... use_fp16=True,\n ... out_path=\"output/track.wav\"\n ... )\n \"\"\"\n\n print(f\"[Generate] start: n_tracks={n_tracks}, clap_candidates={clap_candidates}, use_fp16={use_fp16}, device={self.device}\", flush=True)\n\n fp16_ctx = torch.cuda.amp.autocast if (use_fp16 and self.device.startswith(\"cuda\")) else nullcontext\n\n def _run_once(idx: int) -> str:\n checkpoint_path= self.checkpoint_path\n USE_BF16 = torch.cuda.is_bf16_supported()\n DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n ENABLE_COMPILE = True # False 可快速回退\n STRICT_LOAD = False\n\n print(f\"[Track {idx+1}/{n_tracks}] >>> begin _run_once\", flush=True)\n torch.manual_seed(42 + idx)\n\n # 1) 载入 seed 音频\n seed32, sr = torchaudio.load(seed_path)\n if sr != self.SR:\n seed32 = torchaudio.functional.resample(seed32, sr, self.SR)\n seed32 = seed32.to(self.device)\n ref_emb = self.embed_cpu(seed32)\n \n # 2) 段落规划\n segs = self.request_segment_plan(style, user_prompt, total_sec, window_sec)\n # 修正：初始full应该足够长以包含第一次生成的prompt\n # 需要至少 prompt_sec + safety_tail 的长度\n min_need = (prompt_sec + safety_tail) * self.SR\n need = max(min_need, (window_sec - overlap_sec) * self.SR)\n \n full = seed32[..., :need] if seed32.shape[-1] >= need else seed32.repeat(\n 1, math.ceil(need / seed32.shape[-1])\n )[..., :need]\n cur_seg = 0\n\n # 3) 每段生成参数设置\n def set_params(idx_seg):\n p = segs[idx_seg]\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,#p[\"temperature\"],\n top_p=0.9,#p[\"top_p\"],\n top_k=250,#p[\"cfg\"],\n #use_sampling=True,\n )\n\n set_params(cur_seg)\n retry_count = 0\n max_retries = 6\n\n # 4) 主循环：续写直到达到 total_sec\n # if self.verbose:\n # print(f\"[Debug] 初始full长度: {full.shape[-1]/self.SR:.1f}s, 目标: {total_sec}s\")\n # print(f\"[Debug] 第一个segment end_at: {segs[0]['end_at']}s\")\n \n while full.shape[-1] / self.SR < total_sec:\n # if self.verbose:\n # print(f\"[Debug] 进入生成循环，当前full: {full.shape[-1]/self.SR:.1f}s\")\n \n # 切换到下一个段落\n if full.shape[-1] / self.SR >= segs[cur_seg][\"end_at\"] and cur_seg < len(segs) - 1:\n cur_seg += 1\n set_params(cur_seg)\n retry_count = 0\n if self.verbose:\n print(f\"▶ Enter segment {cur_seg}: {segs[cur_seg]['section']}\")\n \n\n # 准备 prompt 音频片段\n tail = full.shape[-1] - safety_tail * self.SR\n head = max(0, tail - prompt_sec * self.SR)\n pa = full[..., head:tail]\n \n # if self.verbose:\n # print(f\"[Debug] prompt准备过程:\")\n # print(f\" - full总长度: {full.shape[-1]/self.SR:.1f}s\")\n # print(f\" - safety_tail: {safety_tail}s\")\n # print(f\" - prompt_sec: {prompt_sec}s\")\n # print(f\" - tail位置: {tail/self.SR:.1f}s\")\n # print(f\" - head位置: {head/self.SR:.1f}s\")\n # print(f\" - 提取的pa原始长度: {pa.shape[-1]/self.SR:.1f}s\")\n # print(f\" - pa shape: {pa.shape}\")\n \n if pa.shape[-1] < 1024:\n pa = torch.nn.functional.pad(pa, (1024 - pa.shape[-1], 0), \"reflect\")\n # if self.verbose:\n # print(f\" - padding后pa长度: {pa.shape[-1]/self.SR:.3f}s\")\n \n pa = pa.unsqueeze(0)\n \n # if self.verbose:\n # print(f\"[Debug] 最终prompt音频:\")\n # print(f\" - shape: {pa.shape}\")\n # print(f\" - 长度: {pa.shape[-1]/self.SR:.3f}s\")\n # print(f\" - dtype: {pa.dtype}\")\n # print(f\" - device: {pa.device}\")\n # print(f\" - 设置duration: {window_sec}s\")\n\n # 并行多候选生成\n current_prompt = segs[cur_seg][\"prompt\"]\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] generating {clap_candidates} candidates\", flush=True)\n descriptions = [current_prompt] * clap_candidates\n \n\n with fp16_ctx():\n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n \n # 添加详细的模型状态调试信息\n # if self.verbose:\n # print(f\"[Debug] 检查模型内部状态:\")\n # try:\n # print(f\" - self.model.duration: {getattr(self.model, 'duration', 'None')}\")\n # print(f\" - self.model.sample_rate: {getattr(self.model, 'sample_rate', 'None')}\")\n # if hasattr(self.model, 'generation_params'):\n # print(f\" - generation_params: {getattr(self.model, 'generation_params', 'None')}\")\n # if hasattr(self.model, '_gen_model') and self.model._gen_model:\n # gen_model = self.model._gen_model\n # print(f\" - _gen_model类型: {type(gen_model)}\")\n # if hasattr(gen_model, 'max_duration'):\n # print(f\" - _gen_model.max_duration: {gen_model.max_duration}\")\n # except Exception as e:\n # print(f\" - 检查模型状态时出错: {e}\")\n \n # 重新确认generation参数 - 使用更明确的方式\n try:\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n # if self.verbose:\n # print(f\"[Debug] ✅ 成功设置generation_params，duration={window_sec}\")\n except Exception as e:\n # if self.verbose:\n # print(f\"[Debug] ❌ 设置generation_params失败: {e}\")\n pass\n \n # 验证设置后的参数\n # if self.verbose:\n # print(f\"[Debug] 设置后验证:\")\n # try:\n # print(f\" - self.model.duration: {getattr(self.model, 'duration', 'None')}\")\n # if hasattr(self.model, 'compression_model'):\n # print(f\" - compression_model.sample_rate: {getattr(self.model.compression_model, 'sample_rate', 'None')}\")\n # except Exception as e:\n # print(f\" - 验证时出错: {e}\")\n \n # if self.verbose:\n # print(f\"[Debug] 调用generate_continuation参数:\")\n # print(f\" - prompt shape: {pa.shape}\")\n # print(f\" - prompt_sample_rate: {self.SR}\")\n # print(f\" - descriptions数量: {len(descriptions)}\")\n # print(f\" - descriptions[0]: {descriptions[0][:100]}...\")\n # print(f\" - 期望duration: {window_sec}s\")\n # \n # # 检查generate_continuation方法签名\n # import inspect\n # sig = inspect.signature(self.model.generate_continuation)\n # print(f\" - generate_continuation方法签名: {sig}\")\n \n # 尝试显式传递所有参数\n try:\n # if self.verbose:\n # print(f\"[Debug] 🚀 开始调用generate_continuation...\")\n \n wavs = self.model.generate_continuation(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n progress=self.verbose,\n )\n \n # if self.verbose:\n # print(f\"[Debug] ✅ generate_continuation调用成功\")\n \n except Exception as e:\n # if self.verbose:\n # print(f\"[Debug] ❌ generate_continuation调用失败: {e}\")\n # import traceback\n # traceback.print_exc()\n # 如果失败，返回空列表避免崩溃\n wavs = []\n \n # if self.verbose:\n # print(f\"[Debug] generate_continuation返回结果:\")\n # print(f\" - 返回wav数量: {len(wavs)}\")\n # if len(wavs) > 0:\n # print(f\" - wavs[0] shape: {wavs[0].shape}\")\n # print(f\" - wavs[0] dtype: {wavs[0].dtype}\")\n # print(f\" - wavs[0] device: {wavs[0].device}\")\n # print(f\" - 实际生成时长: {wavs[0].shape[-1]/self.SR:.3f}s\")\n # print(f\" - 期望时长: {window_sec}s\")\n # print(f\" - 时长比例: {(wavs[0].shape[-1]/self.SR)/window_sec:.3f}\")\n # \n # # 检查是否包含prompt\n # prompt_len = pa.shape[-1]\n # wav_len = wavs[0].shape[-1]\n # print(f\" - prompt长度: {prompt_len/self.SR:.3f}s\")\n # print(f\" - 生成结果长度: {wav_len/self.SR:.3f}s\")\n # print(f\" - 是否包含prompt: {wav_len >= prompt_len}\")\n # \n # if wav_len < window_sec * self.SR * 0.9: # 如果小于期望的90%\n # print(f\" - ⚠️ 警告：生成长度不足期望的90%\")\n # if wav_len < prompt_len * 1.5: # 如果生成的内容比prompt长度少于50%\n # print(f\" - ⚠️ 警告：生成内容可能主要是prompt重复\")\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] got {len(wavs)} wavs\", flush=True)\n # if len(wavs) > 0:\n # print(f\"[Debug] 生成的音频长度: {wavs[0].shape[-1]/self.SR:.1f}s\")\n\n # 5) 选出相似度最高的 wav，同时收集所有 sim 值\n sim_list = []\n best, best_sim = None, -1.0\n \n def _ensure_mono32(t: torch.Tensor, sr: int = 32000) -> torch.Tensor:\n \"\"\"\n 保证张量变成 [1, T] 单声道并且已是 32 kHz。\n 这里假设外层音频都已重采样到 self.SR=32 kHz；仅做声道合并。\n \"\"\"\n if t.dim() == 2 and t.shape[0] > 1: # 立体声 → 单声道\n t = t.mean(0, keepdim=True)\n return t.to(self.device)\n\n for i, wav in enumerate(wavs):\n # ------------- 1) 规格统一 -------------\n wav_mono = _ensure_mono32(wav) # 生成片段（完整的30秒）\n ref_tail = _ensure_mono32(pa.squeeze(0)) # prompt tail（5秒）\n \n # 相似度计算：只比较开头的prompt部分\n prompt_len = ref_tail.shape[-1]\n wav_prompt = wav_mono[..., :prompt_len] # 取生成结果的前5秒用于相似度比较\n \n # if self.verbose:\n # print(f\"[Debug] 相似度计算:\")\n # print(f\" - wav_mono总长: {wav_mono.shape[-1]/self.SR:.1f}s\")\n # print(f\" - ref_tail长度: {ref_tail.shape[-1]/self.SR:.1f}s\") \n # print(f\" - wav_prompt长度: {wav_prompt.shape[-1]/self.SR:.1f}s\")\n\n # ------------- 2) 计算三项相似度 ----------\n sim_clap = torch.cosine_similarity(\n self.embed_cpu(wav_prompt), self.embed_cpu(ref_tail)\n ).item()\n\n sdr_db = self.compute_si_sdr(ref_tail, wav_prompt).mean().item() # dB\n mel_l2 = self.mel_distance(ref_tail, wav_prompt) # L2\n\n # 映射到 0–1 区间\n sdr_norm = min(max((sdr_db - 0) / 10.0, 0.0), 1.0) # 0 dB→0, 10 dB→1\n mel_norm = min(250.0 / (mel_l2 + 1.0), 1.0) # 250→1, 500→0.5\n\n # 综合得分\n sim_final = 0.4 * sim_clap + 0.4 * sdr_norm + 0.2 * mel_norm\n\n # ------------- 3) 调试输出 --------------\n if self.verbose:\n print(\n f\"CLAP={sim_clap:.3f} \"\n f\"SDR={sdr_db:.2f}dB(n={sdr_norm:.2f}) \"\n f\"MelL2={mel_l2:.0f}(n={mel_norm:.2f}) \"\n f\"final={sim_final:.3f}\"\n )\n\n # ------------- 4) 选最佳 ----------------\n sim_list.append(sim_final)\n if sim_final > best_sim:\n # 重要：保存完整的wav_mono（30秒），而不是只保存prompt部分（5秒）\n best, best_sim = wav_mono.cpu(), sim_final\n del wav\n\n\n torch.cuda.empty_cache()\n\n # 打印这一段所有候选的 sim 数组和选出的 best_sim\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] sims = {sim_list}\")\n print(f\"[Track {idx+1} Segment {cur_seg}] best_sim = {best_sim:.4f}\")\n\n # save_best_each = True\n save_best_each = False\n # 立刻保存本段 best 候选 —— \n if save_best_each and out_path:\n # out_path 父目录下创建 bestwav 文件夹\n best_dir = pathlib.Path(out_path).parent / \"bestwav\"\n best_dir.mkdir(parents=True, exist_ok=True)\n save_name = best_dir / f\"Segment_{cur_seg}.wav\"\n\n # 生成子文件名，并写文件\n # 生成微调结果\n # 把 float16 转成 float32，再保存\n \n audio_ft = best.cpu().float().numpy()\n if audio_ft.ndim == 2 and audio_ft.shape[0] < audio_ft.shape[1]:\n audio_ft = audio_ft.T\n sf.write(str(save_name), audio_ft, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1} Seg {cur_seg}] saved best wav → {save_name}\")\n\n # 6) 相似度太低时重试或跳过\n if best_sim < sim_thresh:\n retry_count += 1\n if retry_count > max_retries:\n if self.verbose:\n print(f\"Segment {cur_seg} reached max retries, skipping segment\")\n retry_count = 0\n break\n p = segs[cur_seg]\n new_temp = max(0.15, p[\"temperature\"] * (0.9 ** retry_count))\n new_top_k = p[\"top_k\"] * (1.15 ** retry_count)\n new_top_p = min(p[\"top_p\"]+0.05,0.95)\n self.model.set_generation_params(\n duration=window_sec,\n temperature=new_temp,\n top_k=new_top_k,\n top_p = new_top_p,\n use_sampling=True,\n )\n if self.verbose:\n print(f\"Retry {retry_count}: temp→{new_temp:.2f}, topk→{new_top_k:.2f}\")\n continue\n\n # 7) 拼接并输出调试信息\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] before xfade: full={full.shape[-1]/self.SR:.1f}s, best={best.shape[-1]/self.SR:.1f}s\")\n full = self.xfade(full, best.to(self.device), overlap_sec)\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] after xfade: {full.shape[-1]/self.SR:.1f}s\")\n # print(f\"[Debug] xfade参数: overlap_sec={overlap_sec}, 预期增长={(window_sec-overlap_sec):.1f}s\")\n del best\n torch.cuda.empty_cache()\n retry_count = 0\n\n # 完成 single-track 生成，导出文件\n final = full[..., : total_sec * self.SR].cpu().T\n if out_path:\n stem, ext = os.path.splitext(out_path)\n out_file = f\"{stem}_{idx+1:02d}{ext}\" if n_tracks > 1 else out_path\n else:\n out_file = f\"final_{idx+1:02d}.wav\" if n_tracks > 1 else \"final.wav\"\n\n pathlib.Path(out_file).parent.mkdir(parents=True, exist_ok=True)\n sf.write(out_file, final, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1}] saved {out_file}, best_sim={best_sim:.4f}\", flush=True)\n return out_file\n\n # 并行生成多条 track\n print(f\"[Generate] launching _run_once for {n_tracks} tracks\", flush=True)\n paths = [_run_once(i) for i in range(n_tracks)]\n print(f\"[Generate] all done, paths = {paths}\", flush=True)\n return paths if n_tracks > 1 else paths[0]\n\n \n\n\ndef generate_music_api(params: Dict) -> Dict:\n # svc = MusicGenService(\n # api_endpoint=params.get('api_endpoint', API_ENDPOINT),\n # openai_api_key=params.get('api_key', OPENAI_API_KEY),\n # openai_model=params.get('model', OPENAI_MODEL),\n # verbose=params.get('verbose', True),\n # checkpoint_path=params.get('checkpoint_path'), # ← 把它传进去\n # )\n svc = MusicGenService(\n use_local_llm=params.get('use_local_llm', DEFAULT_USE_LOCAL_LLM),\n api_endpoint=params.get('api_endpoint'), # 允许外部自定义\n openai_api_key=params.get('api_key', OPENAI_API_KEY),\n openai_model=params.get('model', OPENAI_MODEL),\n verbose=params.get('verbose', True),\n checkpoint_path=params.get('checkpoint_path'),\n )\n out = svc.generate(\n seed_path=params['seed_path'],\n style=params['style'],\n user_prompt=params['user_prompt'],\n total_sec=params.get('total_sec', 120),\n window_sec=params.get('window_sec', 20),\n overlap_sec=params.get('overlap_sec', 5),\n prompt_sec=params.get('prompt_sec', 5),\n safety_tail=params.get('safety_tail', 3),\n clap_candidates= 1,#params.get('clap_candidates', 3),\n n_tracks=params.get('n_tracks', 1), # ← 新增这一行\n use_fp16=params.get('use_fp16', False), # ← 以及这一行\n sim_thresh=params.get('sim_thresh', 0.5),\n sil_thresh=params.get('sil_thresh', -40),\n melody_path=params.get('melody_path'),\n out_path=params.get('out_path'),\n )\n return {'out_path': out}\n\n"}}, "lm_train/.ipynb_checkpoints/musicgen_api-checkpoint.py": {"486bab9207d6499336bfe192c2ed40eb91b6c4b0": {"t": 1785768920.4988408, "n": 34187, "h": "c713407008ea5657", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nmusicgen_api.py\n\n封装 MusicGen 无限续写脚本为后端可调用接口。\n支持 Melody 条件生成、增量续写、重试与显存管理。\n\"\"\"\n\nnetworks = False # 如果没有接入国际互联网，就使用 False\n\n\nimport os\n# 如果没有接入国际互联网，就将 Huggingface 的访问设置为【不访问】\nif not networks:\n # 确保 transformers/hub 都用离线模式\n os.environ[\"HF_HUB_OFFLINE\"] = \"1\"\n os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n # 然后 monkey-patch BertTokenizer.from_pretrained，让它只走本地缓存\n from transformers import BertTokenizer\n _original_bp = BertTokenizer.from_pretrained\n def _from_pretrained_local(repo_id, *args, **kwargs):\n # 强制 local_files_only=True\n return _original_bp(repo_id, local_files_only=True, *args, **kwargs)\n BertTokenizer.from_pretrained = _from_pretrained_local\n\n\nimport json\nimport math\nimport warnings\nimport re\nimport time\nimport ollama\nimport torchaudio\nimport torch\nimport requests\nimport pathlib\nimport soundfile as sf\n\nfrom contextlib import nullcontext\nfrom typing import List, Optional, Union, Dict\nfrom urllib.parse import urljoin\nfrom audiocraft.models import MusicGen\nfrom laion_clap import CLAP_Module\nfrom music_structure import STYLE_STRUCTURE_LIB\n\n\n# ========== 默认配置==========\n\nuse_local_llm = False # 是否使用本地的LLM （需要打开 OLLAMA）\n\nOPENAI_MODEL = 'gpt-4o-mini' # 如果使用网络 LLM API， 这是使用的模型\nmax_retries = 6 # 如果使用网络 LLM API，这是最大重试次数\n\nif use_local_llm :\n API_ENDPOINT = \"http://localhost:11434/api/generate\" #本地LLM访问地址\nelse:\n API_ENDPOINT = 'https://www.dmxapi.com/v1/chat/completions' # 网络LLM访问地址\n\nOPENAI_API_KEY = 'sk-vWyNj5NxixWjJcjknaxNivrtIkHIjXDBVlk1b6wRW2pZc0hz' # 网络LLM API KEY\n\n\nUSE_BF16 = torch.cuda.is_bf16_supported() # \nDTYPE = torch.bfloat16 if USE_BF16 else torch.float16 \n\nENABLE_COMPILE = True # False 可快速回退\nSTRICT_LOAD = False\n\n\n# =========================================\n\n\n\n\ndef cast_musicgen_dtype(mg, dtype):\n \"\"\"把 MusicGen 内所有子模块转成半精度 / 全精度\"\"\"\n mg.lm = mg.lm.to(dtype=dtype)\n if hasattr(mg, \"decoder\"):\n mg.decoder = mg.decoder.to(dtype=dtype)\n if hasattr(mg, \"condition_provider\"):\n mg.condition_provider = mg.condition_provider.to(dtype=dtype)\n return mg\n\n\nclass MusicGenService:\n def __init__(\n self,\n api_endpoint: str,\n openai_api_key: str,\n openai_model: str = OPENAI_MODEL,\n verbose: bool = True,\n device: Optional[str] = None,\n checkpoint_path: Optional[str] = None,\n use_fp16: bool = True,\n ):\n # 参数配置\n self.api_endpoint = api_endpoint\n self.openai_api_key = openai_api_key\n self.openai_model = openai_model\n self.verbose = verbose\n self.device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.use_fp16 = use_fp16\n self.clap = CLAP_Module(enable_fusion=False, device=self.device).eval()\n self.SR = 32000\n self.mel_transform = torchaudio.transforms.MelSpectrogram(\n sample_rate=self.SR, n_mels=80, hop_length=512\n ).to(self.device)\n self.checkpoint_path = checkpoint_path\n\n\n # 1) 加载 small 版本\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}...\")\n self.model = MusicGen.get_pretrained(\"facebook/musicgen-small\", self.device)\n\n # 设置参数\n self.model.set_generation_params(\n duration=30,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n\n # 2) 如果指定了 checkpoint，就加载权重\n if checkpoint_path:\n if self.verbose:\n print(f\"[Init] Loading checkpoint from {checkpoint_path}…\")\n ckpt = torch.load(checkpoint_path, map_location=\"cpu\")\n state_dict = ckpt.get(\"model_state_dict\", ckpt)\n missing, unexpected = self.model.lm.load_state_dict(state_dict, strict=True)\n if self.verbose:\n print(f\"[Init] Checkpoint loaded. missing keys: {len(missing)}, unexpected: {len(unexpected)}\")\n\n # 3) 半精度转换 & 编译\n self.model = cast_musicgen_dtype(self.model, DTYPE)\n print(\"Model dtype:\", next(self.model.lm.parameters()).dtype)\n\n # 编译 lm 子模块以减少 Python 调度开销\n try:\n self.model.lm = torch.compile(self.model.lm,mode=\"reduce-overhead\",fullgraph=True)\n if self.verbose:\n print(\"[Init] 🚀 torch.compile() 已启用\")\n except Exception as e:\n if self.verbose:\n print(f\"[Init] ⚠️ torch.compile 失败，回退: {e}\")\n\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}... OK\")\n\n\n @torch.no_grad()\n def embed_cpu(self, wav32: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 将输入的多通道高采样率音频数据转换为单通道 16kHz，\n 并用 CLAP 模型提取其在 CPU 上的特征嵌入表示，\n 用于后续相似度计算或音频理解任务。\n \"\"\"\n mono = wav32.to(\"cpu\", torch.float32).mean(0, keepdim=True)\n m16 = torchaudio.functional.resample(mono, self.SR, 16000)\n return self.clap.get_audio_embedding_from_data(m16, use_tensor=True).cpu()\n\n \n def request_segment_plan(\n self, style: str, user_prompt: str, total_sec: int, window_sec: int\n ) -> List[Dict]:\n \"\"\"\n 根据用户提供的风格、提示词和时长设置，调用本地或远程 ChatGPT 接口生成音乐分段规划。\n\n 该方法会构造一个详细的自然语言提示（prompt），并附带 CLAP 支持的 token 列表，提交给语言模型，\n 请求生成一个包含多个段落的 JSON 列表。每个段落包含音乐结构（如 intro、verse 等）、持续时间、\n 生成提示词以及采样参数（temperature、top_p、top_k）等信息。返回结果会自动补齐字段并计算结束时间戳。\n\n 如果调用失败，方法将退回使用内置的默认分段方案。\n\n 参数:\n style (str): 音乐结构风格名，对应 STYLE_STRUCTURE_LIB 中的 key。\n user_prompt (str): 用户自定义的风格描述文本，将传入 LLM 构造提示。\n total_sec (int): 希望生成的总时长（秒），提示中允许略超，但不得不足。\n window_sec (int): 每个段落的时间粒度，必须是段落时长的倍数。\n\n 返回:\n List[Dict]: 每个元素为一个分段字典，包含字段：\n - section: 段名称\n - duration: 时长（秒）\n - prompt: 用于音乐生成的提示词\n - temperature, top_p, top_k: 控制采样的超参数\n - end_at: 当前段结束时刻（累加）\n\n 异常:\n ValueError: 若 style 无法匹配结构库、或响应无法被正确解析时抛出。\n\n 注意:\n - 默认使用远程 OpenAI API，也支持通过 `use_local_llm` 使用 Ollama 模型。\n - 方法内部自动进行重试机制，并打印失败原因。\n - 所有生成内容中的 prompt 字段将仅使用提供的 token 列表，保证下游模型兼容性。\n \"\"\"\n structure = STYLE_STRUCTURE_LIB.get(style.lower())\n if not structure:\n raise ValueError(f\"风格 '{style}' 不支持\")\n\n fluent_prompt = f\"\"\"You are an experienced music producer. \n Please generate a JSON array of song segments where each item has the following keys: \n - section (string) \n - duration (int, seconds) \n - prompt (string) \n - temperature (float) \n - top_p (float) \n - top_k (float) \n \n Return only the raw JSON array without any markdown fences.\n \n The total duration should be about {total_sec} seconds (you may exceed but not fall short). \n Use these sections in order: {structure}. \n The style is: {user_prompt}. \n Each segment’s duration must be a multiple of {window_sec} seconds (e.g. if window_sec = 30, valid values are 30, 60, 90, …). \n top_p must be between 0.6 and 0.9, top_k should be about 250, temperature should be higher than 0.9.\n \n When constructing the prompt field inside each segment, use ONLY the following tokens, organized by category:\n \n Structure: \n intro-like, verse-like, pre-chorus-like, chorus-like, bridge-like, build-up-like, drop-like, interlude-like, outro-like.\n \n Mood / Emotion: \n calming, soothing, serene, ethereal, dreamy, happy, joyful, cheerful, gentle, groovy, funky, sad, melancholic, nostalgic, mournful, peaceful, uplifting, hopeful, inspirational, epic, triumphant, energetic, aggressive, angry, dark, mysterious, spooky, tense, suspenseful, anxious, majestic, glorious, romantic, playful, whimsical.\n \n Genre / Style: \n ambient, dark ambient, ambient techno, new age, chillout, lofi hip hop, trip hop, future bass, synthwave, vaporwave, house, deep house, progressive house, tech house, techno, psytrance, trance, drum and bass, jungle, dubstep, brostep, trap, EDM, jazz, smooth jazz, swing, bebop, latin jazz, bossa nova, samba, fusion, classical, baroque, romantic, modern classical, contemporary classical, orchestral, opera, chamber music, rock, classic rock, hard rock, punk rock, post-punk, indie rock, alternative rock, progressive rock, shoegaze, grunge, metal, heavy metal, death metal, black metal, thrash metal, pop, synth pop, indie pop, electropop, k-pop, j-pop, R&B, neo soul, soul, funk, disco, gospel, hip hop, rap, boom bap, trap rap, blues, delta blues, reggae, dub, ska, folk, bluegrass, country, americana, celtic, afrobeat, salsa, tango, flamenco, balkan, bollywood, world music, soundtrack, cinematic, video game, experimental, noise.\n \n Instrument / Voice: \n acoustic guitar, electric guitar, classical guitar, bass guitar, electric bass, acoustic bass, ukulele, banjo, mandolin, piano, electric piano, upright piano, harpsichord, organ, pipe organ, clavinet, synthesizer, bass synth, pad, lead synth, violin, viola, cello, double bass, string ensemble, orchestral strings, flute, piccolo, recorder, pan flute, clarinet, bass clarinet, oboe, english horn, bassoon, contrabassoon, saxophone, soprano saxophone, alto saxophone, tenor saxophone, baritone saxophone, trumpet, piccolo trumpet, cornet, flugelhorn, trombone, bass trombone, tuba, french horn, euphonium, harmonica, accordion, concertina, bagpipe, sitar, shamisen, koto, erhu, pipa, dizi, ocarina, didgeridoo, marimba, xylophone, vibraphone, glockenspiel, celesta, steel drum, drums, acoustic drum kit, electronic drum kit, snare drum, bass drum, cymbals, hi-hat, tom-tom, timpani, congas, bongo, djembe, tabla, triangle, tambourine, shaker, clap, percussion, male vocals, female vocals, mixed choir, children choir, solo voice.\n \n Environment: \n rain, heavy rain, thunder, wind, snowstorm, ocean waves, seashore, underwater ambience, forest birds, rainforest, desert wind, city daytime, city night, traffic noise, street ambience, airport terminal, subway station, train station, bus station, marketplace, stadium crowd, children playground, office background, classroom, library, crowd murmur, cafe background, industrial, construction noise, fireplace crackle, fireworks, police siren, space ambience, windchimes.\n \n Production Style: \n lo-fi, hi-fi, vintage, analog, digital, electronic, acoustic, orchestral, cinematic, soundtrack, psychedelic, experimental, noise, glitch, ambient, warm, bright, dark, harsh, smooth, compressed, bassy, spatial, surround, stereo wide, monophonic, polyphonic, minimal, maximalist, bitcrushed, chiptune, distorted, reverb-heavy, gated reverb, side-chained, auto-tuned.\n \n Energy / Tempo: \n high energy, powerful, relaxed, calm, dynamic, steady, variable, driving, lively, euphoric, hypnotic, pulsing, accelerando, decelerando.\n \n Chord / Harmony: \n C major, C minor, C7, Cmaj7, Cmin7, A major, A minor, A7, Amaj7, Amin7, Bb major, Bb minor, Bb7, Bbmaj7, B major, B minor, B7, Bmaj7, Bmin7, D major, D minor, D7, Dmaj7, Dmin7, E major, E minor, E7, Emaj7, Emin7, F major, F minor, F7, Fmaj7, Fmin7, F# major, F# minor, F#7, F#maj7, F#min7, G major, G minor, G7, Gmaj7, Gmin7, Ab major, Ab minor, Ab7, Abmaj7, Abmin7.\n\n CLAP generate Harmony:\n C, C#, D, D#, E, F, F#, G, G#, A, A#, B\n \n Example of two valid prompt strings for one segment: \n build-up-like, serene, delta blues, acoustic guitar, windchimes, soundtrack, accelerando, C minor, G \n interlude-like, suspenseful, neo soul, orchestral strings, space ambience, soundtrack, dynamic, C minor, G#.\n \n Return only the JSON array.\n \"\"\"\n\n \n raw = None\n for attempt in range(1, max_retries+1):\n try:\n if use_local_llm:\n # ➡️ 调 Ollama Chat —— \n resp = ollama.chat(\n model=\"deepseek-r1:14b\",\n messages=[{\"role\": \"user\", \"content\": fluent_prompt}]\n )\n \n # 直接从响应字典中提取文本\n try:\n text = resp[\"message\"][\"content\"]\n except Exception as e:\n raise ValueError(f\"⚠️ 无法解析 Ollama 返回结构: {e}\\n原始内容:\\n{resp}\")\n \n # 打印内容供 debug\n if self.verbose:\n print(\"the text is : \" + text)\n \n # 尝试提取 JSON 数组（包括 ```json ... ``` 或直接 [...]）\n m = re.search(r\"```json\\s*($begin:math:display$.*?$end:math:display$)\\s*```\", text, re.S)\n if not m:\n m = re.search(r\"\\[.*?\\]\", text, re.S)\n \n if not m:\n raise ValueError(f\"❌ 无法从文本中提取 JSON 数组:\\n{text}\")\n \n # 提取原始 JSON 内容\n raw = m.group(1) if m.lastindex else m.group(0)\n raw = raw.encode('utf-8').decode('unicode_escape')\n raw = raw.replace('\\\\\"', '\"')\n\n else:\n # ➡️ 调 LLM API\n payload = {\n \"model\": self.openai_model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": fluent_prompt}\n ]\n }\n headers = {\n \"Authorization\": f\"Bearer {self.openai_api_key}\",\n \"Content-Type\": \"application/json\"\n }\n\n url = API_ENDPOINT\n \n resp = requests.post(\n url, \n headers=headers, \n json=payload\n )\n\n resp.raise_for_status()\n raw = resp.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n \n \n if self.verbose:\n print(\"☀️ the raw json is : \"+raw)\n print(f\"[SegPlan] 成功获得规划 (attempt {attempt})\")\n break\n \n # 异常处理与多次重试 \n except requests.RequestException as e:\n if attempt < max_retries:\n wait = min(2 ** (attempt-1) , 60)\n if self.verbose:\n print(f\"[SegPlan] 请求失败，第{attempt}次重试，{wait}s后再试… ({e})\")\n time.sleep(wait)\n else:\n if self.verbose:\n print(f\"[SegPlan] 重试{max_retries}次失败，使用默认分段（❌没有完成，寄）\")\n # 默认回退\n if not raw:\n # 请求失败，使用固定默认分段列表，并继续后续解析流程\n if self.verbose:\n print(\"[SegPlan] 请求多次失败，使用固定默认分段列表\")\n default_segs = [\n {'section': 'minimal-pulse', 'duration': 30, 'prompt': 'intro-like, peaceful, ambient, bass synth, pad, warm, reverb-heavy, pulsing, Cmaj7, C', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'arpeggio-loop', 'duration': 30, 'prompt': 'verse-like, calming, ambient, synthesizer, pad, warm, smooth, slow, Amin7, A', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 8.0},\n {'section': 'white-noise-bed', 'duration': 30, 'prompt': 'pre-chorus-like, soothing, ambient, pad, space ambience, warm, reverb-heavy, slow, Dmin7, D', 'temperature': 0.65, 'top_p': 0.75, 'cfg': 6.5},\n {'section': 'modulation-bridge', 'duration': 30, 'prompt': 'bridge-like, ethereal, ambient, string ensemble, pad, warm, reverb-heavy, slow, Gmaj7, G', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 7.5},\n {'section': 'repetitive-theme', 'duration': 30, 'prompt': 'chorus-like, dreamy, ambient, string ensemble, pad, warm, smooth, slow, Fmaj7, F', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'soft-ending', 'duration': 30, 'prompt': 'outro-like, serene, ambient, pad, string ensemble, warm, smooth, decelerando, Cmaj7, C', 'temperature': 0.5, 'top_p': 0.6, 'cfg': 6.0},\n ]\n # 用默认列表生成 raw JSON，继续解析流程\n raw = json.dumps(default_segs, ensure_ascii=False)\n \n # —— 解析并补齐字段 —— 如果 LLM 给的内容有缺损，使用默认的参数补齐它\n segs = json.loads(raw)\n acc = 0\n for s in segs:\n s[\"duration\"] = int(s.get(\"duration\", window_sec))\n s[\"prompt\"] = s.get(\"prompt\", user_prompt)\n s[\"temperature\"] = float(s.get(\"temperature\", 0.7))\n s[\"top_p\"] = float(s.get(\"top_p\", 0.9))\n s[\"top_k\"] = float(s.get(\"top_k\", 250))\n acc += s[\"duration\"]\n s[\"end_at\"] = acc\n if self.verbose:\n print(\"[SegPlan] Parsed segments:\")\n for x in segs: print(x)\n\n return segs\n\n def compute_si_sdr(self, ref: torch.Tensor, est: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算 Scale-Invariant Signal-to-Distortion Ratio（SI-SDR），用于评估估计音频与参考音频之间的相似度。\n\n SI-SDR 是一种用于盲音源分离和音频质量评价的指标，具备尺度不变性，\n 可以排除音量（能量）缩放对相似度评分的影响。\n\n 参数:\n ref (torch.Tensor): 参考音频信号，形状为 [C, T] 或 [T]（C 为通道数，T 为时间长度）。\n est (torch.Tensor): 估计音频信号，形状与 ref 相同或兼容。\n eps (float): 避免除零错误的微小常数，默认值为 1e-8。\n\n 返回:\n torch.Tensor: 每个样本/通道的 SI-SDR 得分（以 dB 为单位），形状为 [C] 或标量。\n \"\"\"\n # 对齐 ref 和 est 的时间长度，支持单通道或多通道输入\n if ref.dim() == 2 and est.dim() == 2:\n T = min(ref.shape[1], est.shape[1])\n ref, est = ref[:, :T], est[:, :T]\n else:\n T = min(ref.shape[-1], est.shape[-1])\n ref, est = ref[:T], est[:T]\n\n # 去直流分量（零均值化）\n ref_zm = ref - ref.mean(-1, keepdim=True)\n est_zm = est - est.mean(-1, keepdim=True)\n\n # 计算参考信号能量\n ref_energy = torch.sum(ref_zm ** 2, dim=-1, keepdim=True)\n\n # 计算最佳缩放因子 scale，使得投影后的信号最接近估计信号\n scale = torch.sum(ref_zm * est_zm, dim=-1, keepdim=True) / (ref_energy + eps)\n\n # 将估计信号投影到参考信号方向\n proj = scale * ref_zm\n\n # 残差噪声 = 去中心化估计信号 - 投影部分\n noise = est_zm - proj\n\n # 计算信噪比（使用投影部分作为信号）\n snr = torch.sum(proj ** 2, dim=-1) / (torch.sum(noise ** 2, dim=-1) + eps)\n\n # 返回以 dB 为单位的 SI-SDR 分数\n return 10 * torch.log10(snr + eps)\n\n\n\n def mel_distance(self, x: torch.Tensor, y: torch.Tensor) -> float:\n \"\"\"\n 计算两个音频信号的梅尔谱（Mel Spectrogram）之间的 L2 距离，用于衡量音频在感知上的相似性。\n\n 此方法首先对两个音频信号做对齐裁剪，并通过 `self.mel_transform` 提取梅尔谱特征，\n 再计算其展开后的欧氏距离（L2 norm）。适用于感知相似度评估、音频生成对比等任务。\n\n 参数:\n x (torch.Tensor): 第一个音频信号，形状为 [T] 或 [C, T]。\n y (torch.Tensor): 第二个音频信号，形状与 x 相同或兼容。\n\n 返回:\n float: 梅尔谱之间的 L2 距离，数值越小表示音频越相似。\n \"\"\"\n # 对齐两个音频在时间维度上的长度\n if x.dim()==2 and y.dim()==2:\n T = min(x.shape[1], y.shape[1])\n x, y = x[:,:T], y[:,:T]\n else:\n T = min(x.shape[-1], y.shape[-1])\n x, y = x[:T], y[:T]\n\n # 通过 mel_transform 提取梅尔谱特征（添加 batch 维度）\n Mx = self.mel_transform(x.unsqueeze(0)) # [1, F, T']\n My = self.mel_transform(y.unsqueeze(0)) # [1, F, T']\n\n # 将梅尔谱差值展平后计算 L2 距离\n return torch.norm((Mx-My).reshape(-1), p=2).item()\n\n def xfade(self, a: torch.Tensor, b: torch.Tensor, ov: int) -> torch.Tensor:\n \"\"\"\n 将两段音频 a 和 b 在尾部和头部以交叉渐变方式进行拼接，实现平滑过渡。\n 自动对齐声道：如果 a 是多声道但 b 单声道，自动复制 b 的声道数。\n \"\"\"\n print(\"xfade input shapes:\", a.shape, b.shape)\n # 1) 自动对齐声道\n if a.dim() == 2 and b.dim() == 2 and a.shape[0] != b.shape[0]:\n # 如果 b 只有 1 个通道，但 a 有多个通道，就复制 b\n if b.shape[0] == 1:\n b = b.repeat(a.shape[0], 1)\n # 反之亦然（极少用到）\n elif a.shape[0] == 1:\n a = a.repeat(b.shape[0], 1)\n\n # 2) 计算 overlap 点数\n n = ov * self.SR\n\n # 3) 构造渐变权重\n w = torch.linspace(0, 1, n, device=a.device).sqrt()\n\n # 4) 交叉区混合\n left = a[..., :-n]\n middle = a[..., -n:] * (1 - w) + b[..., :n] * w\n right = b[..., n:]\n\n # 5) 拼接并返回\n return torch.cat([left, middle, right], dim=-1)\n\n # ====================== generate ======================\n @torch.no_grad()\n def generate(\n self,\n seed_path: str,\n style: str,\n user_prompt: str,\n total_sec: int = 120,\n window_sec: int = 20,\n overlap_sec: int = 5,\n prompt_sec: int = 5,\n safety_tail: int = 3,\n n_tracks: int = 1,\n clap_candidates: int = 3,\n sim_thresh: float = 0.5,\n sil_thresh: float = -40,\n melody_path: Optional[str] = None,\n out_path: Optional[str] = None,\n use_fp16: bool = False,\n ) -> Union[str, List[str]]:\n \"\"\"\n 基于种子音频和用户提示，分段、生成并拼接完整音乐轨道。\n\n 整体流程：\n 1. 读取 seed_path 指定的音频文件并重采样、提取 CLAP 嵌入；\n 2. （可选）读取 melody_path 指定的条件音频；\n 3. 调用 request_segment_plan 获取分段规划；\n 4. 对每个段落并行生成 clap_candidates 条候选，并按相似度选出最佳；\n 5. 通过 xfade 平滑拼接到当前音轨；\n 6. 重复直到累计生成达到 total_sec，并保存最终音轨。\n\n 参数:\n seed_path (str):\n 初始音频文件路径，用于续写输入。\n style (str):\n 分段风格名称，对应内部 STYLE_STRUCTURE_LIB 的键。\n user_prompt (str):\n 用户文本提示，用于精细化每段的生成风格。\n total_sec (int, 可选):\n 目标总时长（秒）。默认 120。\n window_sec (int, 可选):\n 每次生成的片段时长（秒）。默认 20。\n overlap_sec (int, 可选):\n 段落重叠长度（秒），用于淡入淡出拼接。默认 5。\n prompt_sec (int, 可选):\n 每次构建生成 prompt 时要截取的尾部时长（秒）。默认 5。\n safety_tail (int, 可选):\n 生成 prompt 前保留的不参与提示的尾巴长度（秒），避免拼接突变。默认 3。\n n_tracks (int, 可选):\n 并行生成的轨道数量；>1 时返回路径列表。默认 1。\n clap_candidates (int, 可选):\n 每段并行生成的候选数，用于多版本筛选。默认 3。\n sim_thresh (float, 可选):\n 最优候选的相似度阈值；低于则触发重试或跳过。默认 0.5。\n sil_thresh (float, 可选):\n 静音检测阈值（dB）；可用于剔除无声片段。默认 -40。\n melody_path (Optional[str], 可选):\n 条件 melody 音频文件路径；不提供则进入纯生成模式。默认 None。\n out_path (Optional[str], 可选):\n 最终输出文件路径；多轨时自动编号。默认 None。\n use_fp16 (bool, 可选):\n 在 CUDA 上是否启用半精度推理（torch.cuda.amp）。默认 False.\n\n 返回:\n Union[str, List[str]]:\n 生成的音频文件路径。若 n_tracks>1 则为路径列表，否则为单个字符串。\n\n 异常:\n ValueError:\n - 当分段规划（request_segment_plan）失败且无法解析时。\n - 当读取音频文件或网络请求出错并超过重试次数时。\n\n 示例:\n >>> out = svc.generate(\n ... seed_path=\"seed.wav\",\n ... style=\"ambient\",\n ... user_prompt=\"dreamy pad with rain\",\n ... total_sec=180,\n ... n_tracks=2,\n ... clap_candidates=5,\n ... use_fp16=True,\n ... out_path=\"output/track.wav\"\n ... )\n \"\"\"\n\n print(f\"[Generate] start: n_tracks={n_tracks}, clap_candidates={clap_candidates}, use_fp16={use_fp16}, device={self.device}\", flush=True)\n\n fp16_ctx = torch.cuda.amp.autocast if (use_fp16 and self.device.startswith(\"cuda\")) else nullcontext\n\n def _run_once(idx: int) -> str:\n checkpoint_path= self.checkpoint_path\n USE_BF16 = torch.cuda.is_bf16_supported()\n DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n ENABLE_COMPILE = True # False 可快速回退\n STRICT_LOAD = False\n\n print(f\"[Track {idx+1}/{n_tracks}] >>> begin _run_once\", flush=True)\n torch.manual_seed(42 + idx)\n\n # 1) 载入 seed 音频\n seed32, sr = torchaudio.load(seed_path)\n if sr != self.SR:\n seed32 = torchaudio.functional.resample(seed32, sr, self.SR)\n seed32 = seed32.to(self.device)\n ref_emb = self.embed_cpu(seed32)\n\n # 2) 载入可选的 melody 条件\n melody_tensor = None\n if melody_path:\n m32, mr = torchaudio.load(melody_path)\n if mr != self.SR:\n m32 = torchaudio.functional.resample(m32, mr, self.SR)\n melody_tensor = m32.to(self.device).unsqueeze(0)\n\n # 3) 段落规划\n segs = self.request_segment_plan(style, user_prompt, total_sec, window_sec)\n need = (window_sec - overlap_sec) * self.SR\n full = seed32[..., :need] if seed32.shape[-1] >= need else seed32.repeat(\n 1, math.ceil(need / seed32.shape[-1])\n )[..., :need]\n cur_seg = 0\n\n # 4) 每段生成参数设置\n def set_params(idx_seg):\n p = segs[idx_seg]\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,#p[\"temperature\"],\n top_p=0.9,#p[\"top_p\"],\n top_k=250,#p[\"cfg\"],\n #use_sampling=True,\n )\n\n set_params(cur_seg)\n retry_count = 0\n max_retries = 5\n\n # 5) 主循环：续写直到达到 total_sec\n while full.shape[-1] / self.SR < total_sec:\n # 切换到下一个段落\n if full.shape[-1] / self.SR >= segs[cur_seg][\"end_at\"] and cur_seg < len(segs) - 1:\n cur_seg += 1\n set_params(cur_seg)\n retry_count = 0\n if self.verbose:\n print(f\"▶ Enter segment {cur_seg}: {segs[cur_seg]['section']}\")\n \n\n # 准备 prompt 音频片段\n tail = full.shape[-1] - safety_tail * self.SR\n head = max(0, tail - prompt_sec * self.SR)\n pa = full[..., head:tail]\n if pa.shape[-1] < 1024:\n pa = torch.nn.functional.pad(pa, (1024 - pa.shape[-1], 0), \"reflect\")\n pa = pa.unsqueeze(0)\n\n # 并行多候选生成\n current_prompt = segs[cur_seg][\"prompt\"]\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] generating {clap_candidates} candidates\", flush=True)\n descriptions = [current_prompt] * clap_candidates\n \n\n with fp16_ctx():\n if melody_tensor is not None:\n if self.verbose:\n print(\"➡️ now in the melody mode\")\n wavs = self.model.generate(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n melody=melody_tensor,\n duration=window_sec,\n progress=self.verbose,\n )\n else:\n # if self.verbose:\n # print(\"➡️ now in the gen mode\")\n # torch.manual_seed(42) # 固定seed\n # wavs = self.model.generate(\n # descriptions,\n # progress=self.verbose,\n # )\n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # —— 使用 generate_continuation 接口 —— \n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # 续写：使用之前 set_generation_params 设置的 duration\n wavs = self.model.generate_continuation(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n progress=self.verbose,\n )\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] got {len(wavs)} wavs\", flush=True)\n\n # 6) 选出相似度最高的 wav，同时收集所有 sim 值\n sim_list = []\n best, best_sim = None, -1.0\n\n for i, wav in enumerate(wavs):\n if wav.shape[0] == 1:\n wav = wav.repeat(2, 1)\n\n # 计算三种相似度\n sim_clap = torch.cosine_similarity(self.embed_cpu(wav), ref_emb).item()\n sim_sdr = (self.compute_si_sdr(seed32, wav).mean().item() + 50) / 50.0\n sim_mel = 1.0 / (1 + self.mel_distance(seed32, wav))\n sim_final = 0.5 * sim_clap + 0.3 * sim_sdr + 0.2 * sim_mel\n\n sim_list.append(sim_final) # 收集到列表\n if sim_final > best_sim:\n best, best_sim = wav.detach().cpu(), sim_final\n del wav\n\n torch.cuda.empty_cache()\n\n # 打印这一段所有候选的 sim 数组和选出的 best_sim\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] sims = {sim_list}\")\n print(f\"[Track {idx+1} Segment {cur_seg}] best_sim = {best_sim:.4f}\")\n\n save_best_each = True\n # 立刻保存本段 best 候选 —— \n if save_best_each and out_path:\n # out_path 父目录下创建 bestwav 文件夹\n best_dir = pathlib.Path(out_path).parent / \"bestwav\"\n best_dir.mkdir(parents=True, exist_ok=True)\n save_name = best_dir / f\"Segment_{cur_seg}.wav\"\n\n # 生成子文件名，并写文件\n # 生成微调结果\n # 把 float16 转成 float32，再保存\n \n audio_ft = best.cpu().float().numpy()\n if audio_ft.ndim == 2 and audio_ft.shape[0] < audio_ft.shape[1]:\n audio_ft = audio_ft.T\n sf.write(str(save_name), audio_ft, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1} Seg {cur_seg}] saved best wav → {save_name}\")\n\n # 7) 相似度太低时重试或跳过\n if best_sim < sim_thresh:\n retry_count += 1\n if retry_count > max_retries:\n if self.verbose:\n print(f\"Segment {cur_seg} reached max retries, skipping segment\")\n retry_count = 0\n break\n p = segs[cur_seg]\n new_temp = max(0.15, p[\"temperature\"] * (0.9 ** retry_count))\n new_top_k = p[\"top_k\"] * (1.15 ** retry_count)\n new_top_p = min(p[\"top_p\"]+0.05,0.95)\n self.model.set_generation_params(\n duration=window_sec,\n temperature=new_temp,\n top_k=new_top_k,\n top_p = new_top_p,\n use_sampling=True,\n )\n if self.verbose:\n print(f\"Retry {retry_count}: temp→{new_temp:.2f}, topk→{new_top_k:.2f}\")\n continue\n\n # 8) 拼接并输出调试信息\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] before xfade: {full.shape[-1]/self.SR:.1f}s\")\n full = self.xfade(full, best.to(self.device), overlap_sec)\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] after xfade: {full.shape[-1]/self.SR:.1f}s\")\n del best\n torch.cuda.empty_cache()\n retry_count = 0\n\n # 完成 single-track 生成，导出文件\n final = full[..., : total_sec * self.SR].cpu().T\n if out_path:\n stem, ext = os.path.splitext(out_path)\n out_file = f\"{stem}_{idx+1:02d}{ext}\" if n_tracks > 1 else out_path\n else:\n out_file = f\"final_{idx+1:02d}.wav\" if n_tracks > 1 else \"final.wav\"\n\n pathlib.Path(out_file).parent.mkdir(parents=True, exist_ok=True)\n sf.write(out_file, final, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1}] saved {out_file}, best_sim={best_sim:.4f}\", flush=True)\n return out_file\n\n # 并行生成多条 track\n print(f\"[Generate] launching _run_once for {n_tracks} tracks\", flush=True)\n paths = [_run_once(i) for i in range(n_tracks)]\n print(f\"[Generate] all done, paths = {paths}\", flush=True)\n return paths if n_tracks > 1 else paths[0]\n\n \n\n\ndef generate_music_api(params: Dict) -> Dict:\n svc = MusicGenService(\n api_endpoint=params.get('api_endpoint', API_ENDPOINT),\n openai_api_key=params.get('api_key', OPENAI_API_KEY),\n openai_model=params.get('model', OPENAI_MODEL),\n verbose=params.get('verbose', True),\n checkpoint_path=params.get('checkpoint_path'), # ← 把它传进去\n )\n out = svc.generate(\n seed_path=params['seed_path'],\n style=params['style'],\n user_prompt=params['user_prompt'],\n total_sec=params.get('total_sec', 120),\n window_sec=params.get('window_sec', 20),\n overlap_sec=params.get('overlap_sec', 5),\n prompt_sec=params.get('prompt_sec', 5),\n safety_tail=params.get('safety_tail', 3),\n clap_candidates=params.get('clap_candidates', 3),\n n_tracks=params.get('n_tracks', 1), # ← 新增这一行\n use_fp16=params.get('use_fp16', False), # ← 以及这一行\n sim_thresh=params.get('sim_thresh', 0.5),\n sil_thresh=params.get('sil_thresh', -40),\n melody_path=params.get('melody_path'),\n out_path=params.get('out_path'),\n )\n return {'out_path': out}\n\n"}}, "lm_train/musicgen_api.py": {"486bab9207d6499336bfe192c2ed40eb91b6c4b0": {"t": 1785768920.6716874, "n": 35761, "h": "dbc4519aca334609", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nmusicgen_api.py\n\n封装 MusicGen 无限续写脚本为后端可调用接口。\n支持 Melody 条件生成、增量续写、重试与显存管理。\n\"\"\"\n\nnetworks = False # 如果没有接入国际互联网，就使用 False\n\n\nimport os\n# 如果没有接入国际互联网，就将 Huggingface 的访问设置为【不访问】\nif not networks:\n # 确保 transformers/hub 都用离线模式\n os.environ[\"HF_HUB_OFFLINE\"] = \"1\"\n os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n # 然后 monkey-patch BertTokenizer.from_pretrained，让它只走本地缓存\n from transformers import BertTokenizer\n _original_bp = BertTokenizer.from_pretrained\n def _from_pretrained_local(repo_id, *args, **kwargs):\n # 强制 local_files_only=True\n return _original_bp(repo_id, local_files_only=True, *args, **kwargs)\n BertTokenizer.from_pretrained = _from_pretrained_local\n\n\nimport json\nimport math\nimport warnings\nimport re\nimport time\nimport ollama\nimport torchaudio\nimport torch\nimport requests\nimport pathlib\nimport soundfile as sf\n\nfrom contextlib import nullcontext\nfrom typing import List, Optional, Union, Dict\nfrom urllib.parse import urljoin\nfrom audiocraft.models import MusicGen\nfrom laion_clap import CLAP_Module\nfrom music_structure import STYLE_STRUCTURE_LIB\n\n\n# ========== 默认配置==========\n\nuse_local_llm = False # 是否使用本地的LLM （需要打开 OLLAMA）\n\nOPENAI_MODEL = 'gpt-4o-mini' # 如果使用网络 LLM API， 这是使用的模型\nmax_retries = 6 # 如果使用网络 LLM API，这是最大重试次数\n\nif use_local_llm :\n API_ENDPOINT = \"http://localhost:11434/api/generate\" #本地LLM访问地址\nelse:\n API_ENDPOINT = 'https://www.dmxapi.com/v1/chat/completions' # 网络LLM访问地址\n\nOPENAI_API_KEY = 'sk-vWyNj5NxixWjJcjknaxNivrtIkHIjXDBVlk1b6wRW2pZc0hz' # 网络LLM API KEY\n\n\nUSE_BF16 = torch.cuda.is_bf16_supported() # \nDTYPE = torch.bfloat16 if USE_BF16 else torch.float16 \n\nENABLE_COMPILE = True # False 可快速回退\nSTRICT_LOAD = False\n\n\n# =========================================\n\n\n\n\ndef cast_musicgen_dtype(mg, dtype):\n \"\"\"把 MusicGen 内所有子模块转成半精度 / 全精度\"\"\"\n mg.lm = mg.lm.to(dtype=dtype)\n if hasattr(mg, \"decoder\"):\n mg.decoder = mg.decoder.to(dtype=dtype)\n if hasattr(mg, \"condition_provider\"):\n mg.condition_provider = mg.condition_provider.to(dtype=dtype)\n return mg\n\n\nclass MusicGenService:\n def __init__(\n self,\n api_endpoint: str,\n openai_api_key: str,\n openai_model: str = OPENAI_MODEL,\n verbose: bool = True,\n device: Optional[str] = None,\n checkpoint_path: Optional[str] = None,\n use_fp16: bool = True,\n ):\n # 参数配置\n self.api_endpoint = api_endpoint\n self.openai_api_key = openai_api_key\n self.openai_model = openai_model\n self.verbose = verbose\n self.device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n self.use_fp16 = use_fp16\n self.clap = CLAP_Module(enable_fusion=False, device=self.device).eval()\n self.SR = 32000\n self.mel_transform = torchaudio.transforms.MelSpectrogram(\n sample_rate=self.SR, n_mels=80, hop_length=512\n ).to(self.device)\n self.checkpoint_path = checkpoint_path\n\n\n # 1) 加载 small 版本\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}...\")\n self.model = MusicGen.get_pretrained(\"facebook/musicgen-small\", self.device)\n\n # 设置参数\n self.model.set_generation_params(\n duration=30,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n\n # 2) 如果指定了 checkpoint，就加载权重\n if checkpoint_path:\n if self.verbose:\n print(f\"[Init] Loading checkpoint from {checkpoint_path}…\")\n ckpt = torch.load(checkpoint_path, map_location=\"cpu\")\n state_dict = ckpt.get(\"model_state_dict\", ckpt)\n missing, unexpected = self.model.lm.load_state_dict(state_dict, strict=True)\n if self.verbose:\n print(f\"[Init] Checkpoint loaded. missing keys: {len(missing)}, unexpected: {len(unexpected)}\")\n\n # 3) 半精度转换 & 编译\n self.model = cast_musicgen_dtype(self.model, DTYPE)\n print(\"Model dtype:\", next(self.model.lm.parameters()).dtype)\n\n # 编译 lm 子模块以减少 Python 调度开销\n try:\n self.model.lm = torch.compile(self.model.lm,mode=\"reduce-overhead\",fullgraph=True)\n if self.verbose:\n print(\"[Init] 🚀 torch.compile() 已启用\")\n except Exception as e:\n if self.verbose:\n print(f\"[Init] ⚠️ torch.compile 失败，回退: {e}\")\n\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}... OK\")\n\n\n @torch.no_grad()\n def embed_cpu(self, wav32: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 将输入的多通道高采样率音频数据转换为单通道 16kHz，\n 并用 CLAP 模型提取其在 CPU 上的特征嵌入表示，\n 用于后续相似度计算或音频理解任务。\n \"\"\"\n mono = wav32.to(\"cpu\", torch.float32).mean(0, keepdim=True)\n m16 = torchaudio.functional.resample(mono, self.SR, 16000)\n return self.clap.get_audio_embedding_from_data(m16, use_tensor=True).cpu()\n\n \n def request_segment_plan(\n self, style: str, user_prompt: str, total_sec: int, window_sec: int\n ) -> List[Dict]:\n \"\"\"\n 根据用户提供的风格、提示词和时长设置，调用本地或远程 ChatGPT 接口生成音乐分段规划。\n\n 该方法会构造一个详细的自然语言提示（prompt），并附带 CLAP 支持的 token 列表，提交给语言模型，\n 请求生成一个包含多个段落的 JSON 列表。每个段落包含音乐结构（如 intro、verse 等）、持续时间、\n 生成提示词以及采样参数（temperature、top_p、top_k）等信息。返回结果会自动补齐字段并计算结束时间戳。\n\n 如果调用失败，方法将退回使用内置的默认分段方案。\n\n 参数:\n style (str): 音乐结构风格名，对应 STYLE_STRUCTURE_LIB 中的 key。\n user_prompt (str): 用户自定义的风格描述文本，将传入 LLM 构造提示。\n total_sec (int): 希望生成的总时长（秒），提示中允许略超，但不得不足。\n window_sec (int): 每个段落的时间粒度，必须是段落时长的倍数。\n\n 返回:\n List[Dict]: 每个元素为一个分段字典，包含字段：\n - section: 段名称\n - duration: 时长（秒）\n - prompt: 用于音乐生成的提示词\n - temperature, top_p, top_k: 控制采样的超参数\n - end_at: 当前段结束时刻（累加）\n\n 异常:\n ValueError: 若 style 无法匹配结构库、或响应无法被正确解析时抛出。\n\n 注意:\n - 默认使用远程 OpenAI API，也支持通过 `use_local_llm` 使用 Ollama 模型。\n - 方法内部自动进行重试机制，并打印失败原因。\n - 所有生成内容中的 prompt 字段将仅使用提供的 token 列表，保证下游模型兼容性。\n \"\"\"\n structure = STYLE_STRUCTURE_LIB.get(style.lower())\n if not structure:\n raise ValueError(f\"风格 '{style}' 不支持\")\n\n fluent_prompt = f\"\"\"You are an experienced music producer. \n Please generate a JSON array of song segments where each item has the following keys: \n - section (string) \n - duration (int, seconds) \n - prompt (string) \n - temperature (float) \n - top_p (float) \n - top_k (float) \n \n Return only the raw JSON array without any markdown fences.\n \n The total duration should be about {total_sec} seconds (you may exceed but not fall short). \n Use these sections in order: {structure}. \n The style is: {user_prompt}. \n Each segment’s duration must be a multiple of {window_sec} seconds (e.g. if window_sec = 30, valid values are 30, 60, 90, …). \n top_p must be between 0.6 and 0.9, top_k should be about 250, temperature should be lower than 0.6.\n \n When constructing the prompt field inside each segment, use ONLY the following tokens, organized by category:\n \n Structure: \n intro-like, verse-like, pre-chorus-like, chorus-like, bridge-like, build-up-like, drop-like, interlude-like, outro-like.\n \n Mood / Emotion: \n calming, soothing, serene, ethereal, dreamy, happy, joyful, cheerful, gentle, groovy, funky, sad, melancholic, nostalgic, mournful, peaceful, uplifting, hopeful, inspirational, epic, triumphant, energetic, aggressive, angry, dark, mysterious, spooky, tense, suspenseful, anxious, majestic, glorious, romantic, playful, whimsical.\n \n Genre / Style: \n ambient, dark ambient, ambient techno, new age, chillout, lofi hip hop, trip hop, future bass, synthwave, vaporwave, house, deep house, progressive house, tech house, techno, psytrance, trance, drum and bass, jungle, dubstep, brostep, trap, EDM, jazz, smooth jazz, swing, bebop, latin jazz, bossa nova, samba, fusion, classical, baroque, romantic, modern classical, contemporary classical, orchestral, opera, chamber music, rock, classic rock, hard rock, punk rock, post-punk, indie rock, alternative rock, progressive rock, shoegaze, grunge, metal, heavy metal, death metal, black metal, thrash metal, pop, synth pop, indie pop, electropop, k-pop, j-pop, R&B, neo soul, soul, funk, disco, gospel, hip hop, rap, boom bap, trap rap, blues, delta blues, reggae, dub, ska, folk, bluegrass, country, americana, celtic, afrobeat, salsa, tango, flamenco, balkan, bollywood, world music, soundtrack, cinematic, video game, experimental, noise.\n \n Instrument / Voice: \n acoustic guitar, electric guitar, classical guitar, bass guitar, electric bass, acoustic bass, ukulele, banjo, mandolin, piano, electric piano, upright piano, harpsichord, organ, pipe organ, clavinet, synthesizer, bass synth, pad, lead synth, violin, viola, cello, double bass, string ensemble, orchestral strings, flute, piccolo, recorder, pan flute, clarinet, bass clarinet, oboe, english horn, bassoon, contrabassoon, saxophone, soprano saxophone, alto saxophone, tenor saxophone, baritone saxophone, trumpet, piccolo trumpet, cornet, flugelhorn, trombone, bass trombone, tuba, french horn, euphonium, harmonica, accordion, concertina, bagpipe, sitar, shamisen, koto, erhu, pipa, dizi, ocarina, didgeridoo, marimba, xylophone, vibraphone, glockenspiel, celesta, steel drum, drums, acoustic drum kit, electronic drum kit, snare drum, bass drum, cymbals, hi-hat, tom-tom, timpani, congas, bongo, djembe, tabla, triangle, tambourine, shaker, clap, percussion, male vocals, female vocals, mixed choir, children choir, solo voice.\n \n Environment: \n rain, heavy rain, thunder, wind, snowstorm, ocean waves, seashore, underwater ambience, forest birds, rainforest, desert wind, city daytime, city night, traffic noise, street ambience, airport terminal, subway station, train station, bus station, marketplace, stadium crowd, children playground, office background, classroom, library, crowd murmur, cafe background, industrial, construction noise, fireplace crackle, fireworks, police siren, space ambience, windchimes.\n \n Production Style: \n lo-fi, hi-fi, vintage, analog, digital, electronic, acoustic, orchestral, cinematic, soundtrack, psychedelic, experimental, noise, glitch, ambient, warm, bright, dark, harsh, smooth, compressed, bassy, spatial, surround, stereo wide, monophonic, polyphonic, minimal, maximalist, bitcrushed, chiptune, distorted, reverb-heavy, gated reverb, side-chained, auto-tuned.\n \n Energy / Tempo: \n high energy, powerful, relaxed, calm, dynamic, steady, variable, driving, lively, euphoric, hypnotic, pulsing, accelerando, decelerando.\n \n Chord / Harmony: \n C major, C minor, C7, Cmaj7, Cmin7, A major, A minor, A7, Amaj7, Amin7, Bb major, Bb minor, Bb7, Bbmaj7, B major, B minor, B7, Bmaj7, Bmin7, D major, D minor, D7, Dmaj7, Dmin7, E major, E minor, E7, Emaj7, Emin7, F major, F minor, F7, Fmaj7, Fmin7, F# major, F# minor, F#7, F#maj7, F#min7, G major, G minor, G7, Gmaj7, Gmin7, Ab major, Ab minor, Ab7, Abmaj7, Abmin7.\n\n CLAP generate Harmony:\n C, C#, D, D#, E, F, F#, G, G#, A, A#, B\n \n Example of two valid prompt strings for one segment: \n build-up-like, serene, delta blues, acoustic guitar, windchimes, soundtrack, accelerando, C minor, G \n interlude-like, suspenseful, neo soul, orchestral strings, space ambience, soundtrack, dynamic, C minor, G#.\n \n Return only the JSON array.\n \"\"\"\n\n \n raw = None\n for attempt in range(1, max_retries+1):\n try:\n if use_local_llm:\n # ➡️ 调 Ollama Chat —— \n resp = ollama.chat(\n model=\"deepseek-r1:14b\",\n messages=[{\"role\": \"user\", \"content\": fluent_prompt}]\n )\n \n # 直接从响应字典中提取文本\n try:\n text = resp[\"message\"][\"content\"]\n except Exception as e:\n raise ValueError(f\"⚠️ 无法解析 Ollama 返回结构: {e}\\n原始内容:\\n{resp}\")\n \n # 打印内容供 debug\n if self.verbose:\n print(\"the text is : \" + text)\n \n # 尝试提取 JSON 数组（包括 ```json ... ``` 或直接 [...]）\n m = re.search(r\"```json\\s*($begin:math:display$.*?$end:math:display$)\\s*```\", text, re.S)\n if not m:\n m = re.search(r\"\\[.*?\\]\", text, re.S)\n \n if not m:\n raise ValueError(f\"❌ 无法从文本中提取 JSON 数组:\\n{text}\")\n \n # 提取原始 JSON 内容\n raw = m.group(1) if m.lastindex else m.group(0)\n raw = raw.encode('utf-8').decode('unicode_escape')\n raw = raw.replace('\\\\\"', '\"')\n\n else:\n # ➡️ 调 LLM API\n payload = {\n \"model\": self.openai_model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": fluent_prompt}\n ]\n }\n headers = {\n \"Authorization\": f\"Bearer {self.openai_api_key}\",\n \"Content-Type\": \"application/json\"\n }\n\n url = API_ENDPOINT\n \n resp = requests.post(\n url, \n headers=headers, \n json=payload\n )\n\n resp.raise_for_status()\n raw = resp.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n \n \n if self.verbose:\n print(\"☀️ the raw json is : \"+raw)\n print(f\"[SegPlan] 成功获得规划 (attempt {attempt})\")\n break\n \n # 异常处理与多次重试 \n except requests.RequestException as e:\n if attempt < max_retries:\n wait = min(2 ** (attempt-1) , 60)\n if self.verbose:\n print(f\"[SegPlan] 请求失败，第{attempt}次重试，{wait}s后再试… ({e})\")\n time.sleep(wait)\n else:\n if self.verbose:\n print(f\"[SegPlan] 重试{max_retries}次失败，使用默认分段（❌没有完成，寄）\")\n # 默认回退\n if not raw:\n # 请求失败，使用固定默认分段列表，并继续后续解析流程\n if self.verbose:\n print(\"[SegPlan] 请求多次失败，使用固定默认分段列表\")\n default_segs = [\n {'section': 'minimal-pulse', 'duration': 30, 'prompt': 'intro-like, peaceful, ambient, bass synth, pad, warm, reverb-heavy, pulsing, Cmaj7, C', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'arpeggio-loop', 'duration': 30, 'prompt': 'verse-like, calming, ambient, synthesizer, pad, warm, smooth, slow, Amin7, A', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 8.0},\n {'section': 'white-noise-bed', 'duration': 30, 'prompt': 'pre-chorus-like, soothing, ambient, pad, space ambience, warm, reverb-heavy, slow, Dmin7, D', 'temperature': 0.65, 'top_p': 0.75, 'cfg': 6.5},\n {'section': 'modulation-bridge', 'duration': 30, 'prompt': 'bridge-like, ethereal, ambient, string ensemble, pad, warm, reverb-heavy, slow, Gmaj7, G', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 7.5},\n {'section': 'repetitive-theme', 'duration': 30, 'prompt': 'chorus-like, dreamy, ambient, string ensemble, pad, warm, smooth, slow, Fmaj7, F', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'soft-ending', 'duration': 30, 'prompt': 'outro-like, serene, ambient, pad, string ensemble, warm, smooth, decelerando, Cmaj7, C', 'temperature': 0.5, 'top_p': 0.6, 'cfg': 6.0},\n ]\n # 用默认列表生成 raw JSON，继续解析流程\n raw = json.dumps(default_segs, ensure_ascii=False)\n \n # —— 解析并补齐字段 —— 如果 LLM 给的内容有缺损，使用默认的参数补齐它\n segs = json.loads(raw)\n acc = 0\n for s in segs:\n s[\"duration\"] = int(s.get(\"duration\", window_sec))\n s[\"prompt\"] = s.get(\"prompt\", user_prompt)\n s[\"temperature\"] = float(s.get(\"temperature\", 0.7))\n s[\"top_p\"] = float(s.get(\"top_p\", 0.9))\n s[\"top_k\"] = float(s.get(\"top_k\", 250))\n acc += s[\"duration\"]\n s[\"end_at\"] = acc\n if self.verbose:\n print(\"[SegPlan] Parsed segments:\")\n for x in segs: print(x)\n\n return segs\n\n def compute_si_sdr(self, ref: torch.Tensor, est: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算 Scale-Invariant Signal-to-Distortion Ratio（SI-SDR），用于评估估计音频与参考音频之间的相似度。\n\n SI-SDR 是一种用于盲音源分离和音频质量评价的指标，具备尺度不变性，\n 可以排除音量（能量）缩放对相似度评分的影响。\n\n 参数:\n ref (torch.Tensor): 参考音频信号，形状为 [C, T] 或 [T]（C 为通道数，T 为时间长度）。\n est (torch.Tensor): 估计音频信号，形状与 ref 相同或兼容。\n eps (float): 避免除零错误的微小常数，默认值为 1e-8。\n\n 返回:\n torch.Tensor: 每个样本/通道的 SI-SDR 得分（以 dB 为单位），形状为 [C] 或标量。\n \"\"\"\n # 对齐 ref 和 est 的时间长度，支持单通道或多通道输入\n if ref.dim() == 2 and est.dim() == 2:\n T = min(ref.shape[1], est.shape[1])\n ref, est = ref[:, :T], est[:, :T]\n else:\n T = min(ref.shape[-1], est.shape[-1])\n ref, est = ref[:T], est[:T]\n\n # 去直流分量（零均值化）\n ref_zm = ref - ref.mean(-1, keepdim=True)\n est_zm = est - est.mean(-1, keepdim=True)\n\n # 计算参考信号能量\n ref_energy = torch.sum(ref_zm ** 2, dim=-1, keepdim=True)\n\n # 计算最佳缩放因子 scale，使得投影后的信号最接近估计信号\n scale = torch.sum(ref_zm * est_zm, dim=-1, keepdim=True) / (ref_energy + eps)\n\n # 将估计信号投影到参考信号方向\n proj = scale * ref_zm\n\n # 残差噪声 = 去中心化估计信号 - 投影部分\n noise = est_zm - proj\n\n # 计算信噪比（使用投影部分作为信号）\n snr = torch.sum(proj ** 2, dim=-1) / (torch.sum(noise ** 2, dim=-1) + eps)\n\n # 返回以 dB 为单位的 SI-SDR 分数\n return 10 * torch.log10(snr + eps)\n\n\n\n def mel_distance(self, x: torch.Tensor, y: torch.Tensor) -> float:\n \"\"\"\n 计算两个音频信号的梅尔谱（Mel Spectrogram）之间的 L2 距离，用于衡量音频在感知上的相似性。\n\n 此方法首先对两个音频信号做对齐裁剪，并通过 `self.mel_transform` 提取梅尔谱特征，\n 再计算其展开后的欧氏距离（L2 norm）。适用于感知相似度评估、音频生成对比等任务。\n\n 参数:\n x (torch.Tensor): 第一个音频信号，形状为 [T] 或 [C, T]。\n y (torch.Tensor): 第二个音频信号，形状与 x 相同或兼容。\n\n 返回:\n float: 梅尔谱之间的 L2 距离，数值越小表示音频越相似。\n \"\"\"\n # 对齐两个音频在时间维度上的长度\n if x.dim()==2 and y.dim()==2:\n T = min(x.shape[1], y.shape[1])\n x, y = x[:,:T], y[:,:T]\n else:\n T = min(x.shape[-1], y.shape[-1])\n x, y = x[:T], y[:T]\n\n # 通过 mel_transform 提取梅尔谱特征（添加 batch 维度）\n Mx = self.mel_transform(x.unsqueeze(0)) # [1, F, T']\n My = self.mel_transform(y.unsqueeze(0)) # [1, F, T']\n\n # 将梅尔谱差值展平后计算 L2 距离\n return torch.norm((Mx-My).reshape(-1), p=2).item()\n\n def xfade(self, a: torch.Tensor, b: torch.Tensor, ov: int) -> torch.Tensor:\n \"\"\"\n 将两段音频 a 和 b 在尾部和头部以交叉渐变方式进行拼接，实现平滑过渡。\n 自动对齐声道：如果 a 是多声道但 b 单声道，自动复制 b 的声道数。\n \"\"\"\n print(\"xfade input shapes:\", a.shape, b.shape)\n # 1) 自动对齐声道\n if a.dim() == 2 and b.dim() == 2 and a.shape[0] != b.shape[0]:\n # 如果 b 只有 1 个通道，但 a 有多个通道，就复制 b\n if b.shape[0] == 1:\n b = b.repeat(a.shape[0], 1)\n # 反之亦然（极少用到）\n elif a.shape[0] == 1:\n a = a.repeat(b.shape[0], 1)\n\n # 2) 计算 overlap 点数\n n = ov * self.SR\n\n # 3) 构造渐变权重\n w = torch.linspace(0, 1, n, device=a.device).sqrt()\n\n # 4) 交叉区混合\n left = a[..., :-n]\n middle = a[..., -n:] * (1 - w) + b[..., :n] * w\n right = b[..., n:]\n\n # 5) 拼接并返回\n return torch.cat([left, middle, right], dim=-1)\n\n # ====================== generate ======================\n @torch.no_grad()\n def generate(\n self,\n seed_path: str,\n style: str,\n user_prompt: str,\n total_sec: int = 120,\n window_sec: int = 20,\n overlap_sec: int = 5,\n prompt_sec: int = 5,\n safety_tail: int = 3,\n n_tracks: int = 1,\n clap_candidates: int = 3,\n sim_thresh: float = 0.5,\n sil_thresh: float = -40,\n melody_path: Optional[str] = None,\n out_path: Optional[str] = None,\n use_fp16: bool = False,\n ) -> Union[str, List[str]]:\n \"\"\"\n 基于种子音频和用户提示，分段、生成并拼接完整音乐轨道。\n\n 整体流程：\n 1. 读取 seed_path 指定的音频文件并重采样、提取 CLAP 嵌入；\n 2. （可选）读取 melody_path 指定的条件音频；\n 3. 调用 request_segment_plan 获取分段规划；\n 4. 对每个段落并行生成 clap_candidates 条候选，并按相似度选出最佳；\n 5. 通过 xfade 平滑拼接到当前音轨；\n 6. 重复直到累计生成达到 total_sec，并保存最终音轨。\n\n 参数:\n seed_path (str):\n 初始音频文件路径，用于续写输入。\n style (str):\n 分段风格名称，对应内部 STYLE_STRUCTURE_LIB 的键。\n user_prompt (str):\n 用户文本提示，用于精细化每段的生成风格。\n total_sec (int, 可选):\n 目标总时长（秒）。默认 120。\n window_sec (int, 可选):\n 每次生成的片段时长（秒）。默认 20。\n overlap_sec (int, 可选):\n 段落重叠长度（秒），用于淡入淡出拼接。默认 5。\n prompt_sec (int, 可选):\n 每次构建生成 prompt 时要截取的尾部时长（秒）。默认 5。\n safety_tail (int, 可选):\n 生成 prompt 前保留的不参与提示的尾巴长度（秒），避免拼接突变。默认 3。\n n_tracks (int, 可选):\n 并行生成的轨道数量；>1 时返回路径列表。默认 1。\n clap_candidates (int, 可选):\n 每段并行生成的候选数，用于多版本筛选。默认 3。\n sim_thresh (float, 可选):\n 最优候选的相似度阈值；低于则触发重试或跳过。默认 0.5。\n sil_thresh (float, 可选):\n 静音检测阈值（dB）；可用于剔除无声片段。默认 -40。\n melody_path (Optional[str], 可选):\n 条件 melody 音频文件路径；不提供则进入纯生成模式。默认 None。\n out_path (Optional[str], 可选):\n 最终输出文件路径；多轨时自动编号。默认 None。\n use_fp16 (bool, 可选):\n 在 CUDA 上是否启用半精度推理（torch.cuda.amp）。默认 False.\n\n 返回:\n Union[str, List[str]]:\n 生成的音频文件路径。若 n_tracks>1 则为路径列表，否则为单个字符串。\n\n 异常:\n ValueError:\n - 当分段规划（request_segment_plan）失败且无法解析时。\n - 当读取音频文件或网络请求出错并超过重试次数时。\n\n 示例:\n >>> out = svc.generate(\n ... seed_path=\"seed.wav\",\n ... style=\"ambient\",\n ... user_prompt=\"dreamy pad with rain\",\n ... total_sec=180,\n ... n_tracks=2,\n ... clap_candidates=5,\n ... use_fp16=True,\n ... out_path=\"output/track.wav\"\n ... )\n \"\"\"\n\n print(f\"[Generate] start: n_tracks={n_tracks}, clap_candidates={clap_candidates}, use_fp16={use_fp16}, device={self.device}\", flush=True)\n\n fp16_ctx = torch.cuda.amp.autocast if (use_fp16 and self.device.startswith(\"cuda\")) else nullcontext\n\n def _run_once(idx: int) -> str:\n checkpoint_path= self.checkpoint_path\n USE_BF16 = torch.cuda.is_bf16_supported()\n DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n ENABLE_COMPILE = True # False 可快速回退\n STRICT_LOAD = False\n\n print(f\"[Track {idx+1}/{n_tracks}] >>> begin _run_once\", flush=True)\n torch.manual_seed(42 + idx)\n\n # 1) 载入 seed 音频\n seed32, sr = torchaudio.load(seed_path)\n if sr != self.SR:\n seed32 = torchaudio.functional.resample(seed32, sr, self.SR)\n seed32 = seed32.to(self.device)\n ref_emb = self.embed_cpu(seed32)\n\n # 2) 载入可选的 melody 条件\n melody_tensor = None\n if melody_path:\n m32, mr = torchaudio.load(melody_path)\n if mr != self.SR:\n m32 = torchaudio.functional.resample(m32, mr, self.SR)\n melody_tensor = m32.to(self.device).unsqueeze(0)\n\n # 3) 段落规划\n segs = self.request_segment_plan(style, user_prompt, total_sec, window_sec)\n need = (window_sec - overlap_sec) * self.SR\n full = seed32[..., :need] if seed32.shape[-1] >= need else seed32.repeat(\n 1, math.ceil(need / seed32.shape[-1])\n )[..., :need]\n cur_seg = 0\n\n # 4) 每段生成参数设置\n def set_params(idx_seg):\n p = segs[idx_seg]\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,#p[\"temperature\"],\n top_p=0.9,#p[\"top_p\"],\n top_k=250,#p[\"cfg\"],\n #use_sampling=True,\n )\n\n set_params(cur_seg)\n retry_count = 0\n max_retries = 5\n\n # 5) 主循环：续写直到达到 total_sec\n while full.shape[-1] / self.SR < total_sec:\n # 切换到下一个段落\n if full.shape[-1] / self.SR >= segs[cur_seg][\"end_at\"] and cur_seg < len(segs) - 1:\n cur_seg += 1\n set_params(cur_seg)\n retry_count = 0\n if self.verbose:\n print(f\"▶ Enter segment {cur_seg}: {segs[cur_seg]['section']}\")\n \n\n # 准备 prompt 音频片段\n tail = full.shape[-1] - safety_tail * self.SR\n head = max(0, tail - prompt_sec * self.SR)\n pa = full[..., head:tail]\n if pa.shape[-1] < 1024:\n pa = torch.nn.functional.pad(pa, (1024 - pa.shape[-1], 0), \"reflect\")\n pa = pa.unsqueeze(0)\n\n # 并行多候选生成\n current_prompt = segs[cur_seg][\"prompt\"]\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] generating {clap_candidates} candidates\", flush=True)\n descriptions = [current_prompt] * clap_candidates\n \n\n with fp16_ctx():\n if melody_tensor is not None:\n if self.verbose:\n print(\"➡️ now in the melody mode\")\n wavs = self.model.generate(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n melody=melody_tensor,\n duration=window_sec,\n progress=self.verbose,\n )\n else:\n # if self.verbose:\n # print(\"➡️ now in the gen mode\")\n # torch.manual_seed(42) # 固定seed\n # wavs = self.model.generate(\n # descriptions,\n # progress=self.verbose,\n # )\n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # —— 使用 generate_continuation 接口 —— \n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # 续写：使用之前 set_generation_params 设置的 duration\n wavs = self.model.generate_continuation(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n progress=self.verbose,\n )\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] got {len(wavs)} wavs\", flush=True)\n\n # 6) 选出相似度最高的 wav，同时收集所有 sim 值\n sim_list = []\n best, best_sim = None, -1.0\n \n def _ensure_mono32(t: torch.Tensor, sr: int = 32000) -> torch.Tensor:\n \"\"\"\n 保证张量变成 [1, T] 单声道并且已是 32 kHz。\n 这里假设外层音频都已重采样到 self.SR=32 kHz；仅做声道合并。\n \"\"\"\n if t.dim() == 2 and t.shape[0] > 1: # 立体声 → 单声道\n t = t.mean(0, keepdim=True)\n return t.to(self.device)\n\n for i, wav in enumerate(wavs):\n # ------------- 1) 规格统一 -------------\n wav_mono = _ensure_mono32(wav) # 生成片段\n ref_tail = _ensure_mono32(pa.squeeze(0)) # prompt tail\n # 时间对齐：裁到同长（取短的一方）\n T = min(wav_mono.shape[-1], ref_tail.shape[-1])\n wav_mono = wav_mono[..., :T]\n ref_tail = ref_tail[..., :T]\n\n # ------------- 2) 计算三项相似度 ----------\n sim_clap = torch.cosine_similarity(\n self.embed_cpu(wav_mono), self.embed_cpu(ref_tail)\n ).item()\n\n sdr_db = self.compute_si_sdr(ref_tail, wav_mono).mean().item() # dB\n mel_l2 = self.mel_distance(ref_tail, wav_mono) # L2\n\n # 映射到 0–1 区间\n sdr_norm = min(max((sdr_db - 0) / 10.0, 0.0), 1.0) # 0 dB→0, 10 dB→1\n mel_norm = min(250.0 / (mel_l2 + 1.0), 1.0) # 250→1, 500→0.5\n\n # 综合得分\n sim_final = 0.4 * sim_clap + 0.4 * sdr_norm + 0.2 * mel_norm\n\n # ------------- 3) 调试输出 --------------\n if self.verbose:\n print(\n f\"CLAP={sim_clap:.3f} \"\n f\"SDR={sdr_db:.2f}dB(n={sdr_norm:.2f}) \"\n f\"MelL2={mel_l2:.0f}(n={mel_norm:.2f}) \"\n f\"final={sim_final:.3f}\"\n )\n\n # ------------- 4) 选最佳 ----------------\n sim_list.append(sim_final)\n if sim_final > best_sim:\n best, best_sim = wav_mono.cpu(), sim_final\n del wav\n\n torch.cuda.empty_cache()\n\n # 打印这一段所有候选的 sim 数组和选出的 best_sim\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] sims = {sim_list}\")\n print(f\"[Track {idx+1} Segment {cur_seg}] best_sim = {best_sim:.4f}\")\n\n save_best_each = True\n # 立刻保存本段 best 候选 —— \n if save_best_each and out_path:\n # out_path 父目录下创建 bestwav 文件夹\n best_dir = pathlib.Path(out_path).parent / \"bestwav\"\n best_dir.mkdir(parents=True, exist_ok=True)\n save_name = best_dir / f\"Segment_{cur_seg}.wav\"\n\n # 生成子文件名，并写文件\n # 生成微调结果\n # 把 float16 转成 float32，再保存\n \n audio_ft = best.cpu().float().numpy()\n if audio_ft.ndim == 2 and audio_ft.shape[0] < audio_ft.shape[1]:\n audio_ft = audio_ft.T\n sf.write(str(save_name), audio_ft, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1} Seg {cur_seg}] saved best wav → {save_name}\")\n\n # 7) 相似度太低时重试或跳过\n if best_sim < sim_thresh:\n retry_count += 1\n if retry_count > max_retries:\n if self.verbose:\n print(f\"Segment {cur_seg} reached max retries, skipping segment\")\n retry_count = 0\n break\n p = segs[cur_seg]\n new_temp = max(0.15, p[\"temperature\"] * (0.9 ** retry_count))\n new_top_k = p[\"top_k\"] * (1.15 ** retry_count)\n new_top_p = min(p[\"top_p\"]+0.05,0.95)\n self.model.set_generation_params(\n duration=window_sec,\n temperature=new_temp,\n top_k=new_top_k,\n top_p = new_top_p,\n use_sampling=True,\n )\n if self.verbose:\n print(f\"Retry {retry_count}: temp→{new_temp:.2f}, topk→{new_top_k:.2f}\")\n continue\n\n # 8) 拼接并输出调试信息\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] before xfade: {full.shape[-1]/self.SR:.1f}s\")\n full = self.xfade(full, best.to(self.device), overlap_sec)\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] after xfade: {full.shape[-1]/self.SR:.1f}s\")\n del best\n torch.cuda.empty_cache()\n retry_count = 0\n\n # 完成 single-track 生成，导出文件\n final = full[..., : total_sec * self.SR].cpu().T\n if out_path:\n stem, ext = os.path.splitext(out_path)\n out_file = f\"{stem}_{idx+1:02d}{ext}\" if n_tracks > 1 else out_path\n else:\n out_file = f\"final_{idx+1:02d}.wav\" if n_tracks > 1 else \"final.wav\"\n\n pathlib.Path(out_file).parent.mkdir(parents=True, exist_ok=True)\n sf.write(out_file, final, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1}] saved {out_file}, best_sim={best_sim:.4f}\", flush=True)\n return out_file\n\n # 并行生成多条 track\n print(f\"[Generate] launching _run_once for {n_tracks} tracks\", flush=True)\n paths = [_run_once(i) for i in range(n_tracks)]\n print(f\"[Generate] all done, paths = {paths}\", flush=True)\n return paths if n_tracks > 1 else paths[0]\n\n \n\n\ndef generate_music_api(params: Dict) -> Dict:\n svc = MusicGenService(\n api_endpoint=params.get('api_endpoint', API_ENDPOINT),\n openai_api_key=params.get('api_key', OPENAI_API_KEY),\n openai_model=params.get('model', OPENAI_MODEL),\n verbose=params.get('verbose', True),\n checkpoint_path=params.get('checkpoint_path'), # ← 把它传进去\n )\n out = svc.generate(\n seed_path=params['seed_path'],\n style=params['style'],\n user_prompt=params['user_prompt'],\n total_sec=params.get('total_sec', 120),\n window_sec=params.get('window_sec', 20),\n overlap_sec=params.get('overlap_sec', 5),\n prompt_sec=params.get('prompt_sec', 5),\n safety_tail=params.get('safety_tail', 3),\n clap_candidates=params.get('clap_candidates', 3),\n n_tracks=params.get('n_tracks', 1), # ← 新增这一行\n use_fp16=params.get('use_fp16', False), # ← 以及这一行\n sim_thresh=params.get('sim_thresh', 0.5),\n sil_thresh=params.get('sil_thresh', -40),\n melody_path=params.get('melody_path'),\n out_path=params.get('out_path'),\n )\n return {'out_path': out}\n\n"}}, "structure_gen/.ipynb_checkpoints/musicgen_api_-checkpoint.py": {"486bab9207d6499336bfe192c2ed40eb91b6c4b0": {"t": 1785768921.1081285, "n": 35843, "h": "306955e38b2bc869", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nmusicgen_api.py\n\n封装 MusicGen 无限续写脚本为后端可调用接口。\n支持 Melody 条件生成、增量续写、重试与显存管理。\n\"\"\"\n\nnetworks = False # 如果没有接入国际互联网，就使用 False\n\n\nimport os\n# 如果没有接入国际互联网，就将 Huggingface 的访问设置为【不访问】\nif not networks:\n # 确保 transformers/hub 都用离线模式\n os.environ[\"HF_HUB_OFFLINE\"] = \"1\"\n os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n # 然后 monkey-patch BertTokenizer.from_pretrained，让它只走本地缓存\n from transformers import BertTokenizer\n _original_bp = BertTokenizer.from_pretrained\n def _from_pretrained_local(repo_id, *args, **kwargs):\n # 强制 local_files_only=True\n return _original_bp(repo_id, local_files_only=True, *args, **kwargs)\n BertTokenizer.from_pretrained = _from_pretrained_local\n\n\nimport json\nimport math\nimport warnings\nimport re\nimport time\nimport ollama\nimport torchaudio\nimport torch\nimport requests\nimport pathlib\nimport soundfile as sf\n\nfrom contextlib import nullcontext\nfrom typing import List, Optional, Union, Dict\nfrom urllib.parse import urljoin\nfrom audiocraft.models import MusicGen\nfrom laion_clap import CLAP_Module\n#from structure_gen.music_structure import STYLE_STRUCTURE_LIB # lin\nfrom music_structure import STYLE_STRUCTURE_LIB # yuan\n\n# ========== 默认配置==========\n\nDEFAULT_USE_LOCAL_LLM = False # 默认是否走本地 LLM（Ollama）\nOPENAI_MODEL = 'gpt-4o-mini' # 远程 LLM 模型名\nmax_retries = 6 # 远程 LLM 最大重试次数\n\nOPENAI_API_KEY = 'sk-vWyNj5NxixWjJcjknaxNivrtIkHIjXDBVlk1b6wRW2pZc0hz' # 网络LLM API KEY\n\n\nUSE_BF16 = torch.cuda.is_bf16_supported() # \nDTYPE = torch.bfloat16 if USE_BF16 else torch.float16 \n\nENABLE_COMPILE = True # False 可快速回退\nSTRICT_LOAD = False\n\n\n# =========================================\n\n\n\n\ndef cast_musicgen_dtype(mg, dtype):\n \"\"\"把 MusicGen 内所有子模块转成半精度 / 全精度\"\"\"\n mg.lm = mg.lm.to(dtype=dtype)\n if hasattr(mg, \"decoder\"):\n mg.decoder = mg.decoder.to(dtype=dtype)\n if hasattr(mg, \"condition_provider\"):\n mg.condition_provider = mg.condition_provider.to(dtype=dtype)\n return mg\n\n\nclass MusicGenService:\n def __init__(\n self,\n *,\n use_local_llm: bool = DEFAULT_USE_LOCAL_LLM,\n api_endpoint: Optional[str] = None,\n openai_api_key: str = OPENAI_API_KEY,\n openai_model: str = OPENAI_MODEL,\n verbose: bool = True,\n device: Optional[str] = None,\n checkpoint_path: Optional[str] = None,\n use_fp16: bool = True,\n ):\n \n # —— 与 LLM 相关的配置 ——\n self.use_local_llm = use_local_llm\n if api_endpoint is None:\n # 如果调用方没指定，按 use_local_llm 选默认地址\n self.api_endpoint = (\n \"http://localhost:11434/api/generate\"\n if self.use_local_llm\n else \"https://www.dmxapi.com/v1/chat/completions\"\n )\n else:\n self.api_endpoint = api_endpoint\n # 参数配置\n self.openai_api_key = openai_api_key\n self.openai_model = openai_model\n self.max_retries = max_retries\n self.verbose = verbose\n self.device = device or (\"cuda\" if torch.cuda.is_available() else \"cpu\")\n self.use_fp16 = use_fp16\n self.clap = CLAP_Module(enable_fusion=False, device=self.device).eval()\n self.SR = 32000\n self.mel_transform = torchaudio.transforms.MelSpectrogram(\n sample_rate=self.SR, n_mels=80, hop_length=512\n ).to(self.device)\n self.checkpoint_path = checkpoint_path\n\n\n # 1) 加载 small 版本\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}...\")\n self.model = MusicGen.get_pretrained(\"facebook/musicgen-small\", self.device)\n\n # 设置参数\n self.model.set_generation_params(\n duration=30,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n\n # 2) 如果指定了 checkpoint，就加载权重\n if checkpoint_path:\n if self.verbose:\n print(f\"[Init] Loading checkpoint from {checkpoint_path}…\")\n ckpt = torch.load(checkpoint_path, map_location=\"cpu\")\n state_dict = ckpt.get(\"model_state_dict\", ckpt)\n missing, unexpected = self.model.lm.load_state_dict(state_dict, strict=True)\n if self.verbose:\n print(f\"[Init] Checkpoint loaded. missing keys: {len(missing)}, unexpected: {len(unexpected)}\")\n\n # 3) 半精度转换 & 编译\n self.model = cast_musicgen_dtype(self.model, DTYPE)\n print(\"Model dtype:\", next(self.model.lm.parameters()).dtype)\n\n # 编译 lm 子模块以减少 Python 调度开销\n try:\n self.model.lm = torch.compile(self.model.lm,mode=\"reduce-overhead\",fullgraph=True)\n if self.verbose:\n print(\"[Init] 🚀 torch.compile() 已启用\")\n except Exception as e:\n if self.verbose:\n print(f\"[Init] ⚠️ torch.compile 失败，回退: {e}\")\n\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}... OK\")\n\n\n @torch.no_grad()\n def embed_cpu(self, wav32: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 将输入的多通道高采样率音频数据转换为单通道 16kHz，\n 并用 CLAP 模型提取其在 CPU 上的特征嵌入表示，\n 用于后续相似度计算或音频理解任务。\n \"\"\"\n mono = wav32.to(\"cpu\", torch.float32).mean(0, keepdim=True)\n m16 = torchaudio.functional.resample(mono, self.SR, 16000)\n return self.clap.get_audio_embedding_from_data(m16, use_tensor=True).cpu()\n\n \n def request_segment_plan(\n self, style: str, user_prompt: str, total_sec: int, window_sec: int\n ) -> List[Dict]:\n \"\"\"\n 根据用户提供的风格、提示词和时长设置，调用本地或远程 ChatGPT 接口生成音乐分段规划。\n\n 该方法会构造一个详细的自然语言提示（prompt），并附带 CLAP 支持的 token 列表，提交给语言模型，\n 请求生成一个包含多个段落的 JSON 列表。每个段落包含音乐结构（如 intro、verse 等）、持续时间、\n 生成提示词以及采样参数（temperature、top_p、top_k）等信息。返回结果会自动补齐字段并计算结束时间戳。\n\n 如果调用失败，方法将退回使用内置的默认分段方案。\n\n 参数:\n style (str): 音乐结构风格名，对应 STYLE_STRUCTURE_LIB 中的 key。\n user_prompt (str): 用户自定义的风格描述文本，将传入 LLM 构造提示。\n total_sec (int): 希望生成的总时长（秒），提示中允许略超，但不得不足。\n window_sec (int): 每个段落的时间粒度，必须是段落时长的倍数。\n\n 返回:\n List[Dict]: 每个元素为一个分段字典，包含字段：\n - section: 段名称\n - duration: 时长（秒）\n - prompt: 用于音乐生成的提示词\n - temperature, top_p, top_k: 控制采样的超参数\n - end_at: 当前段结束时刻（累加）\n\n 异常:\n ValueError: 若 style 无法匹配结构库、或响应无法被正确解析时抛出。\n\n 注意:\n - 默认使用远程 OpenAI API，也支持通过 `use_local_llm` 使用 Ollama 模型。\n - 方法内部自动进行重试机制，并打印失败原因。\n - 所有生成内容中的 prompt 字段将仅使用提供的 token 列表，保证下游模型兼容性。\n \"\"\"\n structure = STYLE_STRUCTURE_LIB.get(style.lower())\n if not structure:\n raise ValueError(f\"风格 '{style}' 不支持\")\n structure = \", \".join(structure)\n if self.verbose:\n print(\"structure is:\"+ structure)\n\n\n fluent_prompt = f\"\"\"You are an experienced music producer. \n Please generate a JSON array of song segments where each item has the following keys: \n - section (string) \n - duration (int, seconds) \n - prompt (string) \n - temperature (float) \n - top_p (float) \n - top_k (float) \n \n Return only the raw JSON array without any markdown fences.\n \n Description is: {user_prompt}, Style is: {style}.\n The total duration should be about {total_sec} seconds (you may exceed but not fall short).\n Use these sections in order: {structure}.\n Each segment’s duration must be a multiple of {window_sec} seconds (e.g. if window_sec = 30, valid values are 30, 60, 90, …).\n top_p must be between 0.6 and 0.9, top_k should be about 250, temperature should be higher than 1.0.\n Construct the prompt field for each segment depending on description {user_prompt} and style {style}.\n When constructing the prompt field inside each segment, use ONLY the following tokens, organized by category. \n Ensuring that any Genre / Style token you choose exactly matches the value in {style}:\n \n Structure: \n intro-like, verse-like, pre-chorus-like, chorus-like, bridge-like, build-up-like, drop-like, interlude-like, outro-like.\n \n Mood / Emotion: \n calming, soothing, serene, ethereal, dreamy, happy, joyful, cheerful, gentle, groovy, funky, sad, melancholic, nostalgic, mournful, peaceful, uplifting, hopeful, inspirational, epic, triumphant, energetic, aggressive, angry, dark, mysterious, spooky, tense, suspenseful, anxious, majestic, glorious, romantic, playful, whimsical.\n \n Genre / Style: \n {style}.\n \n Instrument / Voice: \n acoustic guitar, electric guitar, classical guitar, bass guitar, electric bass, acoustic bass, ukulele, banjo, mandolin, piano, electric piano, upright piano, harpsichord, organ, pipe organ, clavinet, synthesizer, bass synth, pad, lead synth, violin, viola, cello, double bass, string ensemble, orchestral strings, flute, piccolo, recorder, pan flute, clarinet, bass clarinet, oboe, english horn, bassoon, contrabassoon, saxophone, soprano saxophone, alto saxophone, tenor saxophone, baritone saxophone, trumpet, piccolo trumpet, cornet, flugelhorn, trombone, bass trombone, tuba, french horn, euphonium, harmonica, accordion, concertina, bagpipe, sitar, shamisen, koto, erhu, pipa, dizi, ocarina, didgeridoo, marimba, xylophone, vibraphone, glockenspiel, celesta, steel drum, drums, acoustic drum kit, electronic drum kit, snare drum, bass drum, cymbals, hi-hat, tom-tom, timpani, congas, bongo, djembe, tabla, triangle, tambourine, shaker, clap, percussion, male vocals, female vocals, mixed choir, children choir, solo voice.\n \n Environment: \n rain, heavy rain, thunder, wind, snowstorm, ocean waves, seashore, underwater ambience, forest birds, rainforest, desert wind, city daytime, city night, traffic noise, street ambience, airport terminal, subway station, train station, bus station, marketplace, stadium crowd, children playground, office background, classroom, library, crowd murmur, cafe background, industrial, construction noise, fireplace crackle, fireworks, police siren, space ambience, windchimes.\n \n Production Style: \n lo-fi, hi-fi, vintage, analog, digital, electronic, acoustic, orchestral, cinematic, soundtrack, psychedelic, experimental, noise, glitch, ambient, warm, bright, dark, harsh, smooth, compressed, bassy, spatial, surround, stereo wide, monophonic, polyphonic, minimal, maximalist, bitcrushed, chiptune, distorted, reverb-heavy, gated reverb, side-chained, auto-tuned.\n \n Energy / Tempo: \n high energy, powerful, relaxed, calm, dynamic, steady, variable, driving, lively, euphoric, hypnotic, pulsing, accelerando, decelerando.\n \n Chord / Harmony: \n C major, C minor, C7, Cmaj7, Cmin7, A major, A minor, A7, Amaj7, Amin7, Bb major, Bb minor, Bb7, Bbmaj7, B major, B minor, B7, Bmaj7, Bmin7, D major, D minor, D7, Dmaj7, Dmin7, E major, E minor, E7, Emaj7, Emin7, F major, F minor, F7, Fmaj7, Fmin7, F# major, F# minor, F#7, F#maj7, F#min7, G major, G minor, G7, Gmaj7, Gmin7, Ab major, Ab minor, Ab7, Abmaj7, Abmin7.\n\n CLAP generate Harmony:\n C, C#, D, D#, E, F, F#, G, G#, A, A#, B\n \n Example of two valid prompt strings for one segment: \n build-up-like, serene, delta blues, acoustic guitar, windchimes, soundtrack, accelerando, C minor, G \n interlude-like, suspenseful, neo soul, orchestral strings, space ambience, soundtrack, dynamic, C minor, G#.\n \n Return only the JSON array.\n \"\"\"\n\n # ambient, dark ambient, ambient techno, new age, chillout, lofi hip hop, trip hop, future bass, synthwave, vaporwave, house, deep house, progressive house, tech house, techno, psytrance, trance, drum and bass, jungle, dubstep, brostep, trap, EDM, jazz, smooth jazz, swing, bebop, latin jazz, bossa nova, samba, fusion, classical, baroque, romantic, modern classical, contemporary classical, orchestral, opera, chamber music, rock, classic rock, hard rock, punk rock, post-punk, indie rock, alternative rock, progressive rock, shoegaze, grunge, metal, heavy metal, death metal, black metal, thrash metal, pop, synth pop, indie pop, electropop, k-pop, j-pop, R&B, neo soul, soul, funk, disco, gospel, hip hop, rap, boom bap, trap rap, blues, delta blues, reggae, dub, ska, folk, bluegrass, country, americana, celtic, afrobeat, salsa, tango, flamenco, balkan, bollywood, world music, soundtrack, cinematic, video game, experimental, noise.\n \n raw = None\n for attempt in range(1, max_retries+1):\n try:\n if self.use_local_llm:\n # ➡️ 调 Ollama Chat —— \n resp = ollama.chat(\n model=\"deepseek-r1:14b\",\n messages=[{\"role\": \"user\", \"content\": fluent_prompt}]\n )\n \n # 直接从响应字典中提取文本\n try:\n text = resp[\"message\"][\"content\"]\n except Exception as e:\n raise ValueError(f\"⚠️ 无法解析 Ollama 返回结构: {e}\\n原始内容:\\n{resp}\")\n \n # 打印内容供 debug\n if self.verbose:\n print(\"the text is : \" + text)\n \n # 尝试提取 JSON 数组（包括 ```json ... ``` 或直接 [...]）\n m = re.search(r\"```json\\s*($begin:math:display$.*?$end:math:display$)\\s*```\", text, re.S)\n if not m:\n m = re.search(r\"\\[.*?\\]\", text, re.S)\n \n if not m:\n raise ValueError(f\"❌ 无法从文本中提取 JSON 数组:\\n{text}\")\n \n # 提取原始 JSON 内容\n raw = m.group(1) if m.lastindex else m.group(0)\n raw = raw.encode('utf-8').decode('unicode_escape')\n raw = raw.replace('\\\\\"', '\"')\n\n else:\n # ➡️ 调 LLM API\n payload = {\n \"model\": self.openai_model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": fluent_prompt}\n ]\n }\n headers = {\n \"Authorization\": f\"Bearer {self.openai_api_key}\",\n \"Content-Type\": \"application/json\"\n }\n\n url = self.api_endpoint\n \n resp = requests.post(\n url, \n headers=headers, \n json=payload\n )\n\n resp.raise_for_status()\n raw = resp.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n \n \n if self.verbose:\n print(\"☀️ the raw json is : \"+raw)\n print(f\"[SegPlan] 成功获得规划 (attempt {attempt})\")\n break\n \n # 异常处理与多次重试 \n except requests.RequestException as e:\n if attempt < max_retries:\n wait = min(2 ** (attempt-1) , 60)\n if self.verbose:\n print(f\"[SegPlan] 请求失败，第{attempt}次重试，{wait}s后再试… ({e})\")\n time.sleep(wait)\n else:\n if self.verbose:\n print(f\"[SegPlan] 重试{max_retries}次失败，使用默认分段（❌没有完成，寄）\")\n # 默认回退\n if not raw:\n # 请求失败，使用固定默认分段列表，并继续后续解析流程\n if self.verbose:\n print(\"[SegPlan] 请求多次失败，使用固定默认分段列表\")\n default_segs = [\n {'section': 'minimal-pulse', 'duration': 30, 'prompt': 'intro-like, peaceful, ambient, bass synth, pad, warm, reverb-heavy, pulsing, Cmaj7, C', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'arpeggio-loop', 'duration': 30, 'prompt': 'verse-like, calming, ambient, synthesizer, pad, warm, smooth, slow, Amin7, A', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 8.0},\n {'section': 'white-noise-bed', 'duration': 30, 'prompt': 'pre-chorus-like, soothing, ambient, pad, space ambience, warm, reverb-heavy, slow, Dmin7, D', 'temperature': 0.65, 'top_p': 0.75, 'cfg': 6.5},\n {'section': 'modulation-bridge', 'duration': 30, 'prompt': 'bridge-like, ethereal, ambient, string ensemble, pad, warm, reverb-heavy, slow, Gmaj7, G', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 7.5},\n {'section': 'repetitive-theme', 'duration': 30, 'prompt': 'chorus-like, dreamy, ambient, string ensemble, pad, warm, smooth, slow, Fmaj7, F', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'soft-ending', 'duration': 30, 'prompt': 'outro-like, serene, ambient, pad, string ensemble, warm, smooth, decelerando, Cmaj7, C', 'temperature': 0.5, 'top_p': 0.6, 'cfg': 6.0},\n ]\n # 用默认列表生成 raw JSON，继续解析流程\n raw = json.dumps(default_segs, ensure_ascii=False)\n \n # —— 解析并补齐字段 —— 如果 LLM 给的内容有缺损，使用默认的参数补齐它\n segs = json.loads(raw)\n acc = 0\n for s in segs:\n s[\"duration\"] = int(s.get(\"duration\", window_sec))\n s[\"prompt\"] = s.get(\"prompt\", user_prompt)\n s[\"temperature\"] = float(s.get(\"temperature\", 0.7))\n s[\"top_p\"] = float(s.get(\"top_p\", 0.9))\n s[\"top_k\"] = float(s.get(\"top_k\", 250))\n acc += s[\"duration\"]\n s[\"end_at\"] = acc\n if self.verbose:\n print(\"[SegPlan] Parsed segments:\")\n for x in segs: print(x)\n\n return segs\n\n def compute_si_sdr(self, ref: torch.Tensor, est: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算 Scale-Invariant Signal-to-Distortion Ratio（SI-SDR），用于评估估计音频与参考音频之间的相似度。\n\n SI-SDR 是一种用于盲音源分离和音频质量评价的指标，具备尺度不变性，\n 可以排除音量（能量）缩放对相似度评分的影响。\n\n 参数:\n ref (torch.Tensor): 参考音频信号，形状为 [C, T] 或 [T]（C 为通道数，T 为时间长度）。\n est (torch.Tensor): 估计音频信号，形状与 ref 相同或兼容。\n eps (float): 避免除零错误的微小常数，默认值为 1e-8。\n\n 返回:\n torch.Tensor: 每个样本/通道的 SI-SDR 得分（以 dB 为单位），形状为 [C] 或标量。\n \"\"\"\n # 对齐 ref 和 est 的时间长度，支持单通道或多通道输入\n if ref.dim() == 2 and est.dim() == 2:\n T = min(ref.shape[1], est.shape[1])\n ref, est = ref[:, :T], est[:, :T]\n else:\n T = min(ref.shape[-1], est.shape[-1])\n ref, est = ref[:T], est[:T]\n\n # 去直流分量（零均值化）\n ref_zm = ref - ref.mean(-1, keepdim=True)\n est_zm = est - est.mean(-1, keepdim=True)\n\n # 计算参考信号能量\n ref_energy = torch.sum(ref_zm ** 2, dim=-1, keepdim=True)\n\n # 计算最佳缩放因子 scale，使得投影后的信号最接近估计信号\n scale = torch.sum(ref_zm * est_zm, dim=-1, keepdim=True) / (ref_energy + eps)\n\n # 将估计信号投影到参考信号方向\n proj = scale * ref_zm\n\n # 残差噪声 = 去中心化估计信号 - 投影部分\n noise = est_zm - proj\n\n # 计算信噪比（使用投影部分作为信号）\n snr = torch.sum(proj ** 2, dim=-1) / (torch.sum(noise ** 2, dim=-1) + eps)\n\n # 返回以 dB 为单位的 SI-SDR 分数\n return 10 * torch.log10(snr + eps)\n\n def rms_db(self, x: torch.Tensor) -> float:\n return 20*torch.log10(x.square().mean().sqrt()+1e-9).item()\n\n def mel_distance(self, x: torch.Tensor, y: torch.Tensor) -> float:\n \"\"\"\n 计算两个音频信号的梅尔谱（Mel Spectrogram）之间的 L2 距离，用于衡量音频在感知上的相似性。\n\n 此方法首先对两个音频信号做对齐裁剪，并通过 `self.mel_transform` 提取梅尔谱特征，\n 再计算其展开后的欧氏距离（L2 norm）。适用于感知相似度评估、音频生成对比等任务。\n\n 参数:\n x (torch.Tensor): 第一个音频信号，形状为 [T] 或 [C, T]。\n y (torch.Tensor): 第二个音频信号，形状与 x 相同或兼容。\n\n 返回:\n float: 梅尔谱之间的 L2 距离，数值越小表示音频越相似。\n \"\"\"\n # 对齐两个音频在时间维度上的长度\n if x.dim()==2 and y.dim()==2:\n T = min(x.shape[1], y.shape[1])\n x, y = x[:,:T], y[:,:T]\n else:\n T = min(x.shape[-1], y.shape[-1])\n x, y = x[:T], y[:T]\n\n # 通过 mel_transform 提取梅尔谱特征（添加 batch 维度）\n Mx = self.mel_transform(x.unsqueeze(0)) # [1, F, T']\n My = self.mel_transform(y.unsqueeze(0)) # [1, F, T']\n\n # 将梅尔谱差值展平后计算 L2 距离\n return torch.norm((Mx-My).reshape(-1), p=2).item()\n\n def xfade(self, a: torch.Tensor, b: torch.Tensor, ov: int) -> torch.Tensor:\n \"\"\"\n 将两段音频 a 和 b 在尾部和头部以交叉渐变方式进行拼接，实现平滑过渡。\n 自动对齐声道：如果 a 是多声道但 b 单声道，自动复制 b 的声道数。\n \"\"\"\n print(\"xfade input shapes:\", a.shape, b.shape)\n # 1) 自动对齐声道\n if a.dim() == 2 and b.dim() == 2 and a.shape[0] != b.shape[0]:\n # 如果 b 只有 1 个通道，但 a 有多个通道，就复制 b\n if b.shape[0] == 1:\n b = b.repeat(a.shape[0], 1)\n # 反之亦然（极少用到）\n elif a.shape[0] == 1:\n a = a.repeat(b.shape[0], 1)\n\n # 2) 计算 overlap 点数\n n = ov * self.SR\n\n # 3) 构造渐变权重\n w = torch.linspace(0, 1, n, device=a.device).sqrt()\n\n # 4) 交叉区混合\n left = a[..., :-n]\n middle = a[..., -n:] * (1 - w) + b[..., :n] * w\n right = b[..., n:]\n\n # 5) 拼接并返回\n return torch.cat([left, middle, right], dim=-1)\n\n # ====================== generate ======================\n @torch.no_grad()\n def generate(\n self,\n seed_path: str,\n style: str,\n user_prompt: str,\n total_sec: int = 120,\n window_sec: int = 20,\n overlap_sec: int = 5,\n prompt_sec: int = 5,\n safety_tail: int = 3,\n n_tracks: int = 1,\n clap_candidates: int = 3,\n sim_thresh: float = 0.5,\n sil_thresh: float = -40,\n melody_path: Optional[str] = None,\n out_path: Optional[str] = None,\n use_fp16: bool = False,\n ) -> Union[str, List[str]]:\n \"\"\"\n 基于种子音频和用户提示，分段、生成并拼接完整音乐轨道。\n\n 整体流程：\n 1. 读取 seed_path 指定的音频文件并重采样、提取 CLAP 嵌入；\n 2. （可选）读取 melody_path 指定的条件音频；\n 3. 调用 request_segment_plan 获取分段规划；\n 4. 对每个段落并行生成 clap_candidates 条候选，并按相似度选出最佳；\n 5. 通过 xfade 平滑拼接到当前音轨；\n 6. 重复直到累计生成达到 total_sec，并保存最终音轨。\n\n 参数:\n seed_path (str):\n 初始音频文件路径，用于续写输入。\n style (str):\n 分段风格名称，对应内部 STYLE_STRUCTURE_LIB 的键。\n user_prompt (str):\n 用户文本提示，用于精细化每段的生成风格。\n total_sec (int, 可选):\n 目标总时长（秒）。默认 120。\n window_sec (int, 可选):\n 每次生成的片段时长（秒）。默认 20。\n overlap_sec (int, 可选):\n 段落重叠长度（秒），用于淡入淡出拼接。默认 5。\n prompt_sec (int, 可选):\n 每次构建生成 prompt 时要截取的尾部时长（秒）。默认 5。\n safety_tail (int, 可选):\n 生成 prompt 前保留的不参与提示的尾巴长度（秒），避免拼接突变。默认 3。\n n_tracks (int, 可选):\n 并行生成的轨道数量；>1 时返回路径列表。默认 1。\n clap_candidates (int, 可选):\n 每段并行生成的候选数，用于多版本筛选。默认 3。\n sim_thresh (float, 可选):\n 最优候选的相似度阈值；低于则触发重试或跳过。默认 0.5。\n sil_thresh (float, 可选):\n 静音检测阈值（dB）；可用于剔除无声片段。默认 -40。\n melody_path (Optional[str], 可选):\n 条件 melody 音频文件路径；不提供则进入纯生成模式。默认 None。\n out_path (Optional[str], 可选):\n 最终输出文件路径；多轨时自动编号。默认 None。\n use_fp16 (bool, 可选):\n 在 CUDA 上是否启用半精度推理（torch.cuda.amp）。默认 False.\n\n 返回:\n Union[str, List[str]]:\n 生成的音频文件路径。若 n_tracks>1 则为路径列表，否则为单个字符串。\n\n 异常:\n ValueError:\n - 当分段规划（request_segment_plan）失败且无法解析时。\n - 当读取音频文件或网络请求出错并超过重试次数时。\n\n 示例:\n >>> out = svc.generate(\n ... seed_path=\"seed.wav\",\n ... style=\"ambient\",\n ... user_prompt=\"dreamy pad with rain\",\n ... total_sec=180,\n ... n_tracks=2,\n ... clap_candidates=5,\n ... use_fp16=True,\n ... out_path=\"output/track.wav\"\n ... )\n \"\"\"\n\n print(f\"[Generate] start: n_tracks={n_tracks}, clap_candidates={clap_candidates}, use_fp16={use_fp16}, device={self.device}\", flush=True)\n\n fp16_ctx = torch.cuda.amp.autocast if (use_fp16 and self.device.startswith(\"cuda\")) else nullcontext\n\n def _run_once(idx: int) -> str:\n checkpoint_path= self.checkpoint_path\n USE_BF16 = torch.cuda.is_bf16_supported()\n DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n ENABLE_COMPILE = True # False 可快速回退\n STRICT_LOAD = False\n\n print(f\"[Track {idx+1}/{n_tracks}] >>> begin _run_once\", flush=True)\n torch.manual_seed(42 + idx)\n\n # 1) 载入 seed 音频\n seed32, sr = torchaudio.load(seed_path)\n if sr != self.SR:\n seed32 = torchaudio.functional.resample(seed32, sr, self.SR)\n seed32 = seed32.to(self.device)\n ref_emb = self.embed_cpu(seed32)\n \n # 2) 段落规划\n segs = self.request_segment_plan(style, user_prompt, total_sec, window_sec)\n need = (window_sec - overlap_sec) * self.SR\n full = seed32[..., :need] if seed32.shape[-1] >= need else seed32.repeat(\n 1, math.ceil(need / seed32.shape[-1])\n )[..., :need]\n cur_seg = 0\n\n # 3) 每段生成参数设置\n def set_params(idx_seg):\n p = segs[idx_seg]\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,#p[\"temperature\"],\n top_p=0.9,#p[\"top_p\"],\n top_k=250,#p[\"cfg\"],\n #use_sampling=True,\n )\n\n set_params(cur_seg)\n retry_count = 0\n max_retries = 6\n\n # 4) 主循环：续写直到达到 total_sec\n while full.shape[-1] / self.SR < total_sec:\n # 切换到下一个段落\n if full.shape[-1] / self.SR >= segs[cur_seg][\"end_at\"] and cur_seg < len(segs) - 1:\n cur_seg += 1\n set_params(cur_seg)\n retry_count = 0\n if self.verbose:\n print(f\"▶ Enter segment {cur_seg}: {segs[cur_seg]['section']}\")\n \n\n # 准备 prompt 音频片段\n tail = full.shape[-1] - safety_tail * self.SR\n head = max(0, tail - prompt_sec * self.SR)\n pa = full[..., head:tail]\n if pa.shape[-1] < 1024:\n pa = torch.nn.functional.pad(pa, (1024 - pa.shape[-1], 0), \"reflect\")\n pa = pa.unsqueeze(0)\n\n # 并行多候选生成\n current_prompt = segs[cur_seg][\"prompt\"]\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] generating {clap_candidates} candidates\", flush=True)\n descriptions = [current_prompt] * clap_candidates\n \n\n with fp16_ctx():\n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # —— 使用 generate_continuation 接口 —— \n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n # 续写：使用之前 set_generation_params 设置的 duration\n wavs = self.model.generate_continuation(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n progress=self.verbose,\n )\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] got {len(wavs)} wavs\", flush=True)\n\n # 5) 选出相似度最高的 wav，同时收集所有 sim 值\n sim_list = []\n best, best_sim = None, -1.0\n \n def _ensure_mono32(t: torch.Tensor, sr: int = 32000) -> torch.Tensor:\n \"\"\"\n 保证张量变成 [1, T] 单声道并且已是 32 kHz。\n 这里假设外层音频都已重采样到 self.SR=32 kHz；仅做声道合并。\n \"\"\"\n if t.dim() == 2 and t.shape[0] > 1: # 立体声 → 单声道\n t = t.mean(0, keepdim=True)\n return t.to(self.device)\n\n for i, wav in enumerate(wavs):\n # ------------- 1) 规格统一 -------------\n wav_mono = _ensure_mono32(wav) # 生成片段\n ref_tail = _ensure_mono32(pa.squeeze(0)) # prompt tail\n # 时间对齐：裁到同长（取短的一方）\n T = min(wav_mono.shape[-1], ref_tail.shape[-1])\n wav_mono = wav_mono[..., :T]\n ref_tail = ref_tail[..., :T]\n\n # ------------- 2) 计算三项相似度 ----------\n sim_clap = torch.cosine_similarity(\n self.embed_cpu(wav_mono), self.embed_cpu(ref_tail)\n ).item()\n\n sdr_db = self.compute_si_sdr(ref_tail, wav_mono).mean().item() # dB\n mel_l2 = self.mel_distance(ref_tail, wav_mono) # L2\n\n # 映射到 0–1 区间\n sdr_norm = min(max((sdr_db - 0) / 10.0, 0.0), 1.0) # 0 dB→0, 10 dB→1\n mel_norm = min(250.0 / (mel_l2 + 1.0), 1.0) # 250→1, 500→0.5\n\n # 综合得分\n sim_final = 0.4 * sim_clap + 0.4 * sdr_norm + 0.2 * mel_norm\n\n # ------------- 3) 调试输出 --------------\n if self.verbose:\n print(\n f\"CLAP={sim_clap:.3f} \"\n f\"SDR={sdr_db:.2f}dB(n={sdr_norm:.2f}) \"\n f\"MelL2={mel_l2:.0f}(n={mel_norm:.2f}) \"\n f\"final={sim_final:.3f}\"\n )\n\n # ------------- 4) 选最佳 ----------------\n sim_list.append(sim_final)\n if sim_final > best_sim:\n best, best_sim = wav_mono.cpu(), sim_final\n del wav\n\n\n torch.cuda.empty_cache()\n\n # 打印这一段所有候选的 sim 数组和选出的 best_sim\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] sims = {sim_list}\")\n print(f\"[Track {idx+1} Segment {cur_seg}] best_sim = {best_sim:.4f}\")\n\n # save_best_each = True\n save_best_each = False\n # 立刻保存本段 best 候选 —— \n if save_best_each and out_path:\n # out_path 父目录下创建 bestwav 文件夹\n best_dir = pathlib.Path(out_path).parent / \"bestwav\"\n best_dir.mkdir(parents=True, exist_ok=True)\n save_name = best_dir / f\"Segment_{cur_seg}.wav\"\n\n # 生成子文件名，并写文件\n # 生成微调结果\n # 把 float16 转成 float32，再保存\n \n audio_ft = best.cpu().float().numpy()\n if audio_ft.ndim == 2 and audio_ft.shape[0] < audio_ft.shape[1]:\n audio_ft = audio_ft.T\n sf.write(str(save_name), audio_ft, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1} Seg {cur_seg}] saved best wav → {save_name}\")\n\n # 6) 相似度太低时重试或跳过\n if best_sim < sim_thresh:\n retry_count += 1\n if retry_count > max_retries:\n if self.verbose:\n print(f\"Segment {cur_seg} reached max retries, skipping segment\")\n retry_count = 0\n break\n p = segs[cur_seg]\n new_temp = max(0.15, p[\"temperature\"] * (0.9 ** retry_count))\n new_top_k = p[\"top_k\"] * (1.15 ** retry_count)\n new_top_p = min(p[\"top_p\"]+0.05,0.95)\n self.model.set_generation_params(\n duration=window_sec,\n temperature=new_temp,\n top_k=new_top_k,\n top_p = new_top_p,\n use_sampling=True,\n )\n if self.verbose:\n print(f\"Retry {retry_count}: temp→{new_temp:.2f}, topk→{new_top_k:.2f}\")\n continue\n\n # 7) 拼接并输出调试信息\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] before xfade: {full.shape[-1]/self.SR:.1f}s\")\n full = self.xfade(full, best.to(self.device), overlap_sec)\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] after xfade: {full.shape[-1]/self.SR:.1f}s\")\n del best\n torch.cuda.empty_cache()\n retry_count = 0\n\n # 完成 single-track 生成，导出文件\n final = full[..., : total_sec * self.SR].cpu().T\n if out_path:\n stem, ext = os.path.splitext(out_path)\n out_file = f\"{stem}_{idx+1:02d}{ext}\" if n_tracks > 1 else out_path\n else:\n out_file = f\"final_{idx+1:02d}.wav\" if n_tracks > 1 else \"final.wav\"\n\n pathlib.Path(out_file).parent.mkdir(parents=True, exist_ok=True)\n sf.write(out_file, final, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1}] saved {out_file}, best_sim={best_sim:.4f}\", flush=True)\n return out_file\n\n # 并行生成多条 track\n print(f\"[Generate] launching _run_once for {n_tracks} tracks\", flush=True)\n paths = [_run_once(i) for i in range(n_tracks)]\n print(f\"[Generate] all done, paths = {paths}\", flush=True)\n return paths if n_tracks > 1 else paths[0]\n\n \n\n\ndef generate_music_api(params: Dict) -> Dict:\n # svc = MusicGenService(\n # api_endpoint=params.get('api_endpoint', API_ENDPOINT),\n # openai_api_key=params.get('api_key', OPENAI_API_KEY),\n # openai_model=params.get('model', OPENAI_MODEL),\n # verbose=params.get('verbose', True),\n # checkpoint_path=params.get('checkpoint_path'), # ← 把它传进去\n # )\n svc = MusicGenService(\n use_local_llm=params.get('use_local_llm', DEFAULT_USE_LOCAL_LLM),\n api_endpoint=params.get('api_endpoint'), # 允许外部自定义\n openai_api_key=params.get('api_key', OPENAI_API_KEY),\n openai_model=params.get('model', OPENAI_MODEL),\n verbose=params.get('verbose', True),\n checkpoint_path=params.get('checkpoint_path'),\n )\n out = svc.generate(\n seed_path=params['seed_path'],\n style=params['style'],\n user_prompt=params['user_prompt'],\n total_sec=params.get('total_sec', 120),\n window_sec=params.get('window_sec', 20),\n overlap_sec=params.get('overlap_sec', 5),\n prompt_sec=params.get('prompt_sec', 5),\n safety_tail=params.get('safety_tail', 3),\n clap_candidates= 1,#params.get('clap_candidates', 3),\n n_tracks=params.get('n_tracks', 1), # ← 新增这一行\n use_fp16=params.get('use_fp16', False), # ← 以及这一行\n sim_thresh=params.get('sim_thresh', 0.5),\n sil_thresh=params.get('sil_thresh', -40),\n melody_path=params.get('melody_path'),\n out_path=params.get('out_path'),\n )\n return {'out_path': out}\n\n"}}, "structure_gen/.ipynb_checkpoints/musicgen_api-checkpoint.py": {"486bab9207d6499336bfe192c2ed40eb91b6c4b0": {"t": 1785768923.4994562, "n": 45729, "h": "7b4aeae1961a1f29", "c": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\nmusicgen_api.py\n\n封装 MusicGen 无限续写脚本为后端可调用接口。\n支持 Melody 条件生成、增量续写、重试与显存管理。\n\"\"\"\n\nnetworks = False # 如果没有接入国际互联网，就使用 False\n\n\nimport os\n# 如果没有接入国际互联网，就将 Huggingface 的访问设置为【不访问】\nif not networks:\n # 确保 transformers/hub 都用离线模式\n os.environ[\"HF_HUB_OFFLINE\"] = \"1\"\n os.environ[\"TRANSFORMERS_OFFLINE\"] = \"1\"\n # 然后 monkey-patch BertTokenizer.from_pretrained，让它只走本地缓存\n from transformers import BertTokenizer\n _original_bp = BertTokenizer.from_pretrained\n def _from_pretrained_local(repo_id, *args, **kwargs):\n # 强制 local_files_only=True\n return _original_bp(repo_id, local_files_only=True, *args, **kwargs)\n BertTokenizer.from_pretrained = _from_pretrained_local\n\n\nimport json\nimport math\nimport warnings\nimport re\nimport time\nimport ollama\nimport torchaudio\nimport torch\nimport requests\nimport pathlib\nimport soundfile as sf\n\nfrom contextlib import nullcontext\nfrom typing import List, Optional, Union, Dict\nfrom urllib.parse import urljoin\nfrom audiocraft.models import MusicGen\nfrom laion_clap import CLAP_Module\n# from structure_gen.music_structure import STYLE_STRUCTURE_LIB # lin\nfrom music_structure import STYLE_STRUCTURE_LIB # yuan\n\n# ========== 默认配置==========\n\nDEFAULT_USE_LOCAL_LLM = False # 默认是否走本地 LLM（Ollama）\nOPENAI_MODEL = 'gpt-4o-mini' # 远程 LLM 模型名\nmax_retries = 6 # 远程 LLM 最大重试次数\n\nOPENAI_API_KEY = 'sk-vWyNj5NxixWjJcjknaxNivrtIkHIjXDBVlk1b6wRW2pZc0hz' # 网络LLM API KEY\n\n\nUSE_BF16 = torch.cuda.is_bf16_supported() # \nDTYPE = torch.bfloat16 if USE_BF16 else torch.float16 \n\nENABLE_COMPILE = True # False 可快速回退\nSTRICT_LOAD = False\n\n\n# =========================================\n\n\n\n\ndef cast_musicgen_dtype(mg, dtype):\n \"\"\"把 MusicGen 内所有子模块转成半精度 / 全精度\"\"\"\n mg.lm = mg.lm.to(dtype=dtype)\n if hasattr(mg, \"decoder\"):\n mg.decoder = mg.decoder.to(dtype=dtype)\n if hasattr(mg, \"condition_provider\"):\n mg.condition_provider = mg.condition_provider.to(dtype=dtype)\n return mg\n\n\nclass MusicGenService:\n def __init__(\n self,\n *,\n use_local_llm: bool = DEFAULT_USE_LOCAL_LLM,\n api_endpoint: Optional[str] = None,\n openai_api_key: str = OPENAI_API_KEY,\n openai_model: str = OPENAI_MODEL,\n verbose: bool = True,\n device: Optional[str] = None,\n checkpoint_path: Optional[str] = None,\n use_fp16: bool = True,\n ):\n \n # —— 与 LLM 相关的配置 ——\n self.use_local_llm = use_local_llm\n if api_endpoint is None:\n # 如果调用方没指定，按 use_local_llm 选默认地址\n self.api_endpoint = (\n \"http://localhost:11434/api/generate\"\n if self.use_local_llm\n else \"https://www.dmxapi.com/v1/chat/completions\"\n )\n else:\n self.api_endpoint = api_endpoint\n # 参数配置\n self.openai_api_key = openai_api_key\n self.openai_model = openai_model\n self.max_retries = max_retries\n self.verbose = verbose\n self.device = device or (\"cuda\" if torch.cuda.is_available() else \"cpu\")\n self.use_fp16 = use_fp16\n self.clap = CLAP_Module(enable_fusion=False, device=self.device).eval()\n self.SR = 32000\n self.mel_transform = torchaudio.transforms.MelSpectrogram(\n sample_rate=self.SR, n_mels=80, hop_length=512\n ).to(self.device)\n self.checkpoint_path = checkpoint_path\n\n\n # 1) 加载 small 版本\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}...\")\n # # 检查audiocraft版本\n # try:\n # import audiocraft\n # print(f\"[Init] audiocraft版本: {audiocraft.__version__}\")\n # except:\n # print(f\"[Init] 无法获取audiocraft版本\")\n \n self.model = MusicGen.get_pretrained(\"facebook/musicgen-small\", self.device)\n\n # # 添加模型信息调试\n # if self.verbose:\n # print(f\"[Init] MusicGen模型信息:\")\n # print(f\" - 模型类型: {type(self.model)}\")\n # print(f\" - sample_rate: {getattr(self.model, 'sample_rate', 'None')}\")\n # print(f\" - 压缩模型sample_rate: {getattr(self.model.compression_model, 'sample_rate', 'None')}\")\n # print(f\" - 默认duration: {getattr(self.model, 'duration', 'None')}\")\n # if hasattr(self.model, 'lm'):\n # print(f\" - LM模型: {type(self.model.lm)}\")\n # if hasattr(self.model, 'compression_model'):\n # print(f\" - 压缩模型: {type(self.model.compression_model)}\")\n\n # 设置参数\n self.model.set_generation_params(\n duration=30,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n \n # # 测试基本generation功能\n # if self.verbose:\n # print(f\"[Init] 测试基本generation功能...\")\n # try:\n # # 测试generate方法\n # test_wav = self.model.generate(['test'], progress=False)\n # if len(test_wav) > 0:\n # print(f\" - generate测试成功，生成长度: {test_wav[0].shape[-1]/32000:.1f}s\")\n # else:\n # print(f\" - ⚠️ generate测试失败，返回空结果\")\n # \n # # 测试generate_continuation方法（使用简单prompt）\n # simple_prompt = torch.zeros(1, 1, 16000).to(self.device) # 0.5秒的空白音频\n # test_cont = self.model.generate_continuation(\n # prompt=simple_prompt,\n # prompt_sample_rate=32000,\n # descriptions=['test'],\n # progress=False\n # )\n # if len(test_cont) > 0:\n # print(f\" - generate_continuation测试成功，生成长度: {test_cont[0].shape[-1]/32000:.1f}s\")\n # print(f\" - prompt长度: {simple_prompt.shape[-1]/32000:.1f}s\")\n # print(f\" - 续写比例: {test_cont[0].shape[-1]/simple_prompt.shape[-1]:.1f}x\")\n # else:\n # print(f\" - ⚠️ generate_continuation测试失败，返回空结果\")\n # except Exception as e:\n # print(f\" - ⚠️ 测试失败: {e}\")\n # import traceback\n # traceback.print_exc()\n\n # 2) 如果指定了 checkpoint，就加载权重\n if checkpoint_path:\n if self.verbose:\n print(f\"[Init] Loading checkpoint from {checkpoint_path}…\")\n ckpt = torch.load(checkpoint_path, map_location=\"cpu\")\n state_dict = ckpt.get(\"model_state_dict\", ckpt)\n missing, unexpected = self.model.lm.load_state_dict(state_dict, strict=True)\n if self.verbose:\n print(f\"[Init] Checkpoint loaded. missing keys: {len(missing)}, unexpected: {len(unexpected)}\")\n\n # 3) 半精度转换 & 编译\n self.model = cast_musicgen_dtype(self.model, DTYPE)\n print(\"Model dtype:\", next(self.model.lm.parameters()).dtype)\n\n # 编译 lm 子模块以减少 Python 调度开销\n try:\n self.model.lm = torch.compile(self.model.lm,mode=\"reduce-overhead\",fullgraph=True)\n # if self.verbose:\n # print(\"[Init] 🚀 torch.compile() 已启用\")\n except Exception as e:\n if self.verbose:\n print(f\"[Init] ⚠️ torch.compile 失败，回退: {e}\")\n\n if self.verbose:\n print(f\"[Init] Loading MusicGen on {self.device}... OK\")\n\n\n @torch.no_grad()\n def embed_cpu(self, wav32: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 将输入的多通道高采样率音频数据转换为单通道 16kHz，\n 并用 CLAP 模型提取其在 CPU 上的特征嵌入表示，\n 用于后续相似度计算或音频理解任务。\n \"\"\"\n mono = wav32.to(\"cpu\", torch.float32).mean(0, keepdim=True)\n m16 = torchaudio.functional.resample(mono, self.SR, 16000)\n return self.clap.get_audio_embedding_from_data(m16, use_tensor=True).cpu()\n\n \n def request_segment_plan(\n self, style: str, user_prompt: str, total_sec: int, window_sec: int\n ) -> List[Dict]:\n \"\"\"\n 根据用户提供的风格、提示词和时长设置，调用本地或远程 ChatGPT 接口生成音乐分段规划。\n\n 该方法会构造一个详细的自然语言提示（prompt），并附带 CLAP 支持的 token 列表，提交给语言模型，\n 请求生成一个包含多个段落的 JSON 列表。每个段落包含音乐结构（如 intro、verse 等）、持续时间、\n 生成提示词以及采样参数（temperature、top_p、top_k）等信息。返回结果会自动补齐字段并计算结束时间戳。\n\n 如果调用失败，方法将退回使用内置的默认分段方案。\n\n 参数:\n style (str): 音乐结构风格名，对应 STYLE_STRUCTURE_LIB 中的 key。\n user_prompt (str): 用户自定义的风格描述文本，将传入 LLM 构造提示。\n total_sec (int): 希望生成的总时长（秒），提示中允许略超，但不得不足。\n window_sec (int): 每个段落的时间粒度，必须是段落时长的倍数。\n\n 返回:\n List[Dict]: 每个元素为一个分段字典，包含字段：\n - section: 段名称\n - duration: 时长（秒）\n - prompt: 用于音乐生成的提示词\n - temperature, top_p, top_k: 控制采样的超参数\n - end_at: 当前段结束时刻（累加）\n\n 异常:\n ValueError: 若 style 无法匹配结构库、或响应无法被正确解析时抛出。\n\n 注意:\n - 默认使用远程 OpenAI API，也支持通过 `use_local_llm` 使用 Ollama 模型。\n - 方法内部自动进行重试机制，并打印失败原因。\n - 所有生成内容中的 prompt 字段将仅使用提供的 token 列表，保证下游模型兼容性。\n \"\"\"\n structure = STYLE_STRUCTURE_LIB.get(style.lower())\n if not structure:\n raise ValueError(f\"风格 '{style}' 不支持\")\n structure = \", \".join(structure)\n if self.verbose:\n print(\"structure is:\"+ structure)\n\n\n fluent_prompt = f\"\"\"You are an experienced music producer. \n Please generate a JSON array of song segments where each item has the following keys: \n - section (string) \n - duration (int, seconds) \n - prompt (string) \n - temperature (float) \n - top_p (float) \n - top_k (float) \n \n Return only the raw JSON array without any markdown fences.\n \n Description is: {user_prompt}, Style is: {style}.\n The total duration should be about {total_sec} seconds (you may exceed but not fall short).\n Use these sections in order: {structure}.\n Each segment’s duration must be a multiple of {window_sec} seconds (e.g. if window_sec = 30, valid values are 30, 60, 90, …).\n top_p must be between 0.6 and 0.9, top_k should be about 250, temperature should be higher than 1.0.\n Construct the prompt field for each segment depending on description {user_prompt} and style {style}.\n When constructing the prompt field inside each segment, use ONLY the following tokens, organized by category. \n Ensuring that any Genre / Style token you choose exactly matches the value in {style}:\n \n Structure: \n intro-like, verse-like, pre-chorus-like, chorus-like, bridge-like, build-up-like, drop-like, interlude-like, outro-like.\n \n Mood / Emotion: \n calming, soothing, serene, ethereal, dreamy, happy, joyful, cheerful, gentle, groovy, funky, sad, melancholic, nostalgic, mournful, peaceful, uplifting, hopeful, inspirational, epic, triumphant, energetic, aggressive, angry, dark, mysterious, spooky, tense, suspenseful, anxious, majestic, glorious, romantic, playful, whimsical.\n \n Genre / Style: \n {style}.\n \n Instrument / Voice: \n acoustic guitar, electric guitar, classical guitar, bass guitar, electric bass, acoustic bass, ukulele, banjo, mandolin, piano, electric piano, upright piano, harpsichord, organ, pipe organ, clavinet, synthesizer, bass synth, pad, lead synth, violin, viola, cello, double bass, string ensemble, orchestral strings, flute, piccolo, recorder, pan flute, clarinet, bass clarinet, oboe, english horn, bassoon, contrabassoon, saxophone, soprano saxophone, alto saxophone, tenor saxophone, baritone saxophone, trumpet, piccolo trumpet, cornet, flugelhorn, trombone, bass trombone, tuba, french horn, euphonium, harmonica, accordion, concertina, bagpipe, sitar, shamisen, koto, erhu, pipa, dizi, ocarina, didgeridoo, marimba, xylophone, vibraphone, glockenspiel, celesta, steel drum, drums, acoustic drum kit, electronic drum kit, snare drum, bass drum, cymbals, hi-hat, tom-tom, timpani, congas, bongo, djembe, tabla, triangle, tambourine, shaker, clap, percussion, male vocals, female vocals, mixed choir, children choir, solo voice.\n \n Environment: \n rain, heavy rain, thunder, wind, snowstorm, ocean waves, seashore, underwater ambience, forest birds, rainforest, desert wind, city daytime, city night, traffic noise, street ambience, airport terminal, subway station, train station, bus station, marketplace, stadium crowd, children playground, office background, classroom, library, crowd murmur, cafe background, industrial, construction noise, fireplace crackle, fireworks, police siren, space ambience, windchimes.\n \n Production Style: \n lo-fi, hi-fi, vintage, analog, digital, electronic, acoustic, orchestral, cinematic, soundtrack, psychedelic, experimental, noise, glitch, ambient, warm, bright, dark, harsh, smooth, compressed, bassy, spatial, surround, stereo wide, monophonic, polyphonic, minimal, maximalist, bitcrushed, chiptune, distorted, reverb-heavy, gated reverb, side-chained, auto-tuned.\n \n Energy / Tempo: \n high energy, powerful, relaxed, calm, dynamic, steady, variable, driving, lively, euphoric, hypnotic, pulsing, accelerando, decelerando.\n \n Chord / Harmony: \n C major, C minor, C7, Cmaj7, Cmin7, A major, A minor, A7, Amaj7, Amin7, Bb major, Bb minor, Bb7, Bbmaj7, B major, B minor, B7, Bmaj7, Bmin7, D major, D minor, D7, Dmaj7, Dmin7, E major, E minor, E7, Emaj7, Emin7, F major, F minor, F7, Fmaj7, Fmin7, F# major, F# minor, F#7, F#maj7, F#min7, G major, G minor, G7, Gmaj7, Gmin7, Ab major, Ab minor, Ab7, Abmaj7, Abmin7.\n\n CLAP generate Harmony:\n C, C#, D, D#, E, F, F#, G, G#, A, A#, B\n \n Example of two valid prompt strings for one segment: \n build-up-like, serene, delta blues, acoustic guitar, windchimes, soundtrack, accelerando, C minor, G \n interlude-like, suspenseful, neo soul, orchestral strings, space ambience, soundtrack, dynamic, C minor, G#.\n \n Return only the JSON array.\n \"\"\"\n\n # ambient, dark ambient, ambient techno, new age, chillout, lofi hip hop, trip hop, future bass, synthwave, vaporwave, house, deep house, progressive house, tech house, techno, psytrance, trance, drum and bass, jungle, dubstep, brostep, trap, EDM, jazz, smooth jazz, swing, bebop, latin jazz, bossa nova, samba, fusion, classical, baroque, romantic, modern classical, contemporary classical, orchestral, opera, chamber music, rock, classic rock, hard rock, punk rock, post-punk, indie rock, alternative rock, progressive rock, shoegaze, grunge, metal, heavy metal, death metal, black metal, thrash metal, pop, synth pop, indie pop, electropop, k-pop, j-pop, R&B, neo soul, soul, funk, disco, gospel, hip hop, rap, boom bap, trap rap, blues, delta blues, reggae, dub, ska, folk, bluegrass, country, americana, celtic, afrobeat, salsa, tango, flamenco, balkan, bollywood, world music, soundtrack, cinematic, video game, experimental, noise.\n \n raw = None\n for attempt in range(1, max_retries+1):\n try:\n if self.use_local_llm:\n # ➡️ 调 Ollama Chat —— \n resp = ollama.chat(\n model=\"deepseek-r1:14b\",\n messages=[{\"role\": \"user\", \"content\": fluent_prompt}]\n )\n \n # 直接从响应字典中提取文本\n try:\n text = resp[\"message\"][\"content\"]\n except Exception as e:\n raise ValueError(f\"⚠️ 无法解析 Ollama 返回结构: {e}\\n原始内容:\\n{resp}\")\n \n # 打印内容供 debug\n if self.verbose:\n print(\"the text is : \" + text)\n \n # 尝试提取 JSON 数组（包括 ```json ... ``` 或直接 [...]）\n m = re.search(r\"```json\\s*($begin:math:display$.*?$end:math:display$)\\s*```\", text, re.S)\n if not m:\n m = re.search(r\"\\[.*?\\]\", text, re.S)\n \n if not m:\n raise ValueError(f\"❌ 无法从文本中提取 JSON 数组:\\n{text}\")\n \n # 提取原始 JSON 内容\n raw = m.group(1) if m.lastindex else m.group(0)\n raw = raw.encode('utf-8').decode('unicode_escape')\n raw = raw.replace('\\\\\"', '\"')\n\n else:\n # ➡️ 调 LLM API\n payload = {\n \"model\": self.openai_model,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": fluent_prompt}\n ]\n }\n headers = {\n \"Authorization\": f\"Bearer {self.openai_api_key}\",\n \"Content-Type\": \"application/json\"\n }\n\n url = self.api_endpoint\n \n resp = requests.post(\n url, \n headers=headers, \n json=payload\n )\n\n resp.raise_for_status()\n raw = resp.json()[\"choices\"][0][\"message\"][\"content\"].strip()\n \n \n if self.verbose:\n print(\"☀️ the raw json is : \"+raw)\n print(f\"[SegPlan] 成功获得规划 (attempt {attempt})\")\n break\n \n # 异常处理与多次重试 \n except requests.RequestException as e:\n if attempt < max_retries:\n wait = min(2 ** (attempt-1) , 60)\n if self.verbose:\n print(f\"[SegPlan] 请求失败，第{attempt}次重试，{wait}s后再试… ({e})\")\n time.sleep(wait)\n else:\n if self.verbose:\n print(f\"[SegPlan] 重试{max_retries}次失败，使用默认分段（❌没有完成，寄）\")\n # 默认回退\n if not raw:\n # 请求失败，使用固定默认分段列表，并继续后续解析流程\n if self.verbose:\n print(\"[SegPlan] 请求多次失败，使用固定默认分段列表\")\n default_segs = [\n {'section': 'minimal-pulse', 'duration': 30, 'prompt': 'intro-like, peaceful, ambient, bass synth, pad, warm, reverb-heavy, pulsing, Cmaj7, C', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'arpeggio-loop', 'duration': 30, 'prompt': 'verse-like, calming, ambient, synthesizer, pad, warm, smooth, slow, Amin7, A', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 8.0},\n {'section': 'white-noise-bed', 'duration': 30, 'prompt': 'pre-chorus-like, soothing, ambient, pad, space ambience, warm, reverb-heavy, slow, Dmin7, D', 'temperature': 0.65, 'top_p': 0.75, 'cfg': 6.5},\n {'section': 'modulation-bridge', 'duration': 30, 'prompt': 'bridge-like, ethereal, ambient, string ensemble, pad, warm, reverb-heavy, slow, Gmaj7, G', 'temperature': 0.7, 'top_p': 0.8, 'cfg': 7.5},\n {'section': 'repetitive-theme', 'duration': 30, 'prompt': 'chorus-like, dreamy, ambient, string ensemble, pad, warm, smooth, slow, Fmaj7, F', 'temperature': 0.6, 'top_p': 0.7, 'cfg': 7.0},\n {'section': 'soft-ending', 'duration': 30, 'prompt': 'outro-like, serene, ambient, pad, string ensemble, warm, smooth, decelerando, Cmaj7, C', 'temperature': 0.5, 'top_p': 0.6, 'cfg': 6.0},\n ]\n # 用默认列表生成 raw JSON，继续解析流程\n raw = json.dumps(default_segs, ensure_ascii=False)\n \n # —— 解析并补齐字段 —— 如果 LLM 给的内容有缺损，使用默认的参数补齐它\n segs = json.loads(raw)\n acc = 0\n for s in segs:\n s[\"duration\"] = int(s.get(\"duration\", window_sec))\n s[\"prompt\"] = s.get(\"prompt\", user_prompt)\n s[\"temperature\"] = float(s.get(\"temperature\", 0.7))\n s[\"top_p\"] = float(s.get(\"top_p\", 0.9))\n s[\"top_k\"] = float(s.get(\"top_k\", 250))\n acc += s[\"duration\"]\n s[\"end_at\"] = acc\n if self.verbose:\n print(\"[SegPlan] Parsed segments:\")\n for x in segs: print(x)\n\n return segs\n\n def compute_si_sdr(self, ref: torch.Tensor, est: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算 Scale-Invariant Signal-to-Distortion Ratio（SI-SDR），用于评估估计音频与参考音频之间的相似度。\n\n SI-SDR 是一种用于盲音源分离和音频质量评价的指标，具备尺度不变性，\n 可以排除音量（能量）缩放对相似度评分的影响。\n\n 参数:\n ref (torch.Tensor): 参考音频信号，形状为 [C, T] 或 [T]（C 为通道数，T 为时间长度）。\n est (torch.Tensor): 估计音频信号，形状与 ref 相同或兼容。\n eps (float): 避免除零错误的微小常数，默认值为 1e-8。\n\n 返回:\n torch.Tensor: 每个样本/通道的 SI-SDR 得分（以 dB 为单位），形状为 [C] 或标量。\n \"\"\"\n # 对齐 ref 和 est 的时间长度，支持单通道或多通道输入\n if ref.dim() == 2 and est.dim() == 2:\n T = min(ref.shape[1], est.shape[1])\n ref, est = ref[:, :T], est[:, :T]\n else:\n T = min(ref.shape[-1], est.shape[-1])\n ref, est = ref[:T], est[:T]\n\n # 去直流分量（零均值化）\n ref_zm = ref - ref.mean(-1, keepdim=True)\n est_zm = est - est.mean(-1, keepdim=True)\n\n # 计算参考信号能量\n ref_energy = torch.sum(ref_zm ** 2, dim=-1, keepdim=True)\n\n # 计算最佳缩放因子 scale，使得投影后的信号最接近估计信号\n scale = torch.sum(ref_zm * est_zm, dim=-1, keepdim=True) / (ref_energy + eps)\n\n # 将估计信号投影到参考信号方向\n proj = scale * ref_zm\n\n # 残差噪声 = 去中心化估计信号 - 投影部分\n noise = est_zm - proj\n\n # 计算信噪比（使用投影部分作为信号）\n snr = torch.sum(proj ** 2, dim=-1) / (torch.sum(noise ** 2, dim=-1) + eps)\n\n # 返回以 dB 为单位的 SI-SDR 分数\n return 10 * torch.log10(snr + eps)\n\n def rms_db(self, x: torch.Tensor) -> float:\n return 20*torch.log10(x.square().mean().sqrt()+1e-9).item()\n\n def mel_distance(self, x: torch.Tensor, y: torch.Tensor) -> float:\n \"\"\"\n 计算两个音频信号的梅尔谱（Mel Spectrogram）之间的 L2 距离，用于衡量音频在感知上的相似性。\n\n 此方法首先对两个音频信号做对齐裁剪，并通过 `self.mel_transform` 提取梅尔谱特征，\n 再计算其展开后的欧氏距离（L2 norm）。适用于感知相似度评估、音频生成对比等任务。\n\n 参数:\n x (torch.Tensor): 第一个音频信号，形状为 [T] 或 [C, T]。\n y (torch.Tensor): 第二个音频信号，形状与 x 相同或兼容。\n\n 返回:\n float: 梅尔谱之间的 L2 距离，数值越小表示音频越相似。\n \"\"\"\n # 对齐两个音频在时间维度上的长度\n if x.dim()==2 and y.dim()==2:\n T = min(x.shape[1], y.shape[1])\n x, y = x[:,:T], y[:,:T]\n else:\n T = min(x.shape[-1], y.shape[-1])\n x, y = x[:T], y[:T]\n\n # 通过 mel_transform 提取梅尔谱特征（添加 batch 维度）\n Mx = self.mel_transform(x.unsqueeze(0)) # [1, F, T']\n My = self.mel_transform(y.unsqueeze(0)) # [1, F, T']\n\n # 将梅尔谱差值展平后计算 L2 距离\n return torch.norm((Mx-My).reshape(-1), p=2).item()\n\n def xfade(self, a: torch.Tensor, b: torch.Tensor, ov: int) -> torch.Tensor:\n \"\"\"\n 将两段音频 a 和 b 在尾部和头部以交叉渐变方式进行拼接，实现平滑过渡。\n 自动对齐声道：如果 a 是多声道但 b 单声道，自动复制 b 的声道数。\n \"\"\"\n print(\"xfade input shapes:\", a.shape, b.shape)\n # 1) 自动对齐声道\n if a.dim() == 2 and b.dim() == 2 and a.shape[0] != b.shape[0]:\n # 如果 b 只有 1 个通道，但 a 有多个通道，就复制 b\n if b.shape[0] == 1:\n b = b.repeat(a.shape[0], 1)\n # 反之亦然（极少用到）\n elif a.shape[0] == 1:\n a = a.repeat(b.shape[0], 1)\n\n # 2) 计算 overlap 点数\n n = ov * self.SR\n\n # 3) 构造渐变权重\n w = torch.linspace(0, 1, n, device=a.device).sqrt()\n\n # 4) 交叉区混合\n left = a[..., :-n]\n middle = a[..., -n:] * (1 - w) + b[..., :n] * w\n right = b[..., n:]\n\n # 5) 拼接并返回\n return torch.cat([left, middle, right], dim=-1)\n\n # ====================== generate ======================\n @torch.no_grad()\n def generate(\n self,\n seed_path: str,\n style: str,\n user_prompt: str,\n total_sec: int = 120,\n window_sec: int = 20,\n overlap_sec: int = 5,\n prompt_sec: int = 5,\n safety_tail: int = 3,\n n_tracks: int = 1,\n clap_candidates: int = 3,\n sim_thresh: float = 0.5,\n sil_thresh: float = -40,\n melody_path: Optional[str] = None,\n out_path: Optional[str] = None,\n use_fp16: bool = False,\n ) -> Union[str, List[str]]:\n \"\"\"\n 基于种子音频和用户提示，分段、生成并拼接完整音乐轨道。\n\n 整体流程：\n 1. 读取 seed_path 指定的音频文件并重采样、提取 CLAP 嵌入；\n 2. （可选）读取 melody_path 指定的条件音频；\n 3. 调用 request_segment_plan 获取分段规划；\n 4. 对每个段落并行生成 clap_candidates 条候选，并按相似度选出最佳；\n 5. 通过 xfade 平滑拼接到当前音轨；\n 6. 重复直到累计生成达到 total_sec，并保存最终音轨。\n\n 参数:\n seed_path (str):\n 初始音频文件路径，用于续写输入。\n style (str):\n 分段风格名称，对应内部 STYLE_STRUCTURE_LIB 的键。\n user_prompt (str):\n 用户文本提示，用于精细化每段的生成风格。\n total_sec (int, 可选):\n 目标总时长（秒）。默认 120。\n window_sec (int, 可选):\n 每次生成的片段时长（秒）。默认 20。\n overlap_sec (int, 可选):\n 段落重叠长度（秒），用于淡入淡出拼接。默认 5。\n prompt_sec (int, 可选):\n 每次构建生成 prompt 时要截取的尾部时长（秒）。默认 5。\n safety_tail (int, 可选):\n 生成 prompt 前保留的不参与提示的尾巴长度（秒），避免拼接突变。默认 3。\n n_tracks (int, 可选):\n 并行生成的轨道数量；>1 时返回路径列表。默认 1。\n clap_candidates (int, 可选):\n 每段并行生成的候选数，用于多版本筛选。默认 3。\n sim_thresh (float, 可选):\n 最优候选的相似度阈值；低于则触发重试或跳过。默认 0.5。\n sil_thresh (float, 可选):\n 静音检测阈值（dB）；可用于剔除无声片段。默认 -40。\n melody_path (Optional[str], 可选):\n 条件 melody 音频文件路径；不提供则进入纯生成模式。默认 None。\n out_path (Optional[str], 可选):\n 最终输出文件路径；多轨时自动编号。默认 None。\n use_fp16 (bool, 可选):\n 在 CUDA 上是否启用半精度推理（torch.cuda.amp）。默认 False.\n\n 返回:\n Union[str, List[str]]:\n 生成的音频文件路径。若 n_tracks>1 则为路径列表，否则为单个字符串。\n\n 异常:\n ValueError:\n - 当分段规划（request_segment_plan）失败且无法解析时。\n - 当读取音频文件或网络请求出错并超过重试次数时。\n\n 示例:\n >>> out = svc.generate(\n ... seed_path=\"seed.wav\",\n ... style=\"ambient\",\n ... user_prompt=\"dreamy pad with rain\",\n ... total_sec=180,\n ... n_tracks=2,\n ... clap_candidates=5,\n ... use_fp16=True,\n ... out_path=\"output/track.wav\"\n ... )\n \"\"\"\n\n print(f\"[Generate] start: n_tracks={n_tracks}, clap_candidates={clap_candidates}, use_fp16={use_fp16}, device={self.device}\", flush=True)\n\n fp16_ctx = torch.cuda.amp.autocast if (use_fp16 and self.device.startswith(\"cuda\")) else nullcontext\n\n def _run_once(idx: int) -> str:\n checkpoint_path= self.checkpoint_path\n USE_BF16 = torch.cuda.is_bf16_supported()\n DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n ENABLE_COMPILE = True # False 可快速回退\n STRICT_LOAD = False\n\n print(f\"[Track {idx+1}/{n_tracks}] >>> begin _run_once\", flush=True)\n torch.manual_seed(42 + idx)\n\n # 1) 载入 seed 音频\n seed32, sr = torchaudio.load(seed_path)\n if sr != self.SR:\n seed32 = torchaudio.functional.resample(seed32, sr, self.SR)\n seed32 = seed32.to(self.device)\n ref_emb = self.embed_cpu(seed32)\n \n # 2) 段落规划\n segs = self.request_segment_plan(style, user_prompt, total_sec, window_sec)\n # 修正：初始full应该足够长以包含第一次生成的prompt\n # 需要至少 prompt_sec + safety_tail 的长度\n min_need = (prompt_sec + safety_tail) * self.SR\n need = max(min_need, (window_sec - overlap_sec) * self.SR)\n \n full = seed32[..., :need] if seed32.shape[-1] >= need else seed32.repeat(\n 1, math.ceil(need / seed32.shape[-1])\n )[..., :need]\n cur_seg = 0\n\n # 3) 每段生成参数设置\n def set_params(idx_seg):\n p = segs[idx_seg]\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,#p[\"temperature\"],\n top_p=0.9,#p[\"top_p\"],\n top_k=250,#p[\"cfg\"],\n #use_sampling=True,\n )\n\n set_params(cur_seg)\n retry_count = 0\n max_retries = 6\n\n # 4) 主循环：续写直到达到 total_sec\n # if self.verbose:\n # print(f\"[Debug] 初始full长度: {full.shape[-1]/self.SR:.1f}s, 目标: {total_sec}s\")\n # print(f\"[Debug] 第一个segment end_at: {segs[0]['end_at']}s\")\n \n while full.shape[-1] / self.SR < total_sec:\n # if self.verbose:\n # print(f\"[Debug] 进入生成循环，当前full: {full.shape[-1]/self.SR:.1f}s\")\n \n # 切换到下一个段落\n if full.shape[-1] / self.SR >= segs[cur_seg][\"end_at\"] and cur_seg < len(segs) - 1:\n cur_seg += 1\n set_params(cur_seg)\n retry_count = 0\n if self.verbose:\n print(f\"▶ Enter segment {cur_seg}: {segs[cur_seg]['section']}\")\n \n\n # 准备 prompt 音频片段\n tail = full.shape[-1] - safety_tail * self.SR\n head = max(0, tail - prompt_sec * self.SR)\n pa = full[..., head:tail]\n \n # if self.verbose:\n # print(f\"[Debug] prompt准备过程:\")\n # print(f\" - full总长度: {full.shape[-1]/self.SR:.1f}s\")\n # print(f\" - safety_tail: {safety_tail}s\")\n # print(f\" - prompt_sec: {prompt_sec}s\")\n # print(f\" - tail位置: {tail/self.SR:.1f}s\")\n # print(f\" - head位置: {head/self.SR:.1f}s\")\n # print(f\" - 提取的pa原始长度: {pa.shape[-1]/self.SR:.1f}s\")\n # print(f\" - pa shape: {pa.shape}\")\n \n if pa.shape[-1] < 1024:\n pa = torch.nn.functional.pad(pa, (1024 - pa.shape[-1], 0), \"reflect\")\n # if self.verbose:\n # print(f\" - padding后pa长度: {pa.shape[-1]/self.SR:.3f}s\")\n \n pa = pa.unsqueeze(0)\n \n # if self.verbose:\n # print(f\"[Debug] 最终prompt音频:\")\n # print(f\" - shape: {pa.shape}\")\n # print(f\" - 长度: {pa.shape[-1]/self.SR:.3f}s\")\n # print(f\" - dtype: {pa.dtype}\")\n # print(f\" - device: {pa.device}\")\n # print(f\" - 设置duration: {window_sec}s\")\n\n # 并行多候选生成\n current_prompt = segs[cur_seg][\"prompt\"]\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] generating {clap_candidates} candidates\", flush=True)\n descriptions = [current_prompt] * clap_candidates\n \n\n with fp16_ctx():\n if self.verbose:\n print(\"➡️ now in the gen mode\")\n torch.manual_seed(42) # 固定 seed\n \n # 添加详细的模型状态调试信息\n # if self.verbose:\n # print(f\"[Debug] 检查模型内部状态:\")\n # try:\n # print(f\" - self.model.duration: {getattr(self.model, 'duration', 'None')}\")\n # print(f\" - self.model.sample_rate: {getattr(self.model, 'sample_rate', 'None')}\")\n # if hasattr(self.model, 'generation_params'):\n # print(f\" - generation_params: {getattr(self.model, 'generation_params', 'None')}\")\n # if hasattr(self.model, '_gen_model') and self.model._gen_model:\n # gen_model = self.model._gen_model\n # print(f\" - _gen_model类型: {type(gen_model)}\")\n # if hasattr(gen_model, 'max_duration'):\n # print(f\" - _gen_model.max_duration: {gen_model.max_duration}\")\n # except Exception as e:\n # print(f\" - 检查模型状态时出错: {e}\")\n \n # 重新确认generation参数 - 使用更明确的方式\n try:\n self.model.set_generation_params(\n duration=window_sec,\n temperature=1.0,\n top_p=0.9,\n top_k=250\n )\n # if self.verbose:\n # print(f\"[Debug] ✅ 成功设置generation_params，duration={window_sec}\")\n except Exception as e:\n # if self.verbose:\n # print(f\"[Debug] ❌ 设置generation_params失败: {e}\")\n pass\n \n # 验证设置后的参数\n # if self.verbose:\n # print(f\"[Debug] 设置后验证:\")\n # try:\n # print(f\" - self.model.duration: {getattr(self.model, 'duration', 'None')}\")\n # if hasattr(self.model, 'compression_model'):\n # print(f\" - compression_model.sample_rate: {getattr(self.model.compression_model, 'sample_rate', 'None')}\")\n # except Exception as e:\n # print(f\" - 验证时出错: {e}\")\n \n # if self.verbose:\n # print(f\"[Debug] 调用generate_continuation参数:\")\n # print(f\" - prompt shape: {pa.shape}\")\n # print(f\" - prompt_sample_rate: {self.SR}\")\n # print(f\" - descriptions数量: {len(descriptions)}\")\n # print(f\" - descriptions[0]: {descriptions[0][:100]}...\")\n # print(f\" - 期望duration: {window_sec}s\")\n # \n # # 检查generate_continuation方法签名\n # import inspect\n # sig = inspect.signature(self.model.generate_continuation)\n # print(f\" - generate_continuation方法签名: {sig}\")\n \n # 尝试显式传递所有参数\n try:\n # if self.verbose:\n # print(f\"[Debug] 🚀 开始调用generate_continuation...\")\n \n wavs = self.model.generate_continuation(\n prompt=pa,\n prompt_sample_rate=self.SR,\n descriptions=descriptions,\n progress=self.verbose,\n )\n \n # if self.verbose:\n # print(f\"[Debug] ✅ generate_continuation调用成功\")\n \n except Exception as e:\n # if self.verbose:\n # print(f\"[Debug] ❌ generate_continuation调用失败: {e}\")\n # import traceback\n # traceback.print_exc()\n # 如果失败，返回空列表避免崩溃\n wavs = []\n \n # if self.verbose:\n # print(f\"[Debug] generate_continuation返回结果:\")\n # print(f\" - 返回wav数量: {len(wavs)}\")\n # if len(wavs) > 0:\n # print(f\" - wavs[0] shape: {wavs[0].shape}\")\n # print(f\" - wavs[0] dtype: {wavs[0].dtype}\")\n # print(f\" - wavs[0] device: {wavs[0].device}\")\n # print(f\" - 实际生成时长: {wavs[0].shape[-1]/self.SR:.3f}s\")\n # print(f\" - 期望时长: {window_sec}s\")\n # print(f\" - 时长比例: {(wavs[0].shape[-1]/self.SR)/window_sec:.3f}\")\n # \n # # 检查是否包含prompt\n # prompt_len = pa.shape[-1]\n # wav_len = wavs[0].shape[-1]\n # print(f\" - prompt长度: {prompt_len/self.SR:.3f}s\")\n # print(f\" - 生成结果长度: {wav_len/self.SR:.3f}s\")\n # print(f\" - 是否包含prompt: {wav_len >= prompt_len}\")\n # \n # if wav_len < window_sec * self.SR * 0.9: # 如果小于期望的90%\n # print(f\" - ⚠️ 警告：生成长度不足期望的90%\")\n # if wav_len < prompt_len * 1.5: # 如果生成的内容比prompt长度少于50%\n # print(f\" - ⚠️ 警告：生成内容可能主要是prompt重复\")\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] got {len(wavs)} wavs\", flush=True)\n # if len(wavs) > 0:\n # print(f\"[Debug] 生成的音频长度: {wavs[0].shape[-1]/self.SR:.1f}s\")\n\n # 5) 选出相似度最高的 wav，同时收集所有 sim 值\n sim_list = []\n best, best_sim = None, -1.0\n \n def _ensure_mono32(t: torch.Tensor, sr: int = 32000) -> torch.Tensor:\n \"\"\"\n 保证张量变成 [1, T] 单声道并且已是 32 kHz。\n 这里假设外层音频都已重采样到 self.SR=32 kHz；仅做声道合并。\n \"\"\"\n if t.dim() == 2 and t.shape[0] > 1: # 立体声 → 单声道\n t = t.mean(0, keepdim=True)\n return t.to(self.device)\n\n for i, wav in enumerate(wavs):\n # ------------- 1) 规格统一 -------------\n wav_mono = _ensure_mono32(wav) # 生成片段（完整的30秒）\n ref_tail = _ensure_mono32(pa.squeeze(0)) # prompt tail（5秒）\n \n # 相似度计算：只比较开头的prompt部分\n prompt_len = ref_tail.shape[-1]\n wav_prompt = wav_mono[..., :prompt_len] # 取生成结果的前5秒用于相似度比较\n \n # if self.verbose:\n # print(f\"[Debug] 相似度计算:\")\n # print(f\" - wav_mono总长: {wav_mono.shape[-1]/self.SR:.1f}s\")\n # print(f\" - ref_tail长度: {ref_tail.shape[-1]/self.SR:.1f}s\") \n # print(f\" - wav_prompt长度: {wav_prompt.shape[-1]/self.SR:.1f}s\")\n\n # ------------- 2) 计算三项相似度 ----------\n sim_clap = torch.cosine_similarity(\n self.embed_cpu(wav_prompt), self.embed_cpu(ref_tail)\n ).item()\n\n sdr_db = self.compute_si_sdr(ref_tail, wav_prompt).mean().item() # dB\n mel_l2 = self.mel_distance(ref_tail, wav_prompt) # L2\n\n # 映射到 0–1 区间\n sdr_norm = min(max((sdr_db - 0) / 10.0, 0.0), 1.0) # 0 dB→0, 10 dB→1\n mel_norm = min(250.0 / (mel_l2 + 1.0), 1.0) # 250→1, 500→0.5\n\n # 综合得分\n sim_final = 0.4 * sim_clap + 0.4 * sdr_norm + 0.2 * mel_norm\n\n # ------------- 3) 调试输出 --------------\n if self.verbose:\n print(\n f\"CLAP={sim_clap:.3f} \"\n f\"SDR={sdr_db:.2f}dB(n={sdr_norm:.2f}) \"\n f\"MelL2={mel_l2:.0f}(n={mel_norm:.2f}) \"\n f\"final={sim_final:.3f}\"\n )\n\n # ------------- 4) 选最佳 ----------------\n sim_list.append(sim_final)\n if sim_final > best_sim:\n # 重要：保存完整的wav_mono（30秒），而不是只保存prompt部分（5秒）\n best, best_sim = wav_mono.cpu(), sim_final\n del wav\n\n\n torch.cuda.empty_cache()\n\n # 打印这一段所有候选的 sim 数组和选出的 best_sim\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] sims = {sim_list}\")\n print(f\"[Track {idx+1} Segment {cur_seg}] best_sim = {best_sim:.4f}\")\n\n # save_best_each = True\n save_best_each = False\n # 立刻保存本段 best 候选 —— \n if save_best_each and out_path:\n # out_path 父目录下创建 bestwav 文件夹\n best_dir = pathlib.Path(out_path).parent / \"bestwav\"\n best_dir.mkdir(parents=True, exist_ok=True)\n save_name = best_dir / f\"Segment_{cur_seg}.wav\"\n\n # 生成子文件名，并写文件\n # 生成微调结果\n # 把 float16 转成 float32，再保存\n \n audio_ft = best.cpu().float().numpy()\n if audio_ft.ndim == 2 and audio_ft.shape[0] < audio_ft.shape[1]:\n audio_ft = audio_ft.T\n sf.write(str(save_name), audio_ft, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1} Seg {cur_seg}] saved best wav → {save_name}\")\n\n # 6) 相似度太低时重试或跳过\n if best_sim < sim_thresh:\n retry_count += 1\n if retry_count > max_retries:\n if self.verbose:\n print(f\"Segment {cur_seg} reached max retries, skipping segment\")\n retry_count = 0\n break\n p = segs[cur_seg]\n new_temp = max(0.15, p[\"temperature\"] * (0.9 ** retry_count))\n new_top_k = p[\"top_k\"] * (1.15 ** retry_count)\n new_top_p = min(p[\"top_p\"]+0.05,0.95)\n self.model.set_generation_params(\n duration=window_sec,\n temperature=new_temp,\n top_k=new_top_k,\n top_p = new_top_p,\n use_sampling=True,\n )\n if self.verbose:\n print(f\"Retry {retry_count}: temp→{new_temp:.2f}, topk→{new_top_k:.2f}\")\n continue\n\n # 7) 拼接并输出调试信息\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] before xfade: full={full.shape[-1]/self.SR:.1f}s, best={best.shape[-1]/self.SR:.1f}s\")\n full = self.xfade(full, best.to(self.device), overlap_sec)\n if self.verbose:\n print(f\"[Track {idx+1} Segment {cur_seg}] after xfade: {full.shape[-1]/self.SR:.1f}s\")\n # print(f\"[Debug] xfade参数: overlap_sec={overlap_sec}, 预期增长={(window_sec-overlap_sec):.1f}s\")\n del best\n torch.cuda.empty_cache()\n retry_count = 0\n\n # 完成 single-track 生成，导出文件\n final = full[..., : total_sec * self.SR].cpu().T\n if out_path:\n stem, ext = os.path.splitext(out_path)\n out_file = f\"{stem}_{idx+1:02d}{ext}\" if n_tracks > 1 else out_path\n else:\n out_file = f\"final_{idx+1:02d}.wav\" if n_tracks > 1 else \"final.wav\"\n\n pathlib.Path(out_file).parent.mkdir(parents=True, exist_ok=True)\n sf.write(out_file, final, self.SR)\n if self.verbose:\n print(f\"[Track {idx+1}] saved {out_file}, best_sim={best_sim:.4f}\", flush=True)\n return out_file\n\n # 并行生成多条 track\n print(f\"[Generate] launching _run_once for {n_tracks} tracks\", flush=True)\n paths = [_run_once(i) for i in range(n_tracks)]\n print(f\"[Generate] all done, paths = {paths}\", flush=True)\n return paths if n_tracks > 1 else paths[0]\n\n \n\n\ndef generate_music_api(params: Dict) -> Dict:\n # svc = MusicGenService(\n # api_endpoint=params.get('api_endpoint', API_ENDPOINT),\n # openai_api_key=params.get('api_key', OPENAI_API_KEY),\n # openai_model=params.get('model', OPENAI_MODEL),\n # verbose=params.get('verbose', True),\n # checkpoint_path=params.get('checkpoint_path'), # ← 把它传进去\n # )\n svc = MusicGenService(\n use_local_llm=params.get('use_local_llm', DEFAULT_USE_LOCAL_LLM),\n api_endpoint=params.get('api_endpoint'), # 允许外部自定义\n openai_api_key=params.get('api_key', OPENAI_API_KEY),\n openai_model=params.get('model', OPENAI_MODEL),\n verbose=params.get('verbose', True),\n checkpoint_path=params.get('checkpoint_path'),\n )\n out = svc.generate(\n seed_path=params['seed_path'],\n style=params['style'],\n user_prompt=params['user_prompt'],\n total_sec=params.get('total_sec', 120),\n window_sec=params.get('window_sec', 20),\n overlap_sec=params.get('overlap_sec', 5),\n prompt_sec=params.get('prompt_sec', 5),\n safety_tail=params.get('safety_tail', 3),\n clap_candidates= 1,#params.get('clap_candidates', 3),\n n_tracks=params.get('n_tracks', 1), # ← 新增这一行\n use_fp16=params.get('use_fp16', False), # ← 以及这一行\n sim_thresh=params.get('sim_thresh', 0.5),\n sil_thresh=params.get('sil_thresh', -40),\n melody_path=params.get('melody_path'),\n out_path=params.get('out_path'),\n )\n return {'out_path': out}\n\n"}}}} |