From b45d8a6691bf711de9beed77d720e35e49d3604e Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 24 Aug 2026 16:04:08 -0600 Subject: [PATCH 01/12] data_transforms dataloader block for automatically RVCing inputs --- documentation/DATALOADER.es.md | 7 + documentation/DATALOADER.hi.md | 7 + documentation/DATALOADER.ja.md | 7 + documentation/DATALOADER.md | 7 + documentation/DATALOADER.pt-BR.md | 7 + documentation/DATALOADER.zh.md | 7 + .../experimental/VOICE_CLONING.es.md | 359 +++++++++++++ .../experimental/VOICE_CLONING.hi.md | 359 +++++++++++++ .../experimental/VOICE_CLONING.ja.md | 359 +++++++++++++ documentation/experimental/VOICE_CLONING.md | 366 +++++++++++++ .../experimental/VOICE_CLONING.pt-BR.md | 359 +++++++++++++ .../experimental/VOICE_CLONING.zh.md | 359 +++++++++++++ documentation/index.es.md | 4 +- documentation/index.hi.md | 4 +- documentation/index.ja.md | 4 +- documentation/index.md | 4 +- documentation/index.pt-BR.md | 4 +- documentation/index.zh.md | 4 +- mkdocs.yml | 1 + simpletuner/helpers/data_backend/factory.py | 24 +- .../helpers/data_transforms/__init__.py | 10 + simpletuner/helpers/data_transforms/base.py | 111 ++++ .../data_transforms/identity_transfer.py | 491 ++++++++++++++++++ .../helpers/metadata/utils/duplicator.py | 1 + .../server/data/dataset_blueprints.py | 8 + .../static/js/dataloader-section-component.js | 69 +++ .../dataloader/sections/audio_body.html | 127 +++++ simpletuner/templates/trainer_htmx.html | 78 +++ .../js/dataloader_audio_capabilities.test.js | 40 ++ .../test_data_transforms_identity_transfer.py | 161 ++++++ tests/test_dataset_blueprints.py | 11 + tests/test_factory_edge_cases.py | 49 ++ 32 files changed, 3388 insertions(+), 20 deletions(-) create mode 100644 documentation/experimental/VOICE_CLONING.es.md create mode 100644 documentation/experimental/VOICE_CLONING.hi.md create mode 100644 documentation/experimental/VOICE_CLONING.ja.md create mode 100644 documentation/experimental/VOICE_CLONING.md create mode 100644 documentation/experimental/VOICE_CLONING.pt-BR.md create mode 100644 documentation/experimental/VOICE_CLONING.zh.md create mode 100644 simpletuner/helpers/data_transforms/__init__.py create mode 100644 simpletuner/helpers/data_transforms/base.py create mode 100644 simpletuner/helpers/data_transforms/identity_transfer.py create mode 100644 tests/test_data_transforms_identity_transfer.py diff --git a/documentation/DATALOADER.es.md b/documentation/DATALOADER.es.md index 0d7b91b26..a6ae307d7 100644 --- a/documentation/DATALOADER.es.md +++ b/documentation/DATALOADER.es.md @@ -258,6 +258,13 @@ Los backends de memoria requieren Linux o macOS y suficiente RAM o swap para la - **Nota:** Si tienes varios datasets de condicionamiento, puedes especificarlos como un arreglo de valores `id`. Al entrenar Flux Kontext, esto permite cambiar aleatoriamente entre condiciones o unir entradas para entrenar tareas avanzadas de composición multi-imagen. - **Flow-DPO:** Empareja aquí un dataset `reference_strict` cuando uses [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.es.md). +### `data_transforms` + +- **Valores:** un objeto de transform o un arreglo de objetos de transform +- **Descripción:** Expande un dataset fuente en uno o más datasets de entrenamiento generados antes de que empiece la configuración normal del dataloader. Los datasets generados se tratan como datasets primarios normales salvo que el transform pida explícitamente clonar metadatos. +- **Identity transfer de audio:** `{"task": "identity_transfer", "method": "rvc"}` está disponible para backends con `dataset_type: "audio"`. Prepara un split de audio generado para transferencia de identidad vocal y usa el directorio de salida para artefactos de voz cacheados y archivos generados. Consulta [Voice Cloning Data Transforms](experimental/VOICE_CLONING.es.md). +- **Estado:** La primera implementación aporta el registro de transforms, comprobaciones de manifiestos de caché, reutilización/subida de artefactos a Hub, hooks de sharding de inicio compatibles con DDP y logs locales de RVC. El entrenamiento/conversión RVC nativo sigue siendo experimental y fallará de forma explícita salvo que ya exista un split generado compatible en la caché. + ### `instance_data_dir` / `aws_data_prefix` - **Local:** Ruta a los datos en el filesystem. diff --git a/documentation/DATALOADER.hi.md b/documentation/DATALOADER.hi.md index be8ca63aa..137dea0de 100644 --- a/documentation/DATALOADER.hi.md +++ b/documentation/DATALOADER.hi.md @@ -258,6 +258,13 @@ Memory backends के लिए Linux या macOS और मौजूदा ca - **Note:** यदि आपके पास कई conditioning datasets हैं, तो आप उन्हें `id` values के array के रूप में दे सकते हैं। Flux Kontext ट्रेन करते समय, यह conditions के बीच random switching या multi‑image compositing tasks के लिए inputs stitch करने की अनुमति देता है। - **Flow-DPO:** [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.hi.md) इस्तेमाल करते समय यहां `reference_strict` conditioning dataset pair करें। +### `data_transforms` + +- **Values:** एक transform object या transform objects का array +- **Description:** Normal dataloader setup शुरू होने से पहले source dataset को एक या अधिक generated training datasets में expand करता है। Generated datasets को regular primary datasets की तरह treat किया जाता है, जब तक transform explicitly metadata clone न मांगे। +- **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` `dataset_type: "audio"` backends के लिए उपलब्ध है। यह voice identity transfer के लिए generated audio split तैयार करता है और output directory में cached voice artifacts तथा generated files रखता है। देखें [Voice Cloning Data Transforms](experimental/VOICE_CLONING.hi.md)। +- **Status:** पहली implementation transform registry, cache manifest checks, Hub artifact reuse/push plumbing, DDP-aware startup sharding hooks, और local RVC logs देती है। Native RVC training/conversion अभी experimental है और cache में matching generated split न होने पर साफ error देगा। + ### `instance_data_dir` / `aws_data_prefix` - **Local:** filesystem पर डेटा का path. diff --git a/documentation/DATALOADER.ja.md b/documentation/DATALOADER.ja.md index 5b29dbe8f..cf98745cf 100644 --- a/documentation/DATALOADER.ja.md +++ b/documentation/DATALOADER.ja.md @@ -258,6 +258,13 @@ Hugging Face の音声データセットでは、キャプション（プロン - **注記:** 複数の条件データセットがある場合は `id` の配列を指定できます。Flux Kontext の学習時には、条件をランダムに切り替えたり入力を結合したりして、より高度な複数画像合成タスクの学習が可能です。 - **Flow-DPO:** [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.ja.md) を使う場合は、ここで `reference_strict` conditioning データセットをペアにします。 +### `data_transforms` + +- **Values:** transform オブジェクト、または transform オブジェクトの配列 +- **Description:** 通常の dataloader 設定が始まる前に、ソース dataset から 1 つ以上の生成済み training dataset を展開します。transform が明示的に metadata clone を要求しない限り、生成 dataset は通常の primary dataset として扱われます。 +- **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` は `dataset_type: "audio"` の backend で利用できます。音声 identity transfer 用の生成 audio split を準備し、output directory に voice artifact と生成ファイルをキャッシュします。詳しくは [Voice Cloning Data Transforms](experimental/VOICE_CLONING.ja.md) を参照してください。 +- **Status:** 最初の実装では transform registry、cache manifest の確認、Hub artifact の reuse/push、DDP 対応の startup sharding hook、ローカル RVC log を提供します。ネイティブ RVC training/conversion はまだ experimental であり、互換性のある生成 split が cache に存在しない場合は明示的に失敗します。 + ### `instance_data_dir` / `aws_data_prefix` - **Local:** ファイルシステム上のデータパス。 diff --git a/documentation/DATALOADER.md b/documentation/DATALOADER.md index a3248371a..716b411b1 100644 --- a/documentation/DATALOADER.md +++ b/documentation/DATALOADER.md @@ -260,6 +260,13 @@ Memory backends require Linux or macOS and enough RAM or swap for the existing c - **Note:** If you have multiple conditioning datasets, you can specify them as an array of `id` values. When training Flux Kontext, this allows switching between conditions randomly or stitching inputs together to train in more advanced multi-image compositing tasks. - **Flow-DPO:** Pair a `reference_strict` conditioning dataset here when using [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.md). +### `data_transforms` + +- **Values:** a transform object or an array of transform objects +- **Description:** Expands a source dataset into one or more generated training datasets before normal dataloader setup begins. Generated datasets are treated as regular primary datasets unless the transform explicitly asks to clone metadata. +- **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` is available for `dataset_type: "audio"` backends. It prepares a generated audio split for voice identity transfer and uses the output directory for cached voice artifacts and generated files. See [Voice Cloning Data Transforms](experimental/VOICE_CLONING.md). +- **Status:** The first implementation provides the transform registry, cache manifest checks, Hub artifact reuse/push plumbing, DDP-aware startup sharding hooks, and local RVC logs. Native RVC training/conversion is still experimental and will fail loudly unless a matching generated split already exists in the cache. + ### `instance_data_dir` / `aws_data_prefix` - **Local:** Path to the data on the filesystem. diff --git a/documentation/DATALOADER.pt-BR.md b/documentation/DATALOADER.pt-BR.md index 3f1afcee6..1d6d95154 100644 --- a/documentation/DATALOADER.pt-BR.md +++ b/documentation/DATALOADER.pt-BR.md @@ -258,6 +258,13 @@ Backends de memória exigem Linux ou macOS e RAM ou swap suficiente para o cache - **Nota:** Se você tiver múltiplos datasets de condicionamento, pode especificá-los como um array de valores `id`. Ao treinar Flux Kontext, isso permite alternar aleatoriamente entre condições ou juntar entradas para treinar tarefas mais avançadas de composição multi-imagem. - **Flow-DPO:** Pareie aqui um dataset `reference_strict` ao usar [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.pt-BR.md). +### `data_transforms` + +- **Valores:** um objeto de transform ou um array de objetos de transform +- **Descrição:** Expande um dataset fonte em um ou mais datasets de treinamento gerados antes da configuração normal do dataloader. Datasets gerados são tratados como datasets primários normais, a menos que o transform peça explicitamente clonagem de metadados. +- **Identity transfer de áudio:** `{"task": "identity_transfer", "method": "rvc"}` está disponível para backends com `dataset_type: "audio"`. Ele prepara um split de áudio gerado para transferência de identidade vocal e usa o diretório de saída para artefatos de voz em cache e arquivos gerados. Consulte [Voice Cloning Data Transforms](experimental/VOICE_CLONING.pt-BR.md). +- **Status:** A primeira implementação fornece o registro de transforms, verificações de manifest de cache, reutilização/envio de artefatos para o Hub, hooks de sharding de inicialização compatíveis com DDP e logs locais de RVC. O treinamento/conversão RVC nativo ainda é experimental e falhará explicitamente salvo se já existir um split gerado compatível no cache. + ### `instance_data_dir` / `aws_data_prefix` - **Local:** Caminho para os dados no sistema de arquivos. diff --git a/documentation/DATALOADER.zh.md b/documentation/DATALOADER.zh.md index 53a180a84..fbd83c339 100644 --- a/documentation/DATALOADER.zh.md +++ b/documentation/DATALOADER.zh.md @@ -257,6 +257,13 @@ LTX-2 使用原生纯音频分支；MiniMax-H3 在打包序列中为每个 laten - **注记:** 如果有多个条件数据集，可指定 `id` 数组。训练 Flux Kontext 时，可在条件之间随机切换或拼接输入，用于更高级的多图像合成任务。 - **Flow-DPO:** 使用 [`--distillation_method=flow_dpo`](experimental/FLOW_DPO.zh.md) 时，在这里配对一个 `reference_strict` conditioning 数据集。 +### `data_transforms` + +- **取值:** 一个 transform 对象，或 transform 对象数组 +- **说明:** 在常规 dataloader 设置开始前，把源数据集展开为一个或多个生成的训练数据集。除非 transform 明确要求克隆元数据，生成的数据集会被当作普通主训练数据集处理。 +- **音频 identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` 可用于 `dataset_type: "audio"` 后端。它会为声音身份迁移准备生成音频 split，并在输出目录中缓存 voice artifacts 和生成文件。参见 [Voice Cloning Data Transforms](experimental/VOICE_CLONING.zh.md)。 +- **状态:** 第一版实现包含 transform registry、缓存 manifest 检查、Hub artifact 复用/上传流程、DDP 感知的启动 sharding hook，以及本地 RVC 日志。原生 RVC 训练/转换仍为实验性质；除非缓存中已经存在匹配的生成 split，否则会明确失败。 + ### `instance_data_dir` / `aws_data_prefix` - **Local:** 文件系统中的数据路径。 diff --git a/documentation/experimental/VOICE_CLONING.es.md b/documentation/experimental/VOICE_CLONING.es.md new file mode 100644 index 000000000..ea0182516 --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.es.md @@ -0,0 +1,359 @@ +# Transforms de Clonacion de Voz + +Los transforms de clonacion de voz son una funcion experimental planificada para datasets de audio. Sirven para expandir un conjunto de entrenamiento transfiriendo una identidad vocal a canciones, stems o interpretaciones adicionales antes de que empiece el entrenamiento principal del modelo. + +El objetivo no es convertir SimpleTuner en una estacion separada de conversion de voz. El objetivo es reducir el entrelazado del dataset de audio. Si una voz solo aparece en un estilo estrecho, una LoRA puede aprender "esta voz dentro de este arreglo" en vez de la identidad vocal en si. Un split expandido con voice cloning puede poner la misma identidad vocal en arreglos, captions, letras y estructuras musicales mas variadas. + +Esta funcion esta pensada solo para datasets de audio. + +!!! warning "Consentimiento y derechos" + Usa este workflow solo con voces y grabaciones que tienes permiso para usar. La identidad vocal es dato biometrico y creativo sensible. El transform puede crear audio derivado que suena como una persona real, asi que permisos, licencias y divulgacion importan. + +## ELI5 + +Imagina que tienes seis grabaciones de una persona cantando, pero todas son de la misma banda y del mismo genero. Si entrenas solo con esas canciones, el modelo puede aprender que la voz, el tono de guitarra, la bateria, el rango de tempo y la estructura de cancion son una sola cosa. + +Los transforms de clonacion de voz intentan separar esas ideas: + +1. Aprenden un pequeno modelo de conversion de voz a partir de ejemplos de la voz objetivo. +2. Toman un conjunto mas amplio de canciones o vocal stems. +3. Sustituyen el timbre vocal de origen por el timbre objetivo. +4. Mantienen las nuevas captions y letras alineadas con el audio generado. +5. Agregan el audio generado como otro split normal de entrenamiento. + +Asi el modelo principal ve la voz objetivo en mas contextos, no solo memorizando el dataset estrecho original. + +## Para Que Sirve + +Usalo cuando: + +- tienes grabaciones autorizadas de la voz objetivo +- la identidad objetivo esta demasiado ligada a un genero, banda, produccion o estructura de cancion +- los trigger words solo funcionan dentro del dominio original +- dos o mas voces en un dataset se mezclan en una voz promedio +- quieres LoRAs separadas para identidades vocales separadas +- quieres que SimpleTuner prepare el split expandido dentro del mismo setup de entrenamiento + +Evitalo cuando: + +- ya tienes un dataset grande, variado y limpio de la misma voz +- el audio fuente de expansion es de baja calidad o no coincide con las captions +- necesitas publicar resultados y no tienes derechos claros +- el modelo generativo base no aprende la identidad ni con ejemplos directos limpios + +## Como Entra En El Entrenamiento + +La clonacion de voz es un transform de preparacion de datos, no un dataset de conditioning. + +`conditioning_data` es para entradas auxiliares pareadas que permanecen unidas a una muestra primaria durante el entrenamiento, como imagenes de referencia o mapas de conditioning generados. + +La clonacion de voz debe vivir en una lista `data_transforms` a nivel de dataset. El transform materializa nuevos archivos de audio, captions y letras opcionales, y registra el resultado como otro dataset primario `audio`. Despues, el dataloader normal lo ve como cualquier otro split de entrenamiento. + +Forma de pseudo config: + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +Comportamiento de inicio en pseudocodigo: + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## Transferencia de Identidad Estilo RVC + +La primera implementacion prevista es conversion de voz estilo RVC. + +En este contexto, el "modelo RVC" es especifico de la voz. Se entrena desde el dataset de identidad objetivo. El indice de recuperacion tambien es especifico de la voz y se construye desde features de esa misma voz. Componentes preentrenados amplios, como features de contenido, extraccion de pitch o modelos de separacion, son infraestructura reutilizable; el modelo de conversion y el indice son artifacts especificos del cantante o hablante. + +SimpleTuner deberia poder: + +1. Reutilizar un modelo de conversion de voz y un indice provistos. +2. Entrenar el modelo de conversion si no se proporciona ninguno. +3. Construir el indice de recuperacion desde los datos de la voz objetivo. +4. Cachear modelo, indice y audio generado bajo el directorio de salida del entrenamiento. +5. Reutilizar artifacts cacheados al inicio cuando los datos y settings no cambiaron. +6. Opcionalmente reutilizar o publicar el modelo de conversion de voz mediante un repositorio de modelo en el Hub. + +## Comportamiento Por Defecto + +Los defaults planificados son conservadores: + +| Setting | Default | Por que | +| --- | --- | --- | +| `task` | `identity_transfer` | Identifica explicitamente el transform. | +| `method` | `rvc` | Primer backend de transferencia vocal soportado. | +| `train_if_missing` | `true` | SimpleTuner debe poder bootstrapear el modelo vocal desde el dataset objetivo. | +| `force_retrain` | `false` | Reutiliza un modelo cacheado valido cuando sea posible. | +| `build_index` | `true` | Retrieval suele mejorar estabilidad de identidad y reducir leakage. | +| `hub_model_id` | sin definir | No se usa cache remoto de modelo vocal sin opt-in del usuario. | +| `reuse_from_hub` | `true` cuando `hub_model_id` esta definido | Revisa el Hub antes de gastar tiempo entrenando un modelo bajo demanda. | +| `push_to_hub` | `false` | Subir un modelo vocal debe ser explicito porque el artifact representa una identidad vocal. | +| `audio_mode` | `separate_convert_remix` para canciones completas, `vocal_only` para vocal stems | Las mezclas completas necesitan separacion; los stems no. | +| `separation_method` | `demucs` cuando se necesita separacion | Demucs es el stem separator default esperado. | +| tipo del split generado | dataset primario `audio` | Los datos generados entrenan como audio normal, no como conditioning. | +| ubicacion de cache | dentro de `output_dir` | Mantiene artifacts ligados al entrenamiento y reutilizables al reiniciar. | +| captions | copia captions fuente salvo configuracion diferente | El nuevo split debe preservar letras y contexto de arreglo. | + +Si se proporciona un modelo de conversion existente, SimpleTuner debe usarlo y solo entrenar uno nuevo cuando se pida explicitamente o falten artifacts necesarios. + +## Cache en el Hub + +Un modelo de conversion de voz puede ser lo bastante caro como para que entrenarlo bajo demanda repetidamente sea una trampa. Por eso el transform debe soportar un cache opcional en el Hub para el modelo vocal y el indice de retrieval. + +El orden seguro de busqueda es: + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +El repositorio del Hub debe usar un layout especifico de SimpleTuner, no una coleccion suelta de archivos: + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +El manifest es el contrato. Debe registrar el fingerprint del dataset de identidad objetivo, settings de entrenamiento RVC, settings del indice, sample rate esperado, versiones de herramientas y version del formato voice-transform de SimpleTuner. SimpleTuner no debe reutilizar un artifact del Hub si no tiene este manifest o si el manifest no coincide con el transform actual. Eso evita aplicar silenciosamente el modelo vocal equivocado a un dataset nuevo. + +Publicar debe ser opt-in. Una pseudo config razonable: + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +Para identidades privadas, mantén el repositorio del Hub privado salvo permiso explicito para publicar el modelo vocal. El audio generado y los artifacts del modelo pueden tener derechos diferentes, asi que trata sus settings de upload por separado. + +## Configuracion en WebUI + +El entrenamiento del modelo RVC debe poder configurarse desde WebUI, no solo con JSON crudo del dataloader. + +La forma esperada en WebUI es un editor de transforms dentro del dataset de audio: + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +La WebUI debe hacer obvios los dos setups comunes: + +- **Ya tienes vocal stems:** elige `vocal_only`, deja Demucs desactivado y escribe vocal stems generados. +- **Tienes canciones completas:** elige `separate_convert_remix`, usa separacion con Demucs, convierte solo el vocal stem y remezcla con los stems instrumentales originales. + +La interfaz debe mostrar que el audio generado se convierte en otro split primario de entrenamiento de audio. No debe presentar identity transfer como `conditioning_data`, porque eso implicaria comportamiento de conditioning pareado durante el entrenamiento. + +## Comportamiento Distribuido al Inicio + +Cuando SimpleTuner inicia con varios ranks data-parallel, el startup de voice cloning debe usar las GPUs disponibles en vez de hacer que rank 0 haga todo el trabajo. + +Hay dos fases distribuidas separadas: + +1. **Entrenamiento del modelo RVC:** si `train_if_missing=true`, no hay cache local coincidente y no hay artifact coincidente en el Hub, el loop de entrenamiento RVC debe ejecutarse con DDP cuando `world_size > 1`. Cada rank debe recibir batches distintos de la voz objetivo mediante el patron normal de distributed sampler. +2. **Preparacion del audio generado:** las entradas fuente de expansion deben dividirse por rank, parecido a TextEmbedCache y VAECache. Cada rank separa, convierte y escribe solo su shard; luego todos los ranks sincronizan antes de continuar metadata discovery. + +Pseudocodigo: + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +Solo un proceso debe publicar el modelo vocal final en el Hub. Lo mismo aplica a updates finales del manifest. Los outputs generados por rank pueden escribirse independientemente si los nombres son deterministicos y no se superponen. + +Esto evita desperdiciar tiempo de GPU en sistemas multi-GPU y mantiene el startup alineado con el modelo existente de preparacion de cache de SimpleTuner. + +## Logs del Entrenamiento RVC + +El entrenamiento RVC en startup no debe crear runs de TensorBoard o WandB todavia. Esos loggers se configuran para el trabajo principal de entrenamiento de SimpleTuner, y reutilizarlos para un trabajo anidado de conversion de voz exigiria nombres de run, rutas, reglas de resume y politicas de artifact adicionales. + +La etapa RVC aun puede reportar stats utiles mediante el logger nativo de entrenamiento de SimpleTuner: + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +Stats locales utiles incluyen loss de entrenamiento RVC, pitch loss si esta habilitado, reconstruction o discriminator loss cuando aplique, samples procesados, tiempo transcurrido, DDP world size, motivo de cache hit o miss, y si el modelo final vino de cache local, cache del Hub o entrenamiento bajo demanda. + +Estos stats son solo locales salvo que una implementacion futura agregue explicitamente integracion con logger externo para RVC transforms. + +## Elegir `audio_mode` + +### `vocal_only` + +Usalo cuando tu dataset de expansion ya esta preprocesado como vocal stems limpios. + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +Gotchas: + +- No ejecutes Demucs de nuevo sobre stems limpios sin una razon clara. +- Las captions deben describir voces y letras, no un arreglo completo de banda, salvo que vayas a remezclar despues. +- Si el modelo principal espera canciones completas, los datos vocal-only pueden ensenar una distribucion distinta. + +### `separate_convert_remix` + +Usalo cuando el dataset de expansion contiene canciones completas mezcladas. + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +Este es el modo preferido para expansion de canciones completas porque evita convertir bateria, bajo, guitarras, sala y artifacts de master como si fueran parte de la voz. + +Gotchas: + +- La separacion de stems puede dejar bleed, artifacts o problemas de fase. +- Si el vocal stem es debil, reverberante o enterrado, la voz convertida puede volverse inestable. +- El loudness del remix importa. Un split generado consistentemente mas alto o bajo puede sesgar el entrenamiento. +- Las captions deben describir el resultado remixado final, no solo la cancion fuente. + +### `full_mix_convert` + +Usalo solo para pruebas rapidas. + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +Es rapido, pero normalmente tiene menor calidad. Puede arrastrar instrumentos por el conversor de voz y ensenar artifacts no deseados a la LoRA final. + +## Captions y Letras + +El split generado debe tener captions que coincidan con el audio generado. + +Un buen default: + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +Para letras, copiar suele ser correcto cuando la interpretacion fuente y la convertida usan las mismas palabras. No es correcto cuando el transform cambia la cancion, edita secciones, quita voces o usa una fuente sin letra. + +Copiar captions a ciegas puede estar mal. Si la caption fuente dice "female pop vocal" y la salida convertida tiene timbre masculino de rock, la caption debe ajustarse. El transform debe soportar reglas simples de append/remove; reescritura avanzada de captions puede venir despues. + +## Cache y Reuso + +El transform debe escribir dos tipos de cache: + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +El manifest debe registrar fingerprint del dataset de identidad, settings del transform, fingerprint de los datos fuente de expansion y versiones de herramientas. Si coinciden, el inicio puede reutilizar artifacts existentes. Si cambian, SimpleTuner debe regenerar solo la etapa afectada. + +## Consejos Practicos + +- Mantén una voz objetivo por LoRA cuando importa el control de identidad. +- Prefiere ejemplos vocales limpios y secos para entrenar el modelo de conversion. +- Evita duetos salvo que el objetivo sea aprender la mezcla del dueto. +- Usa canciones de expansion con variedad de tempo, tonalidad, genero, dinamica y fraseo. +- Varia captions para que los tokens de identidad no queden pegados a un solo arreglo. +- Revisa audio generado antes de entrenamientos largos. + +## Fallos Comunes + +| Sintoma | Causa probable | +| --- | --- | +| La LoRA solo funciona en un genero | La identidad vocal sigue entrelazada con captions de arreglo o datos fuente. | +| El split generado suena hueco o con fase rara | Artifacts de separacion/remix en procesamiento de canciones completas. | +| Los instrumentos suenan convertidos como voz | Se uso `full_mix_convert` cuando hacia falta separacion. | +| La identidad vocal es debil | Faltan datos objetivo mas limpios, mas datos o mejor indice. | +| Las captions no controlan la voz | Las captions aun mencionan la voz fuente u omiten la identidad objetivo. | +| El modelo principal aprende artifacts | El audio generado tiene baja calidad o demasiado peso en el mix de entrenamiento. | + +## Relacion Con Datos de Regularizacion + +Los datos generados por identity transfer no son datos de regularizacion por defecto. + +Los datos de regularizacion suelen ensenar a la LoRA a preservar el comportamiento del modelo base. Los datos de identity transfer ensenan una voz objetivo en mas contextos. Demasiada regularizacion con pocos datos directos de identidad puede debilitar tokens de identidad. Demasiados datos generados pueden ensenar artifacts de conversion. + +Tratalos como controles separados: + +- dataset objetivo directo: senal de identidad mas fuerte +- dataset generado por identity transfer: mayor cobertura de contexto y estilo +- dataset de regularizacion: preservacion del modelo base + +## Estado + +Esta pagina describe el comportamiento previsto para un workflow experimental `data_transforms`. La restriccion importante de diseno es que identity transfer sea una funcion de audio de primera clase en SimpleTuner: entrenar o reutilizar el modelo de voice conversion, construir o reutilizar el indice, generar el split expandido, cachear resultados y continuar al entrenamiento normal sin exigir una segunda etapa manual de preprocessing. diff --git a/documentation/experimental/VOICE_CLONING.hi.md b/documentation/experimental/VOICE_CLONING.hi.md new file mode 100644 index 000000000..349bf803e --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.hi.md @@ -0,0 +1,359 @@ +# Voice Cloning Data Transforms + +Voice cloning data transforms audio datasets के लिए planned experimental feature हैं। ये main model training शुरू होने से पहले किसी target vocal identity को extra songs, stems, या performances पर transfer करके training set expand करते हैं। + +मकसद SimpleTuner को अलग voice-conversion workstation बनाना नहीं है। मकसद audio fine-tuning dataset में identity और arrangement के entanglement को कम करना है। अगर कोई singer सिर्फ एक narrow style में आता है, तो LoRA "इस arrangement में यह singer" सीख सकती है, singer identity खुद नहीं। Voice-cloned expansion split उसी vocal identity को ज्यादा varied arrangements, captions, lyrics, और song structures में दिखा सकता है। + +यह feature सिर्फ audio datasets के लिए है। + +!!! warning "Consent और rights" + इस workflow का उपयोग केवल उन voices और recordings पर करें जिन्हें इस्तेमाल करने की अनुमति आपके पास है। Voice identity sensitive biometric और creative data है। Transform ऐसा derivative audio बना सकता है जो किसी real person जैसा सुनाई दे, इसलिए permission, licensing, और disclosure जरूरी हैं। + +## ELI5 + +मान लीजिए आपके पास एक singer की छह recordings हैं, लेकिन सभी एक ही band और genre में हैं। अगर आप सिर्फ उन्हीं songs पर train करते हैं, model singer, guitar tone, drums, tempo range, और song structure को एक ही चीज मानकर सीख सकता है। + +Voice cloning data transforms इन ideas को अलग करने की कोशिश करते हैं: + +1. Singer examples से छोटा voice-conversion model सीखना। +2. ज्यादा broad songs या vocal stems लेना। +3. Source vocal timbre को target singer timbre से बदलना। +4. नए captions और lyrics को generated audio से aligned रखना। +5. Generated audio को एक और normal training split की तरह add करना। + +फिर main model target voice को ज्यादा contexts में देखता है, सिर्फ original narrow dataset याद नहीं करता। + +## कब उपयोग करें + +Use करें जब: + +- आपके पास target vocalist की permissioned recordings हों +- target identity एक genre, band, production style, या song structure से बहुत entangled हो +- trigger words सिर्फ original domain में काम करें +- एक dataset में कई singers हों और model averaged voice बना रहा हो +- आप अलग vocal identities के लिए अलग LoRAs चाहते हों +- आप चाहते हों कि SimpleTuner उसी training setup में expanded split तैयार करे + +Avoid करें जब: + +- उसी voice का large, varied, clean dataset पहले से हो +- source expansion audio low quality हो या captions से aligned न हो +- public release के लिए clear rights न हों +- base generative model clean direct examples से भी target identity न सीख पाए + +## Training में कैसे फिट होता है + +Voice cloning data preparation transform है, conditioning dataset नहीं। + +`conditioning_data` paired auxiliary inputs के लिए है जो training के दौरान primary sample से जुड़े रहते हैं, जैसे reference images या generated conditioning maps। + +Voice cloning dataset-level `data_transforms` list में रहना चाहिए। Transform नए audio files, captions, और optional lyrics materialize करता है, फिर result को दूसरे primary `audio` dataset की तरह register करता है। उसके बाद normal dataloader इसे किसी भी training split जैसा देखता है। + +Pseudo config shape: + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +Pseudo startup behavior: + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## RVC-Style Identity Transfer + +पहली intended implementation RVC-style voice conversion है। + +इस context में "RVC model" voice-specific है। यह target identity dataset से train होता है। Retrieval index भी voice-specific है और उसी target voice की features से बनता है। Content features, pitch extraction, या separation models जैसे broad pretrained components reusable infrastructure हैं; conversion model और index singer या speaker-specific artifacts हैं। + +SimpleTuner को ये कर पाना चाहिए: + +1. Provided voice-conversion model और index reuse करना। +2. Model न मिलने पर voice-conversion model train करना। +3. Target voice data से retrieval index build करना। +4. Training output directory में model, index, और generated audio cache करना। +5. Source data और transform settings न बदलें तो startup पर cached artifacts reuse करना। +6. Optional रूप से Hub model repository से voice-conversion model reuse या publish करना। + +## Default Behavior + +Planned defaults conservative हैं: + +| Setting | Default | Why | +| --- | --- | --- | +| `task` | `identity_transfer` | Transform को clearly identify करता है। | +| `method` | `rvc` | पहला supported voice-transfer backend। | +| `train_if_missing` | `true` | SimpleTuner target dataset से voice model bootstrap कर सके। | +| `force_retrain` | `false` | Valid cached voice model reuse हो सके। | +| `build_index` | `true` | Retrieval identity stability improve कर सकता है और leakage घटा सकता है। | +| `hub_model_id` | unset | User opt-in न करे तो remote voice-model cache use नहीं होती। | +| `reuse_from_hub` | `hub_model_id` set हो तो `true` | On-demand model training से पहले Hub check करता है। | +| `push_to_hub` | `false` | Voice model upload explicit होना चाहिए क्योंकि artifact voice identity represent करता है। | +| `audio_mode` | full songs के लिए `separate_convert_remix`, vocal stems के लिए `vocal_only` | Full mixes को separation चाहिए; stems को नहीं। | +| `separation_method` | separation चाहिए तो `demucs` | Demucs expected default stem separator है। | +| generated split type | primary `audio` dataset | Generated data normal audio की तरह train होता है, conditioning नहीं। | +| cache location | `output_dir` के अंदर | Artifacts training run से जुड़े रहते हैं और restart पर reuse हो सकते हैं। | +| captions | configured न हो तो source captions copy | नया split lyrics और arrangement context preserve करे। | + +अगर existing voice-conversion model दिया गया है, SimpleTuner को उसे use करना चाहिए और नया model सिर्फ explicit request या missing required artifacts पर train करना चाहिए। + +## Hub Cache + +Voice-conversion model इतना expensive हो सकता है कि repeated on-demand training user के लिए footgun बन जाए। इसलिए transform को voice model और retrieval index के लिए optional Hub-backed cache support करना चाहिए। + +Safe lookup order: + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +Hub repository को loose files की जगह SimpleTuner-specific layout use करना चाहिए: + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +Manifest contract है। इसमें target identity dataset fingerprint, RVC training settings, index settings, expected sample rate, tool versions, और SimpleTuner voice-transform format version record होने चाहिए। SimpleTuner को ऐसा Hub artifact reuse नहीं करना चाहिए जिसमें manifest न हो या manifest current transform से match न करे। इससे गलत voice model को नए dataset पर silently apply करने से बचते हैं। + +Publishing opt-in होना चाहिए। Reasonable pseudo config: + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +Private identities के लिए Hub repository private रखें जब तक voice model publish करने की explicit permission न हो। Generated audio और model artifacts के sharing rights अलग हो सकते हैं, इसलिए upload settings अलग-अलग treat करें। + +## WebUI Configuration + +RVC model training WebUI से configurable होनी चाहिए, सिर्फ raw dataloader JSON से नहीं। + +Expected WebUI shape audio datasets के लिए dataset transform editor है: + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +WebUI को दो common setups साफ दिखाने चाहिए: + +- **Vocal stems पहले से हैं:** `vocal_only` चुनें, Demucs disabled रखें, और generated vocal stems लिखें। +- **Full songs हैं:** `separate_convert_remix` चुनें, Demucs separation use करें, सिर्फ vocal stem convert करें, और original instrumental stems के साथ remix करें। + +Interface को दिखाना चाहिए कि generated audio दूसरा primary audio training split बनता है। Identity transfer को `conditioning_data` की तरह present नहीं करना चाहिए, क्योंकि इससे training के दौरान paired conditioning behavior का गलत संकेत मिलेगा। + +## Distributed Startup Behavior + +जब SimpleTuner multiple data-parallel ranks के साथ start होता है, voice cloning startup को available GPUs use करनी चाहिए, rank 0 से सारा काम नहीं करवाना चाहिए। + +दो अलग distributed phases हैं: + +1. **RVC model training:** अगर `train_if_missing=true` है, matching local cache नहीं है, और matching Hub artifact भी नहीं है, तो `world_size > 1` होने पर RVC training loop DDP में चलना चाहिए। हर rank normal distributed sampler pattern से अलग target-voice batches ले। +2. **Generated audio preparation:** source expansion inputs rank के हिसाब से split होने चाहिए, TextEmbedCache और VAECache जैसे। हर rank सिर्फ अपना shard separate, convert, और write करे, फिर सभी ranks synchronize हों और metadata discovery continue हो। + +Pseudo behavior: + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +Final voice model को Hub पर सिर्फ एक process publish करे। Final manifest updates के लिए भी यही rule है। Per-rank generated outputs independently लिखे जा सकते हैं, जब filenames deterministic और non-overlapping हों। + +इससे multi-GPU systems पर GPU time waste नहीं होता और startup behavior SimpleTuner के existing cache preparation model से aligned रहता है। + +## RVC Training Logs + +Startup RVC training अभी TensorBoard या WandB runs create नहीं करनी चाहिए। ये loggers main SimpleTuner training job के लिए configured होते हैं, और nested voice-conversion job के लिए reuse करने पर extra run names, paths, resume rules, और artifact policies चाहिए होंगी। + +RVC stage फिर भी SimpleTuner native training logger से useful stats report कर सकता है: + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +Useful local stats में RVC training loss, pitch loss अगर enabled हो, reconstruction या discriminator loss जब लागू हो, samples processed, elapsed time, DDP world size, cache hit या miss reason, और final model local cache, Hub cache, या on-demand training से आया या नहीं शामिल हैं। + +जब तक future implementation RVC transforms के लिए external logger integration explicitly add न करे, ये stats local-only हैं। + +## `audio_mode` चुनना + +### `vocal_only` + +जब expansion dataset पहले से clean vocal stems में preprocessed हो, इसे use करें। + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +Gotchas: + +- Clean stems पर Demucs दोबारा न चलाएं जब तक कोई साफ कारण न हो। +- Captions vocals और lyrics describe करें; full band arrangement नहीं, जब तक आप बाद में remix नहीं करेंगे। +- अगर main training model full songs expect करता है, vocal-only generated data अलग distribution सिखा सकता है। + +### `separate_convert_remix` + +जब expansion dataset full mixed songs हो, इसे use करें। + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +Full-song expansion के लिए यह preferred mode है क्योंकि drums, bass, guitars, room sound, और mastering artifacts को voice का हिस्सा मानकर convert करने से बचता है। + +Gotchas: + +- Stem separation bleed, artifacts, या phase issues छोड़ सकता है। +- Vocal stem weak, reverberant, या buried हो तो converted voice unstable हो सकती है। +- Remix loudness मायने रखता है। Generated split हमेशा ज्यादा loud या quiet हो तो training bias हो सकता है। +- Captions final remixed result describe करें, सिर्फ source song नहीं। + +### `full_mix_convert` + +इसे सिर्फ quick tests के लिए use करें। + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +यह fast है, लेकिन usually lower quality है। Instruments voice converter से गुजर सकते हैं और final LoRA unwanted artifacts सीख सकती है। + +## Captions और Lyrics Policy + +Generated split की captions generated audio से match करनी चाहिए। + +अच्छा default: + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +Lyrics copy करना usually सही है जब source vocal performance और converted performance वही words गाते हैं। अगर transform song बदलता है, sections edit करता है, vocals drop करता है, या non-lyrical source use करता है, तो copy सही नहीं है। + +Captions blind copy नहीं होनी चाहिए। अगर source caption कहती है "female pop vocal" और converted output male rock vocal timbre है, caption adjust होनी चाहिए। Transform simple append/remove rules support करे; advanced caption rewriting बाद में add हो सकती है। + +## Cache और Reuse + +Transform को दो तरह की cache लिखनी चाहिए: + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +Manifest में identity dataset fingerprint, transform settings, source expansion data fingerprint, और tool versions record होने चाहिए। Values match करें तो startup existing artifacts reuse कर सकता है। Values change हों तो SimpleTuner affected stage regenerate करे। + +## Practical Dataset Advice + +- Identity control important हो तो एक LoRA में एक target vocalist रखें। +- Voice-conversion model के लिए clean, dry vocal examples prefer करें। +- Duets avoid करें, जब तक goal duet blend सीखना न हो। +- Expansion songs में tempo, key, genre, dynamics, और lyrical phrasing varied रखें। +- Captions varied रखें ताकि identity tokens एक arrangement से चिपक न जाएं। +- Long training runs से पहले generated audio spot-check करें। + +## Common Failure Modes + +| Symptom | Likely cause | +| --- | --- | +| LoRA सिर्फ एक genre में काम करती है | Voice identity अभी भी arrangement captions या source data से entangled है। | +| Generated split hollow या phasey सुनाई देता है | Full-song processing में separation/remix artifacts। | +| Instruments voice-converted जैसे सुनते हैं | Separation चाहिए थी लेकिन `full_mix_convert` use हुआ। | +| Vocal identity weak है | Voice-conversion model को cleaner target data, more data, या stronger retrieval index चाहिए। | +| Captions voice control नहीं करतीं | Captions में source-vocal identity बची है या target identity missing है। | +| Main model artifacts सीखता है | Generated audio low quality है या train mix में बहुत dominant है। | + +## Regularisation Data से संबंध + +Generated identity-transfer data default रूप से regularisation data नहीं है। + +Regularisation data usually LoRA को base model behavior preserve करना सिखाता है। Identity-transfer data LoRA को target voice ज्यादा contexts में सिखाता है। Too much regularisation और too little direct identity data identity tokens कमजोर कर सकते हैं। Too much generated data conversion artifacts सिखा सकता है। + +इन्हें अलग controls मानें: + +- direct target dataset: सबसे strong identity signal +- generated identity-transfer dataset: broader context और style coverage +- regularisation dataset: base-model preservation + +## Status + +यह page experimental `data_transforms` workflow के intended user-facing behavior को describe करता है। Important design constraint यह है कि identity transfer SimpleTuner audio training की first-class feature बने: voice-conversion model train या reuse करना, retrieval index build या reuse करना, expanded split generate करना, results cache करना, और separate manual preprocessing stage के बिना normal training में continue करना। diff --git a/documentation/experimental/VOICE_CLONING.ja.md b/documentation/experimental/VOICE_CLONING.ja.md new file mode 100644 index 000000000..febea5683 --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.ja.md @@ -0,0 +1,359 @@ +# 音声クローニング Data Transform + +音声クローニング data transform は、メインのモデル学習が始まる前に、あるボーカル identity を追加の曲、stem、歌唱へ転写して学習セットを拡張するための、計画中の実験的な音声データセット機能です。 + +目的は SimpleTuner を別の voice conversion 作業環境にすることではありません。音声 fine-tuning データの絡み合いを減らすことです。ある歌手が狭いスタイルにしか出てこない場合、LoRA は歌手本人ではなく「この編曲の中のこの歌手」を学んでしまうことがあります。voice-cloned expansion split により、同じ声をより多様な編曲、caption、歌詞、曲構造で見せられます。 + +この機能は音声データセット専用です。 + +!!! warning "同意と権利" + この workflow は、使用する権利がある声と録音だけに使ってください。声の identity はセンシティブな生体情報であり創作データです。この transform は実在人物のように聞こえる派生音声を作れるため、許可、ライセンス、開示が重要です。 + +## ELI5 + +同じ歌手の録音が 6 曲あるとして、その全てが同じバンド、同じジャンルだとします。それだけで学習すると、モデルは歌手、ギター音色、ドラムの感触、テンポ範囲、曲構造を一体のものとして覚えるかもしれません。 + +音声クローニング data transform は、それらを分けるためのものです。 + +1. 歌手サンプルから小さな voice-conversion model を学習します。 +2. より広い曲や vocal stem のセットを読み込みます。 +3. 元のボーカル timbre を対象歌手の timbre に置き換えます。 +4. 新しい caption と歌詞を生成音声に合わせます。 +5. 生成音声を通常の training split として追加します。 + +これにより、メインモデルは元の狭いデータセットを暗記するだけでなく、より多くの文脈で対象の声を見ることができます。 + +## 使う場面 + +使うべき場面: + +- 対象ボーカリストの許可された録音がある +- 対象 identity が単一ジャンル、バンド、制作スタイル、曲構造と絡みすぎている +- trigger word が元のドメイン内でしか効かない +- 1 つのデータセットに複数の歌手がいて平均化された声になる +- 声ごとに別々の LoRA を作りたい +- SimpleTuner に同じ training setup の中で expansion split を準備してほしい + +避けるべき場面: + +- 同じ声の大規模で多様かつクリーンなデータが既にある +- expansion source の音質が悪い、または caption と合っていない +- 公開リリース向けの権利が明確でない +- クリーンな直接サンプルでもベース生成モデルが対象 identity を学べない + +## Training への入り方 + +音声クローニングはデータ準備 transform であり、conditioning dataset ではありません。 + +`conditioning_data` は、reference image や generated conditioning map のように、training 中に主サンプルへ付いたままになる補助入力のためのものです。 + +音声クローニングは、dataset レベルの `data_transforms` リストに入るべきです。transform は新しい音声ファイル、caption、必要なら歌詞を materialize し、その結果を別の primary `audio` dataset として登録します。その後、通常の dataloader は他の training split と同じように扱います。 + +疑似 config: + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +起動時の疑似処理: + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## RVC 形式の Identity Transfer + +最初に想定している実装は RVC 形式の voice conversion です。 + +ここでいう「RVC model」は voice-specific です。対象 identity dataset から学習されます。retrieval index も voice-specific で、同じ対象声の特徴から構築されます。content feature、pitch 抽出、separation model などの広い pretrained component は再利用される基盤です。一方、conversion model と index は歌手または話者固有の artifact です。 + +SimpleTuner は以下を行えるべきです。 + +1. 提供済みの voice-conversion model と index を再利用する。 +2. model がなければ voice-conversion model を学習する。 +3. 対象声データから retrieval index を構築する。 +4. model、index、生成音声を training output directory 以下に cache する。 +5. source data と transform 設定が変わっていなければ起動時に cache を再利用する。 +6. 必要に応じて Hub model repository から voice-conversion model を再利用、またはそこへ publish する。 + +## Default Behavior + +計画中の default は保守的です。 + +| Setting | Default | 理由 | +| --- | --- | --- | +| `task` | `identity_transfer` | transform を明示します。 | +| `method` | `rvc` | 最初の voice-transfer backend です。 | +| `train_if_missing` | `true` | SimpleTuner が対象 dataset から voice model を bootstrap できるようにします。 | +| `force_retrain` | `false` | 有効な cache model をできるだけ再利用します。 | +| `build_index` | `true` | retrieval は identity の安定性を上げ、漏れを減らしやすいです。 | +| `hub_model_id` | unset | user が opt-in しない限り remote voice-model cache は使いません。 | +| `reuse_from_hub` | `hub_model_id` が設定されている場合は `true` | on-demand model を学習する前に Hub を確認します。 | +| `push_to_hub` | `false` | voice model は声の identity を表すため、upload は明示的であるべきです。 | +| `audio_mode` | full song は `separate_convert_remix`、vocal stem は `vocal_only` | full mix には分離が必要で、stem には不要です。 | +| `separation_method` | 分離が必要なら `demucs` | Demucs が想定 default の stem separator です。 | +| generated split type | primary `audio` dataset | 生成データは conditioning ではなく通常音声として学習します。 | +| cache location | `output_dir` 内 | artifact を training run に結び付け、restart で再利用しやすくします。 | +| captions | 設定がなければ source captions を copy | 新しい split は歌詞と編曲文脈を保持すべきです。 | + +既存の voice-conversion model が指定されている場合、SimpleTuner はそれを使い、明示的に要求された場合や必要 artifact が欠けている場合だけ新規学習すべきです。 + +## Hub Cache + +voice-conversion model は再学習コストが高くなることがあり、毎回 on-demand training すると無駄が大きくなります。そのため transform は、voice model と retrieval index の optional Hub-backed cache をサポートすべきです。 + +安全な lookup order: + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +Hub repository は、単なる file collection ではなく SimpleTuner-specific layout を使うべきです。 + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +manifest が contract です。target identity dataset fingerprint、RVC training settings、index settings、expected sample rate、tool versions、SimpleTuner voice-transform format version を記録します。manifest がない、または current transform と一致しない Hub artifact を SimpleTuner は再利用すべきではありません。これにより、間違った voice model を新しい dataset に黙って適用する事故を避けられます。 + +publish は opt-in にします。妥当な pseudo config: + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +private identity の場合、明示的な許可がない限り Hub repository は private のままにしてください。generated audio と model artifact は共有権利が異なる場合があるため、upload settings は別々に扱います。 + +## WebUI Configuration + +RVC model training は raw dataloader JSON だけでなく WebUI から設定できるべきです。 + +想定される WebUI 形状は、audio dataset 用の dataset transform editor です。 + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +WebUI はよくある 2 つの setup を分かりやすくするべきです。 + +- **既に vocal stems がある場合:** `vocal_only` を選び、Demucs は無効のまま、generated vocal stems を書き出します。 +- **full songs がある場合:** `separate_convert_remix` を選び、Demucs separation を使い、vocal stem だけを変換して original instrumental stems と remix します。 + +interface は、generated audio が別の primary audio training split になることを示すべきです。identity transfer を `conditioning_data` として見せてはいけません。training 中の paired conditioning behavior だと誤解されるためです。 + +## Distributed Startup Behavior + +SimpleTuner が複数の data-parallel rank で起動する場合、voice cloning startup は rank 0 だけに作業させるのではなく、利用可能な GPU を使うべきです。 + +分散処理には 2 つの段階があります。 + +1. **RVC model training:** `train_if_missing=true` で、matching local cache がなく、matching Hub artifact もない場合、`world_size > 1` なら RVC training loop は DDP で動くべきです。各 rank は通常の distributed sampler pattern で異なる target-voice batch を受け取ります。 +2. **Generated audio preparation:** expansion source inputs は TextEmbedCache や VAECache と同じように rank ごとに分割します。各 rank は自分の shard だけを separate、convert、write し、全 rank が同期してから metadata discovery を続けます。 + +疑似処理: + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +final voice model を Hub に publish する process は 1 つだけにします。final manifest update も同じです。per-rank generated outputs は、filenames が deterministic で重複しない限り、各 rank が独立して書けます。 + +これにより multi-GPU system で GPU 時間を無駄にせず、startup behavior を SimpleTuner の既存 cache preparation model と揃えられます。 + +## RVC Training Logs + +startup RVC training は、現時点では TensorBoard や WandB run を作成すべきではありません。これらの logger は main SimpleTuner training job 用に設定されており、nested voice-conversion job に再利用すると、追加の run names、paths、resume rules、artifact policies が必要になります。 + +RVC stage は SimpleTuner native training logger を通して有用な stats を記録できます。 + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +有用な local stats には、RVC training loss、pitch loss が有効な場合の値、該当する reconstruction/discriminator loss、processed samples、elapsed time、DDP world size、cache hit/miss reason、final model が local cache、Hub cache、on-demand training のどれから来たかが含まれます。 + +これらの stats は、将来 RVC transforms 向けの external logger integration が明示的に追加されるまでは local-only です。 + +## `audio_mode` の選び方 + +### `vocal_only` + +expansion dataset が既にクリーンな vocal stem に preprocessing 済みの場合に使います。 + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +注意点: + +- 理由がない限り、クリーンな stem に Demucs を再実行しないでください。 +- caption は vocal と lyrics を説明します。後で伴奏へ remix しないなら、full band arrangement として説明しないでください。 +- メイン training model が full song を期待する場合、vocal-only 生成データは別の分布を教える可能性があります。 + +### `separate_convert_remix` + +expansion dataset が full mixed song の場合に使います。 + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +これは full-song expansion の推奨 mode です。drums、bass、guitars、room sound、mastering artifact を声の一部として変換するのを避けられます。 + +注意点: + +- stem separation は bleed、artifact、phase 問題を残すことがあります。 +- vocal stem が弱い、reverb が多い、埋もれている場合、変換声が不安定になります。 +- remix loudness は重要です。生成 split が常に大きすぎたり小さすぎたりすると training に偏りが出ます。 +- caption は source song だけでなく最終 remix 結果を説明するべきです。 + +### `full_mix_convert` + +簡単なテストだけに使います。 + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +高速ですが、通常は品質が低くなります。楽器まで voice converter に通してしまい、不要な artifact を最終 LoRA に教える可能性があります。 + +## Caption と Lyrics + +生成 split の caption は、生成された音声と一致している必要があります。 + +よい default: + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +source vocal performance と converted performance が同じ歌詞を歌う場合、lyrics の copy は普通は正しいです。曲を変える、section を落とす、vocal を削除する、非 lyrics source を使う場合は正しくありません。 + +caption は盲目的に copy できません。source caption が "female pop vocal" で、変換後が male rock vocal timbre なら調整が必要です。transform は単純な append/remove rules を持つべきで、高度な caption rewrite は後から重ねられます。 + +## Cache と Reuse + +transform は 2 種類の cache を書くべきです。 + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +manifest には identity dataset fingerprint、transform settings、source expansion data fingerprint、tool versions を記録します。一致すれば起動時に既存 artifact を再利用できます。変わった場合は、影響を受ける stage だけ再生成します。 + +## Dataset Advice + +- identity control が重要なら、1 つの LoRA に target vocalist は 1 人だけにします。 +- voice-conversion model にはクリーンで dry な vocal examples を優先します。 +- duet blend を学びたい場合以外は duet を避けます。 +- expansion songs は tempo、key、genre、dynamics、lyrical phrasing を広くします。 +- caption を十分に多様化し、identity token が 1 つの編曲に貼り付かないようにします。 +- 長い training run の前に生成音声を spot-check します。 + +## Common Failure Modes + +| 症状 | ありがちな原因 | +| --- | --- | +| LoRA が 1 ジャンルでしか効かない | 声 identity が arrangement caption や source data とまだ絡んでいます。 | +| generated split が phasey または hollow に聞こえる | full-song 処理の separation/remix artifact。 | +| 楽器まで変声されたように聞こえる | 分離が必要なのに `full_mix_convert` を使っています。 | +| vocal identity が弱い | target data の品質、量、または retrieval index が不足しています。 | +| caption が声を制御しない | source vocal identity が caption に残っている、または target identity がありません。 | +| main model が artifact を学ぶ | generated audio の品質が低い、または train mix 内で強すぎます。 | + +## Regularisation Data との関係 + +生成された identity-transfer data は、default では regularisation data ではありません。 + +regularisation data は通常、LoRA に base model の挙動を保たせるためのものです。identity-transfer data は、より多くの文脈で target voice を教えるためのものです。regularisation が多すぎて direct identity data が少なすぎると identity token が弱くなります。generated data が多すぎると conversion artifact を教えることがあります。 + +別々のレバーとして扱ってください。 + +- direct target dataset: 最も強い identity signal +- generated identity-transfer dataset: より広い context と style coverage +- regularisation dataset: base-model preservation + +## Status + +このページは、実験的な `data_transforms` workflow の意図した user-facing behavior を説明しています。重要な設計制約は、identity transfer を SimpleTuner の first-class audio training feature にすることです。voice-conversion model の学習または再利用、retrieval index の構築または再利用、expanded split の生成、結果の cache、そして別の手動 preprocessing stage なしで通常 training へ進むことを目指します。 diff --git a/documentation/experimental/VOICE_CLONING.md b/documentation/experimental/VOICE_CLONING.md new file mode 100644 index 000000000..661b50b4d --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.md @@ -0,0 +1,366 @@ +# Voice Cloning Data Transforms + +Voice cloning data transforms are a planned experimental audio-dataset feature for expanding a training set by transferring one vocal identity onto additional songs, stems, or performances before the main model training begins. + +The goal is not to make SimpleTuner a separate voice-conversion workstation. The goal is to make audio fine-tuning datasets less entangled. If a singer only appears in one narrow style, a LoRA can learn "this singer inside this arrangement" instead of the singer identity itself. A voice-cloned expansion split can place the same vocal identity across more varied arrangements, captions, lyrics, and song structures. + +This feature is intended for audio datasets only. + +!!! warning "Consent and rights" + Use this workflow only with voices and recordings you are allowed to use. A voice identity is sensitive biometric and creative data. The transform can make derivative audio that sounds like a real person, so permission, licensing, and disclosure matter. + +## ELI5 + +Imagine you have six recordings of one singer, but every recording is in the same band and genre. If you train on only those songs, the model may learn that the singer, guitar tone, drum feel, tempo range, and song structure are one inseparable thing. + +Voice cloning data transforms try to separate those ideas: + +1. Learn a small voice-conversion model from the singer examples. +2. Take a broader set of songs or vocal stems. +3. Replace the source vocal timbre with the target singer timbre. +4. Keep the new captions and lyrics aligned to the new generated audio. +5. Add the generated audio as another normal training split. + +The main model then sees the target voice in more contexts instead of only memorizing the original narrow dataset. + +## What This Is For + +Use voice cloning transforms when: + +- you have permissioned recordings for a target vocalist +- the target identity is too entangled with one genre, band, production style, or song structure +- prompts work only in-domain and fail when the genre changes +- two or more singers in one dataset blend into an averaged voice +- you want separate LoRAs for separate vocal identities +- you want SimpleTuner to prepare the expanded split as part of the same training setup + +Do not use it when: + +- you already have a large, varied, clean dataset for the same voice +- the source expansion audio is low quality or badly aligned with captions +- you need legally clean public release material and do not have explicit rights +- the base generative model cannot learn the target identity even from clean direct examples + +## How It Fits Into Training + +Voice cloning is a data preparation transform, not a conditioning dataset. + +`conditioning_data` is for paired auxiliary inputs that stay attached to a primary sample during training, such as reference images or generated conditioning maps. + +Voice cloning should instead live under a dataset-level `data_transforms` list. The transform materializes new audio files, captions, and optional lyrics, then registers the result as another primary `audio` dataset. After that point, the normal dataloader sees it like any other training split. + +Pseudo config shape: + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +Pseudo startup behavior: + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## RVC-Style Identity Transfer + +The first intended implementation is RVC-style voice conversion. + +In this context, the "RVC model" is voice-specific. It is trained from the target identity dataset. The retrieval index is also voice-specific and is built from features from the same target voice. Broad pretrained components, such as content features, pitch extraction, or separation models, are reusable infrastructure; the conversion model and index are the artist- or speaker-specific artifacts. + +SimpleTuner should be able to: + +1. Reuse a provided voice-conversion model and index. +2. Train the voice-conversion model if no model is provided. +3. Build the retrieval index from the target voice data. +4. Cache the model, index, and generated audio under the training output directory. +5. Reuse cached artifacts at startup when the source data and transform settings have not changed. +6. Optionally reuse or publish the voice-conversion model through a Hub model repository. + +## Default Behavior + +The planned defaults are conservative: + +| Setting | Default | Why | +| --- | --- | --- | +| `task` | `identity_transfer` | Explicitly identifies the transform. | +| `method` | `rvc` | The first supported voice-transfer backend. | +| `train_if_missing` | `true` | SimpleTuner should be able to bootstrap the voice model from the target dataset. | +| `force_retrain` | `false` | Reuse a valid cached voice model when possible. | +| `build_index` | `true` | Retrieval usually improves identity stability and reduces leakage. | +| `hub_model_id` | unset | No remote voice-model cache is used unless the user opts in. | +| `reuse_from_hub` | `true` when `hub_model_id` is set | Check the Hub before spending time training an on-demand model. | +| `push_to_hub` | `false` | Uploading a voice model should be explicit because the artifact represents a voice identity. | +| `audio_mode` | `separate_convert_remix` for full songs, `vocal_only` for vocal stems | Full mixes need separation; stems do not. | +| `separation_method` | `demucs` when separation is needed | Demucs is the expected default stem separator. | +| generated split type | primary `audio` dataset | The generated data is trained like normal audio, not used as conditioning. | +| cache location | inside `output_dir` | Keeps generated artifacts tied to the training run and reusable on restart. | +| captions | copy source captions unless configured otherwise | The new split should preserve lyrics and arrangement context. | + +If an existing voice-conversion model is supplied, SimpleTuner should use it and only train a new model when explicitly requested or when required artifacts are missing. + +## Hub Cache + +A voice-conversion model can be expensive enough that repeated on-demand training becomes a footgun. The transform should therefore support an optional Hub-backed cache for the voice model and retrieval index. + +The safe lookup order is: + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +The Hub repository should use a SimpleTuner-specific layout rather than a loose collection of files: + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +The manifest is the contract. It should record the target identity dataset fingerprint, RVC training settings, index settings, expected sample rate, tool versions, and the SimpleTuner voice-transform format version. SimpleTuner should not reuse a Hub artifact that lacks this manifest or whose manifest does not match the current transform. That avoids silently applying the wrong voice model to a new dataset. + +Publishing should be opt-in. A reasonable pseudo config is: + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +For private identities, keep the Hub repository private unless you have explicit permission to publish the voice model. Generated audio and model artifacts may have different sharing rights, so treat their upload settings separately. + +## WebUI Configuration + +RVC model training should be configurable from the WebUI, not only through raw dataloader JSON. + +The expected WebUI shape is a dataset transform editor for audio datasets: + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +The WebUI should make the two common setups obvious: + +- **Already have vocal stems:** choose `vocal_only`, leave Demucs disabled, and write generated vocal stems. +- **Have full songs:** choose `separate_convert_remix`, use Demucs separation, convert only the vocal stem, and remix with the original instrumental stems. + +The interface should show that generated audio becomes another primary audio training split. It should not present identity transfer as `conditioning_data`, because that would imply paired conditioning behavior during training. + +## Distributed Startup Behavior + +When SimpleTuner starts with multiple data-parallel ranks, voice cloning startup should use the available GPUs instead of making rank 0 do all of the work. + +There are two separate distributed phases: + +1. **RVC model training:** if `train_if_missing=true`, no matching local cache exists, and no matching Hub artifact is available, the RVC training loop should run under DDP when `world_size > 1`. Each rank should receive different target-voice batches through the normal distributed sampler pattern. +2. **Generated audio preparation:** source expansion inputs should be split by rank, similar to TextEmbedCache and VAECache. Each rank separates, converts, and writes only its assigned shard, then all ranks synchronize before metadata discovery continues. + +Pseudo behavior: + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +Only one process should publish the final voice model to the Hub. The same applies to final manifest updates. Per-rank generated outputs can be written independently as long as filenames are deterministic and non-overlapping. + +This avoids wasting GPU time on multi-GPU systems and keeps startup behavior aligned with SimpleTuner's existing cache preparation model. + +## RVC Training Logs + +Startup RVC training should not create TensorBoard or WandB runs yet. Those loggers are configured for the main SimpleTuner training job, and reusing them for a nested voice-conversion job would require extra run names, paths, resume rules, and artifact policies. + +The RVC stage can still report useful stats through SimpleTuner's native training logger: + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +Useful local stats include RVC training loss, pitch loss if enabled, reconstruction or discriminator loss when applicable, samples processed, elapsed time, DDP world size, cache hit or miss reason, and whether the final model came from local cache, Hub cache, or on-demand training. + +These stats are local-only unless a future implementation explicitly adds external logger integration for RVC transforms. + +## Choosing `audio_mode` + +### `vocal_only` + +Use this when your expansion dataset is already preprocessed into clean vocal stems. + +This is the simplest and least destructive mode: + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +Gotchas: + +- Do not run Demucs again on clean stems unless you have a reason. +- Captions should describe vocals and lyrics, not a full band arrangement, unless you will later remix the stem. +- If the main training model expects full songs, vocal-only generated data may teach a different distribution than your full-song dataset. + +### `separate_convert_remix` + +Use this when your expansion dataset contains full mixed songs. + +The expected flow is: + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +This is the preferred mode for full-song expansion because it avoids converting drums, bass, guitars, room sound, and mastering artifacts as if they were part of the voice. + +Gotchas: + +- Stem separation can leave bleed, artifacts, or phase issues. +- If the vocal stem is weak, reverberant, or buried, the converted voice can sound unstable. +- Remix loudness matters. A generated split that is consistently louder or quieter than the original data can bias training. +- Captions should describe the final remixed result, not only the source song. + +### `full_mix_convert` + +Use this only for quick tests. + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +This is fast but usually lower quality. It can drag instruments through the voice converter and teach unwanted artifacts to the final LoRA. + +## Caption and Lyrics Policy + +The generated split should have captions that match the generated audio. + +A good default is: + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +For lyrics, copying is usually correct when the source vocal performance and converted performance use the same words. It is not correct when the transform changes the song, edits sections, drops vocals, or uses a non-lyrical source. + +For captions, copying blindly can be wrong. If the source caption says "female pop vocal" and the converted output is a male rock vocal timbre, the caption should be adjusted. The transform should support simple append/remove rules, and more advanced caption rewriting can be layered later. + +## Cache and Reuse + +The transform should write two kinds of cache: + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +The manifest should record the identity dataset fingerprint, transform settings, source expansion data fingerprint, and tool versions. If those values match, startup can reuse the existing artifacts. If they change, SimpleTuner should regenerate the affected stage. + +This is important because voice-conversion training and full-song separation can be expensive. Restarting a main LoRA training run should not redo every generated audio file when nothing relevant changed. + +## Practical Dataset Advice + +- Keep one target vocalist per generated LoRA when identity control matters. +- Prefer clean, dry vocal examples for training the voice-conversion model. +- Avoid duets unless the goal is specifically to learn the duet blend. +- Use expansion songs with varied tempo, key, genre, dynamics, and lyrical phrasing. +- Keep captions varied enough that identity tokens and style descriptors are not glued to one arrangement. +- Spot-check generated audio before long training runs. +- Compare direct training data, generated data, and mixed training runs separately before combining everything. + +## Common Failure Modes + +| Symptom | Likely cause | +| --- | --- | +| The LoRA only works in one genre | Voice identity is still entangled with arrangement captions or source data. | +| The generated split sounds phasey or hollow | Separation/remix artifacts from full-song processing. | +| Instruments sound like they were voice-converted | `full_mix_convert` was used where separation was needed. | +| Vocal identity is weak | Voice-conversion model needs cleaner target data, more target data, or a stronger retrieval index. | +| Captions do not control the voice | Captions still mention source-vocal identity or omit the target identity. | +| The main model learns artifacts | Generated audio quality is too low or too dominant in the train mix. | + +## Relationship To Regularisation Data + +Generated identity-transfer data is not regularisation data by default. + +Regularisation data usually teaches the LoRA to preserve the base model's behavior. Identity-transfer data teaches the LoRA a target voice in more contexts. Mixing too much regularisation with too little direct identity data can weaken the identity tokens. Mixing too much generated data can teach conversion artifacts. + +Treat these as separate levers: + +- direct target dataset: strongest identity signal +- generated identity-transfer dataset: broader context and style coverage +- regularisation dataset: base-model preservation + +## Status + +This page describes the intended user-facing behavior for an experimental `data_transforms` workflow. The important design constraint is that identity transfer should become a first-class SimpleTuner audio training feature: train or reuse the voice-conversion model, build or reuse the retrieval index, generate the expanded split, cache the results, and then continue into normal training without requiring a separate manual preprocessing stage. diff --git a/documentation/experimental/VOICE_CLONING.pt-BR.md b/documentation/experimental/VOICE_CLONING.pt-BR.md new file mode 100644 index 000000000..0ccbd11a4 --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.pt-BR.md @@ -0,0 +1,359 @@ +# Transforms de Clonagem de Voz + +Transforms de clonagem de voz sao uma funcionalidade experimental planejada para datasets de audio. Eles expandem o conjunto de treino transferindo uma identidade vocal para musicas, stems ou performances adicionais antes do treino principal do modelo. + +O objetivo nao e transformar o SimpleTuner em uma estacao separada de voice conversion. O objetivo e reduzir o entrelacamento no dataset de audio. Se uma cantora ou cantor aparece apenas em um estilo muito estreito, uma LoRA pode aprender "essa voz dentro desse arranjo", em vez da identidade vocal em si. Um split expandido com voice cloning pode colocar a mesma identidade vocal em arranjos, captions, letras e estruturas musicais mais variados. + +Esta funcionalidade e apenas para datasets de audio. + +!!! warning "Consentimento e direitos" + Use este workflow somente com vozes e gravacoes que voce tem permissao para usar. Identidade vocal e dado biometrico e criativo sensivel. O transform pode criar audio derivado que soa como uma pessoa real, entao permissao, licenciamento e divulgacao importam. + +## ELI5 + +Imagine que voce tem seis gravacoes de uma pessoa cantando, mas todas sao da mesma banda e do mesmo genero. Se voce treina apenas nessas musicas, o modelo pode aprender que a voz, a guitarra, a bateria, o tempo e a estrutura da musica sao uma coisa so. + +Transforms de clonagem de voz tentam separar essas ideias: + +1. Aprendem um pequeno modelo de conversao de voz a partir dos exemplos da voz alvo. +2. Pegam um conjunto mais amplo de musicas ou vocal stems. +3. Substituem o timbre vocal de origem pelo timbre alvo. +4. Mantem as novas captions e letras alinhadas ao audio gerado. +5. Adicionam o audio gerado como outro split normal de treino. + +Assim o modelo principal ve a voz alvo em mais contextos, em vez de memorizar apenas o dataset estreito original. + +## Para Que Serve + +Use quando: + +- voce tem gravacoes autorizadas da voz alvo +- a identidade esta muito ligada a um unico genero, banda, estilo de producao ou estrutura musical +- trigger words funcionam apenas no dominio original +- duas ou mais vozes no mesmo dataset viram uma voz media +- voce quer LoRAs separadas para identidades vocais separadas +- voce quer que o SimpleTuner prepare o split expandido dentro do mesmo setup de treino + +Evite quando: + +- voce ja tem um dataset grande, variado e limpo da mesma voz +- o audio fonte de expansao tem baixa qualidade ou nao bate com as captions +- voce precisa publicar resultados e nao tem direitos claros +- o modelo generativo base nao aprende a identidade alvo nem com exemplos diretos limpos + +## Como Entra No Treino + +Clonagem de voz e um transform de preparacao de dados, nao um dataset de conditioning. + +`conditioning_data` e para entradas auxiliares pareadas que ficam anexadas a uma amostra primaria durante o treino, como imagens de referencia ou mapas de conditioning gerados. + +Clonagem de voz deve ficar em uma lista `data_transforms` no nivel do dataset. O transform materializa novos arquivos de audio, captions e letras opcionais, e registra o resultado como outro dataset primario `audio`. Depois disso, o dataloader normal o ve como qualquer outro split de treino. + +Formato de pseudo config: + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +Comportamento de startup em pseudocodigo: + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## Transferencia de Identidade Estilo RVC + +A primeira implementacao pretendida e conversao de voz estilo RVC. + +Neste contexto, o "modelo RVC" e especifico da voz. Ele e treinado a partir do dataset de identidade alvo. O indice de recuperacao tambem e especifico da voz e e construido com features da mesma voz alvo. Componentes pre-treinados amplos, como features de conteudo, extracao de pitch ou modelos de separacao, sao infraestrutura reutilizavel; o modelo de conversao e o indice sao artifacts especificos da cantora, cantor ou locutor. + +O SimpleTuner deve conseguir: + +1. Reutilizar um modelo de voice conversion e indice fornecidos. +2. Treinar o modelo de voice conversion se nenhum modelo for fornecido. +3. Construir o indice de recuperacao a partir dos dados da voz alvo. +4. Cachear modelo, indice e audio gerado no diretorio de saida do treino. +5. Reutilizar artifacts em cache no startup quando os dados e settings nao mudaram. +6. Opcionalmente reutilizar ou publicar o modelo de voice conversion por um repositorio de modelo no Hub. + +## Comportamento Padrao + +Os defaults planejados sao conservadores: + +| Setting | Default | Por que | +| --- | --- | --- | +| `task` | `identity_transfer` | Identifica explicitamente o transform. | +| `method` | `rvc` | Primeiro backend de transferencia vocal suportado. | +| `train_if_missing` | `true` | O SimpleTuner deve bootstrapar o modelo vocal a partir do dataset alvo. | +| `force_retrain` | `false` | Reutiliza um modelo em cache valido quando possivel. | +| `build_index` | `true` | Retrieval costuma melhorar estabilidade de identidade e reduzir vazamento. | +| `hub_model_id` | nao definido | Nenhum cache remoto de modelo vocal e usado sem opt-in do usuario. | +| `reuse_from_hub` | `true` quando `hub_model_id` esta definido | Verifica o Hub antes de gastar tempo treinando um modelo sob demanda. | +| `push_to_hub` | `false` | Upload de modelo vocal deve ser explicito porque o artifact representa uma identidade vocal. | +| `audio_mode` | `separate_convert_remix` para musicas completas, `vocal_only` para vocal stems | Mix completo precisa de separacao; stems nao. | +| `separation_method` | `demucs` quando separacao e necessaria | Demucs e o stem separator default esperado. | +| tipo do split gerado | dataset primario `audio` | Dados gerados treinam como audio normal, nao conditioning. | +| local de cache | dentro de `output_dir` | Mantem artifacts ligados ao treino e reutilizaveis no restart. | +| captions | copia captions da fonte salvo configuracao diferente | O novo split deve preservar letras e contexto de arranjo. | + +Se um modelo de voice conversion existente for fornecido, o SimpleTuner deve usa-lo e so treinar um novo quando isso for solicitado ou quando artifacts necessarios estiverem ausentes. + +## Cache no Hub + +Um modelo de voice conversion pode ser caro o suficiente para que treino sob demanda repetido vire uma armadilha. O transform deve entao suportar um cache opcional no Hub para o modelo vocal e o indice de retrieval. + +A ordem segura de busca e: + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +O repositorio no Hub deve usar um layout especifico do SimpleTuner, nao apenas arquivos soltos: + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +O manifest e o contrato. Ele deve registrar fingerprint do dataset de identidade alvo, settings de treino RVC, settings do indice, sample rate esperado, versoes das ferramentas e versao do formato voice-transform do SimpleTuner. O SimpleTuner nao deve reutilizar um artifact do Hub sem esse manifest ou com manifest que nao corresponde ao transform atual. Isso evita aplicar silenciosamente o modelo vocal errado a um novo dataset. + +Publicacao deve ser opt-in. Uma pseudo config razoavel: + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +Para identidades privadas, mantenha o repositorio do Hub privado salvo permissao explicita para publicar o modelo vocal. Audio gerado e artifacts de modelo podem ter direitos diferentes, entao trate seus settings de upload separadamente. + +## Configuracao no WebUI + +O treino do modelo RVC deve ser configuravel pelo WebUI, nao apenas por JSON bruto do dataloader. + +O formato esperado no WebUI e um editor de transforms no dataset de audio: + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +O WebUI deve deixar os dois setups comuns bem claros: + +- **Ja tem vocal stems:** escolha `vocal_only`, deixe Demucs desativado e escreva vocal stems gerados. +- **Tem musicas completas:** escolha `separate_convert_remix`, use separacao com Demucs, converta somente o vocal stem e faça remix com os stems instrumentais originais. + +A interface deve mostrar que o audio gerado vira outro split primario de treino de audio. Ela nao deve apresentar identity transfer como `conditioning_data`, porque isso sugeriria comportamento de conditioning pareado durante o treino. + +## Comportamento Distribuido no Startup + +Quando o SimpleTuner inicia com varios ranks data-parallel, o startup de voice cloning deve usar as GPUs disponiveis em vez de deixar rank 0 fazer todo o trabalho. + +Existem duas fases distribuidas separadas: + +1. **Treino do modelo RVC:** se `train_if_missing=true`, nao existe cache local correspondente e nao existe artifact correspondente no Hub, o loop de treino RVC deve rodar com DDP quando `world_size > 1`. Cada rank deve receber batches diferentes da voz alvo pelo padrao normal de distributed sampler. +2. **Preparacao do audio gerado:** entradas fonte de expansao devem ser divididas por rank, de forma parecida com TextEmbedCache e VAECache. Cada rank separa, converte e escreve apenas seu shard; depois todos os ranks sincronizam antes da metadata discovery continuar. + +Pseudocodigo: + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +Apenas um processo deve publicar o modelo vocal final no Hub. O mesmo vale para updates finais do manifest. Outputs gerados por rank podem ser escritos independentemente, desde que os nomes sejam deterministicos e nao sobrepostos. + +Isso evita desperdiçar tempo de GPU em sistemas multi-GPU e mantem o startup alinhado ao modelo existente de preparacao de cache do SimpleTuner. + +## Logs do Treino RVC + +O treino RVC no startup ainda nao deve criar runs do TensorBoard ou WandB. Esses loggers sao configurados para o job principal de treino do SimpleTuner, e reutiliza-los para um job aninhado de voice conversion exigiria nomes de run, paths, regras de resume e politicas de artifact extras. + +O estagio RVC ainda pode reportar stats uteis pelo logger nativo de treino do SimpleTuner: + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +Stats locais uteis incluem loss de treino RVC, pitch loss se ativado, reconstruction ou discriminator loss quando aplicavel, samples processados, tempo decorrido, DDP world size, motivo de cache hit ou miss, e se o modelo final veio de cache local, cache do Hub ou treino sob demanda. + +Esses stats sao apenas locais, a menos que uma implementacao futura adicione explicitamente integracao com logger externo para RVC transforms. + +## Escolhendo `audio_mode` + +### `vocal_only` + +Use quando seu dataset de expansao ja esta pre-processado em vocal stems limpos. + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +Gotchas: + +- Nao rode Demucs novamente em stems limpos sem motivo. +- Captions devem descrever vocais e letras, nao um arranjo de banda completo, a menos que voce va remixar depois. +- Se o modelo principal espera musicas completas, dados gerados vocal-only podem ensinar uma distribuicao diferente. + +### `separate_convert_remix` + +Use quando seu dataset de expansao contem musicas completas mixadas. + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +Este e o modo preferido para expansao de musicas completas, porque evita converter bateria, baixo, guitarras, sala e artifacts de masterizacao como se fossem parte da voz. + +Gotchas: + +- Separacao de stems pode deixar bleed, artifacts ou problemas de fase. +- Se o vocal stem for fraco, reverberante ou enterrado, a voz convertida pode ficar instavel. +- Loudness do remix importa. Um split gerado sempre mais alto ou mais baixo pode enviesar o treino. +- Captions devem descrever o resultado remixado final, nao apenas a musica fonte. + +### `full_mix_convert` + +Use somente para testes rapidos. + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +E rapido, mas geralmente tem menor qualidade. Pode arrastar instrumentos pelo conversor de voz e ensinar artifacts indesejados para a LoRA final. + +## Captions e Letras + +O split gerado deve ter captions que batem com o audio gerado. + +Um bom default: + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +Para letras, copiar costuma estar correto quando a performance fonte e a convertida usam as mesmas palavras. Nao esta correto quando o transform muda a musica, edita secoes, remove vocais ou usa uma fonte sem letra. + +Para captions, copiar cegamente pode estar errado. Se a caption fonte diz "female pop vocal" e a saida convertida tem timbre masculino de rock, a caption precisa mudar. O transform deve suportar regras simples de append/remove, e reescrita avancada pode vir depois. + +## Cache e Reuso + +O transform deve escrever dois tipos de cache: + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +O manifest deve registrar fingerprint do dataset de identidade, settings do transform, fingerprint dos dados fonte de expansao e versoes das ferramentas. Se esses valores baterem, o startup pode reutilizar artifacts existentes. Se mudarem, o SimpleTuner deve regenerar o estagio afetado. + +## Conselhos Praticos + +- Mantenha uma voz alvo por LoRA quando controle de identidade importa. +- Prefira exemplos vocais limpos e secos para treinar o modelo de voice conversion. +- Evite duetos, a menos que o objetivo seja aprender o blend do dueto. +- Use musicas de expansao com variedade de tempo, tom, genero, dinamica e fraseado. +- Varie captions para que tokens de identidade nao fiquem grudados em um unico arranjo. +- Ouça amostras geradas antes de treinos longos. + +## Falhas Comuns + +| Sintoma | Causa provavel | +| --- | --- | +| A LoRA so funciona em um genero | A identidade vocal ainda esta entrelacada com captions de arranjo ou dados fonte. | +| O split gerado soa oco ou com fase estranha | Artifacts de separacao/remix em processamento de musica completa. | +| Instrumentos parecem convertidos como voz | `full_mix_convert` foi usado quando separacao era necessaria. | +| Identidade vocal fraca | O modelo precisa de dados alvo mais limpos, mais dados ou indice melhor. | +| Captions nao controlam a voz | Captions ainda mencionam a voz fonte ou omitem a identidade alvo. | +| O modelo principal aprende artifacts | Audio gerado tem baixa qualidade ou peso demais no mix de treino. | + +## Relacao Com Dados de Regularizacao + +Dados gerados por identity transfer nao sao dados de regularizacao por default. + +Dados de regularizacao normalmente ensinam a LoRA a preservar o comportamento do modelo base. Dados de identity transfer ensinam a voz alvo em mais contextos. Regularizacao demais com poucos dados diretos de identidade pode enfraquecer os tokens de identidade. Dados gerados demais podem ensinar artifacts de conversao. + +Trate como controles separados: + +- dataset alvo direto: sinal de identidade mais forte +- dataset gerado por identity transfer: maior cobertura de contexto e estilo +- dataset de regularizacao: preservacao do modelo base + +## Status + +Esta pagina descreve o comportamento pretendido para um workflow experimental `data_transforms`. A restricao central de design e que identity transfer seja uma funcionalidade de audio de primeira classe no SimpleTuner: treinar ou reutilizar o modelo de voice conversion, construir ou reutilizar o indice, gerar o split expandido, cachear resultados e seguir para o treino normal sem exigir uma segunda etapa manual de preprocessing. diff --git a/documentation/experimental/VOICE_CLONING.zh.md b/documentation/experimental/VOICE_CLONING.zh.md new file mode 100644 index 000000000..d72c06b98 --- /dev/null +++ b/documentation/experimental/VOICE_CLONING.zh.md @@ -0,0 +1,359 @@ +# 语音克隆数据变换 + +语音克隆数据变换是一个计划中的实验性音频数据集功能。它会在主模型训练开始前，把一个目标人声身份迁移到更多歌曲、人声 stem 或表演上，从而扩展训练集。 + +它的目标不是把 SimpleTuner 变成独立的变声工作站，而是减少音频微调数据里的纠缠。如果某个歌手只出现在一种很窄的风格里，LoRA 可能学到的是“这个歌手在这种编曲里”，而不是歌手身份本身。通过语音克隆生成的扩展 split，可以让同一个人声身份出现在更多编曲、caption、歌词和歌曲结构中。 + +此功能仅面向音频数据集。 + +!!! warning "授权和权利" + 只应在你有权使用的声音和录音上使用此流程。声音身份是敏感的生物特征和创作数据。该变换可能生成听起来像真实人物的衍生音频，因此权限、授权和披露都很重要。 + +## ELI5 + +假设你有六首同一个歌手的录音，但每首都来自同一个乐队和同一种风格。只用这些歌训练时，模型可能会把歌手、吉他音色、鼓组感觉、速度范围和歌曲结构都当成一件事。 + +语音克隆数据变换试图拆开这些概念： + +1. 从歌手示例中学习一个小型语音转换模型。 +2. 读取更广泛的一组歌曲或人声 stem。 +3. 把源人声的音色替换成目标歌手音色。 +4. 让新的 captions 和歌词继续对齐生成后的音频。 +5. 把生成音频作为另一个普通训练 split 加入。 + +这样主模型就能在更多上下文中看到目标声音，而不是只记住原始窄数据集。 + +## 适用场景 + +适合使用： + +- 你有目标歌手的授权录音 +- 目标身份和单一流派、乐队、制作风格或歌曲结构纠缠太强 +- trigger 词只在原风格里有效，换流派就失效 +- 一个数据集里有多个歌手，模型学成了平均声音 +- 你想为不同人声身份训练不同 LoRA +- 你希望 SimpleTuner 在同一次训练配置里准备扩展 split + +不适合使用： + +- 你已经有大量、多样、干净的同一声音数据 +- 扩展源音频质量差，或与 caption 对不齐 +- 你需要公开发布且没有明确权利 +- 基础生成模型即使用干净直接样本也学不会目标身份 + +## 它如何进入训练 + +语音克隆是数据准备变换，不是 conditioning 数据集。 + +`conditioning_data` 用于训练时一直和主样本绑定的辅助输入，例如参考图像或生成的条件图。 + +语音克隆应该放在数据集级别的 `data_transforms` 列表中。它会生成新的音频文件、caption 和可选歌词，然后把结果注册为另一个主 `audio` 数据集。之后普通 dataloader 会像读取其他训练 split 一样读取它。 + +伪配置形状： + +```text +audio dataset: + id: target-voice + dataset_type: audio + data_transforms: + - task: identity_transfer + source: expansion-audio-backend + target: generated-audio-backend + method: rvc + audio_mode: separate_convert_remix +``` + +启动时的伪流程： + +```text +for each audio dataset: + for each data transform: + if task is identity_transfer: + prepare or reuse the target voice-conversion model + prepare or reuse generated audio + append generated audio as a normal train split + +continue with normal metadata discovery, bucketing, caching, and training +``` + +## RVC 风格的身份迁移 + +第一个计划支持的实现是 RVC 风格的语音转换。 + +这里的“RVC 模型”是声音专用的。它从目标身份数据集中训练得到。检索索引也是声音专用的，并由同一目标声音的特征构建。内容特征、音高提取、分离模型等预训练组件是可复用基础设施；转换模型和索引才是歌手或说话人专用的工件。 + +SimpleTuner 应该能够： + +1. 复用用户提供的语音转换模型和索引。 +2. 如果没有提供模型，则训练语音转换模型。 +3. 从目标声音数据构建检索索引。 +4. 在训练输出目录下缓存模型、索引和生成音频。 +5. 当源数据和变换设置未变化时，在启动时复用缓存工件。 +6. 可选地通过 Hub 模型仓库复用或发布语音转换模型。 + +## 默认行为 + +计划默认值比较保守： + +| 设置 | 默认值 | 原因 | +| --- | --- | --- | +| `task` | `identity_transfer` | 明确标识该变换。 | +| `method` | `rvc` | 首个支持的声音迁移后端。 | +| `train_if_missing` | `true` | SimpleTuner 应能从目标数据集启动训练声音模型。 | +| `force_retrain` | `false` | 尽量复用有效缓存模型。 | +| `build_index` | `true` | 检索通常能提高身份稳定性并减少泄漏。 | +| `hub_model_id` | 未设置 | 用户未显式启用时，不使用远程声音模型缓存。 | +| `reuse_from_hub` | 设置 `hub_model_id` 时为 `true` | 在花时间按需训练前先检查 Hub。 | +| `push_to_hub` | `false` | 声音模型代表一个声音身份，上传必须显式开启。 | +| `audio_mode` | 完整歌曲默认 `separate_convert_remix`，人声 stem 默认 `vocal_only` | 完整混音需要分离；stem 不需要。 | +| `separation_method` | 需要分离时使用 `demucs` | Demucs 是预期默认 stem 分离器。 | +| 生成 split 类型 | 主 `audio` 数据集 | 生成数据像普通音频一样训练，不作为 conditioning。 | +| 缓存位置 | `output_dir` 内 | 让生成工件绑定训练运行，并能重启复用。 | +| captions | 默认复制源 captions，除非另有配置 | 新 split 应保留歌词和编曲上下文。 | + +如果提供了已有语音转换模型，SimpleTuner 应使用它；只有显式要求或必要工件缺失时才训练新模型。 + +## Hub 缓存 + +语音转换模型可能训练成本较高，重复按需训练会变成明显的资源浪费。因此该 transform 应支持可选的 Hub 后端缓存，用于保存声音模型和检索索引。 + +安全查找顺序： + +```text +if local voice-conversion cache matches: + reuse local model and index +else if hub_model_id is configured and reuse_from_hub is enabled: + check the Hub repository + download only if it has a SimpleTuner voice-transform manifest + reuse only if the manifest matches this transform +else if train_if_missing is enabled: + train the voice-conversion model + build the retrieval index + cache locally + push to hub only when push_to_hub is true +else: + stop and ask for a model path or a reusable cache +``` + +Hub 仓库应使用 SimpleTuner 专用布局，而不是松散文件集合： + +```text +voice_transform/ + manifest.json + model.pth + index.index + README.md +``` + +Manifest 是契约。它应记录目标身份数据集指纹、RVC 训练设置、索引设置、预期采样率、工具版本和 SimpleTuner voice-transform 格式版本。缺少 manifest 或 manifest 与当前 transform 不匹配时，SimpleTuner 不应复用该 Hub 工件。这样可以避免把错误声音模型静默应用到新数据集。 + +发布应为 opt-in。合理伪配置： + +```text +identity_transfer: + method: rvc + model: + train_if_missing: true + hub_model_id: org/target-voice-rvc + reuse_from_hub: true + push_to_hub: true + private: true +``` + +对于私有声音身份，除非有明确授权，否则 Hub 仓库应保持私有。生成音频和模型工件可能有不同共享权利，因此应分别处理上传设置。 + +## WebUI 配置 + +RVC 模型训练应该可以通过 WebUI 配置，而不只依赖原始 dataloader JSON。 + +预期 WebUI 形状是在音频数据集里提供 dataset transform 编辑器： + +```text +Audio dataset + Data transforms + Add transform: Identity transfer + Method: RVC + Audio mode: vocal_only / separate_convert_remix / full_mix_convert + Train RVC model if missing: on + Force retrain: off + Build retrieval index: on + Hub model id: optional + Reuse from Hub: on when Hub model id is set + Push RVC model to Hub: off by default + Hub repo privacy: private by default + Caption rules: copy, append, remove +``` + +WebUI 应让两种常见设置一眼可见： + +- **已经有人声 stem：** 选择 `vocal_only`，保持 Demucs 关闭，并写出生成的人声 stem。 +- **有完整歌曲：** 选择 `separate_convert_remix`，使用 Demucs 分离，只转换人声 stem，再和原始伴奏 stem 混回。 + +界面应明确显示生成音频会成为另一个主音频训练 split。不要把身份迁移展示成 `conditioning_data`，否则用户会以为它是训练期间的配对 conditioning 行为。 + +## 分布式启动行为 + +当 SimpleTuner 使用多个 data-parallel rank 启动时，语音克隆启动阶段应该利用可用 GPU，而不是让 rank 0 完成所有工作。 + +这里有两个不同的分布式阶段： + +1. **RVC 模型训练：** 如果 `train_if_missing=true`，没有匹配的本地缓存，也没有匹配的 Hub 工件，则当 `world_size > 1` 时，RVC 训练循环应使用 DDP。每个 rank 应通过普通 distributed sampler 模式接收不同的目标声音 batch。 +2. **生成音频准备：** 扩展源输入应按 rank 切分，类似 TextEmbedCache 和 VAECache。每个 rank 只分离、转换并写出自己负责的 shard，然后所有 rank 同步，再继续 metadata discovery。 + +伪行为： + +```text +if world_size > 1: + if RVC model must be trained: + train RVC with DDP across all ranks + save final model and index once + + split expansion inputs by global rank + each rank generates its own audio shard + barrier + rank 0 writes or verifies the combined manifest + barrier +else: + train and generate serially +``` + +最终声音模型只应由一个进程发布到 Hub。最终 manifest 更新也一样。每个 rank 可以独立写出自己的生成结果，只要文件名是确定的且互不重叠。 + +这样可以避免在多 GPU 系统上浪费 GPU 时间，并让启动行为与 SimpleTuner 现有缓存准备模型保持一致。 + +## RVC 训练日志 + +启动阶段的 RVC 训练暂时不应创建 TensorBoard 或 WandB run。这些 logger 是给主 SimpleTuner 训练任务配置的，把它们复用于嵌套语音转换任务会需要额外 run 名称、路径、resume 规则和 artifact 策略。 + +RVC 阶段仍然可以通过 SimpleTuner 原生训练 logger 报告有用统计： + +```text +output_dir/ + logs/ + rvc/ + training_stats.jsonl + summary.json +``` + +有用的本地统计包括 RVC 训练 loss、启用时的 pitch loss、适用时的 reconstruction 或 discriminator loss、已处理样本数、耗时、DDP world size、缓存命中或未命中原因，以及最终模型来自本地缓存、Hub 缓存还是按需训练。 + +除非未来实现明确为 RVC transforms 添加外部 logger 集成，否则这些统计仅保存在本地。 + +## 选择 `audio_mode` + +### `vocal_only` + +当扩展数据集已经预处理成干净人声 stem 时使用。 + +```text +source vocal stem -> RVC conversion -> generated vocal stem +``` + +注意事项： + +- 不要对干净 stem 再跑 Demucs，除非你确实有理由。 +- caption 应描述人声和歌词；除非之后会混回伴奏，否则不要描述完整乐队编曲。 +- 如果主训练模型期望完整歌曲，纯人声生成数据可能会形成不同的数据分布。 + +### `separate_convert_remix` + +当扩展数据集是完整混音歌曲时使用。 + +```text +source full song + -> Demucs separates vocals and instrumental stems + -> RVC converts the vocal stem + -> converted vocal is remixed with the original instrumental stems + -> generated full song is added to training +``` + +这是完整歌曲扩展的首选模式，因为它避免把鼓、贝斯、吉他、房间声和母带痕迹当成声音的一部分去转换。 + +注意事项： + +- Stem 分离可能留下串音、伪影或相位问题。 +- 人声 stem 太弱、混响太多或被埋住时，转换后声音可能不稳定。 +- Remix 的响度很重要。生成 split 如果一直更响或更小，会影响训练偏向。 +- caption 应描述最终混音结果，而不只是源歌曲。 + +### `full_mix_convert` + +仅建议快速测试时使用。 + +```text +source full song -> RVC conversion over the whole mix -> generated full song +``` + +它更快，但质量通常更低，可能把乐器也拖进语音转换器，并把不需要的伪影教给最终 LoRA。 + +## Caption 和歌词策略 + +生成 split 的 caption 应匹配生成后的音频。 + +合理默认值： + +```text +copy source caption +remove source-vocal identity words when configured +append target-vocal identity or style words when configured +copy lyrics sidecar when lyrics still match +``` + +当源人声和转换后人声唱的是同一段词时，复制歌词通常是正确的。如果变换改了歌曲、剪掉段落、删除人声或使用非歌词源，则不应复制。 + +Caption 不能总是盲目复制。如果源 caption 写着“female pop vocal”，但生成结果是男性摇滚音色，就应该调整 caption。变换应支持简单的追加和删除规则，更高级的 caption 重写可以之后叠加。 + +## 缓存和复用 + +变换应写入两类缓存： + +```text +voice-conversion cache: + model checkpoint + retrieval index + manifest + +generated audio cache: + generated audio files + captions + lyrics, when available + manifest +``` + +Manifest 应记录身份数据集指纹、变换设置、扩展源数据指纹和工具版本。如果这些值匹配，启动时可以复用现有工件；如果变化，则只重新生成受影响阶段。 + +## 实用数据集建议 + +- 身份控制重要时，每个 LoRA 保持一个目标歌手。 +- 用干净、较干的人声样本训练语音转换模型。 +- 除非目标就是学习合唱混合，否则避免二重唱。 +- 扩展歌曲应覆盖不同速度、调性、流派、动态和歌词 phrasing。 +- Caption 要足够多样，避免身份 token 和一种编曲永久绑定。 +- 长训练前先抽查生成音频。 + +## 常见失败模式 + +| 现象 | 可能原因 | +| --- | --- | +| LoRA 只在一种流派里有效 | 声音身份仍与编曲 caption 或源数据纠缠。 | +| 生成 split 听起来空、相位怪 | 完整歌曲处理中的分离或 remix 伪影。 | +| 乐器像被变声了一样 | 需要分离时使用了 `full_mix_convert`。 | +| 人声身份很弱 | 目标数据需要更干净、更多样，或检索索引更强。 | +| Caption 控制不了声音 | Caption 仍含源声音身份，或没有目标身份。 | +| 主模型学到伪影 | 生成音频质量太低或在训练 mix 中占比太高。 | + +## 与正则化数据的关系 + +生成的身份迁移数据默认不是正则化数据。 + +正则化数据通常用于让 LoRA 保留基础模型行为。身份迁移数据用于让 LoRA 在更多上下文中学习目标声音。太多正则化配太少直接身份数据，会削弱身份 token；太多生成数据则可能教会转换伪影。 + +把它们看成三个独立旋钮： + +- 直接目标数据集：最强身份信号 +- 生成身份迁移数据集：更广的上下文和风格覆盖 +- 正则化数据集：保留基础模型行为 + +## 状态 + +本文描述计划中的实验性 `data_transforms` 工作流的用户体验。核心设计约束是：身份迁移应成为 SimpleTuner 的一等音频训练功能，能够训练或复用语音转换模型、构建或复用检索索引、生成扩展 split、缓存结果，然后直接进入正常训练，而不要求用户手动执行第二个预处理阶段。 diff --git a/documentation/index.es.md b/documentation/index.es.md index 6791eb179..26ffd8399 100644 --- a/documentation/index.es.md +++ b/documentation/index.es.md @@ -80,9 +80,9 @@ --- - Funciones de investigación como AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, cuantización SDNQ Hadamard estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention y Video CREPA + Funciones de investigación como AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, cuantización SDNQ Hadamard estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention, Video CREPA y clonacion de voz - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.es.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.es.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.es.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.es.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.es.md) · [:octicons-arrow-right-24: Clonacion de voz](experimental/VOICE_CLONING.es.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.es.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.hi.md b/documentation/index.hi.md index 8a3cbbcf6..4f8786960 100644 --- a/documentation/index.hi.md +++ b/documentation/index.hi.md @@ -80,9 +80,9 @@ --- - AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention और Video CREPA जैसी research features + AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention, Video CREPA और Voice Cloning जैसी research features - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.hi.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.hi.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.hi.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.hi.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.hi.md) · [:octicons-arrow-right-24: Voice Cloning](experimental/VOICE_CLONING.hi.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.hi.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.ja.md b/documentation/index.ja.md index 3f4ad06cf..0545f5645 100644 --- a/documentation/index.ja.md +++ b/documentation/index.ja.md @@ -80,9 +80,9 @@ --- - AnyFlow、MixFlow、Explorative Modeling、NextLat、DiffusionBlocks、ConvRot 形式の SDNQ Hadamard 量子化、segmented checkpointing、Unsloth 形式 checkpointing、Prompt2Effect、Self-Flow、Self-Transcendence、Flow-DPO、iREPA、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA などの研究向け機能 + AnyFlow、MixFlow、Explorative Modeling、NextLat、DiffusionBlocks、ConvRot 形式の SDNQ Hadamard 量子化、segmented checkpointing、Unsloth 形式 checkpointing、Prompt2Effect、Self-Flow、Self-Transcendence、Flow-DPO、iREPA、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA、音声クローニングなどの研究向け機能 - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.ja.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.ja.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.ja.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.ja.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.ja.md) · [:octicons-arrow-right-24: 音声クローニング](experimental/VOICE_CLONING.ja.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.ja.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.md b/documentation/index.md index c678a1f9e..ecce83137 100644 --- a/documentation/index.md +++ b/documentation/index.md @@ -80,9 +80,9 @@ --- - Research features such as AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention, and Video CREPA + Research features such as AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, ConvRot-style SDNQ Hadamard quantization, segmented checkpointing, Unsloth-style checkpointing, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention, Video CREPA, and Voice Cloning - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.md) · [:octicons-arrow-right-24: Voice Cloning](experimental/VOICE_CLONING.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.pt-BR.md b/documentation/index.pt-BR.md index d5392c873..f9fda24b2 100644 --- a/documentation/index.pt-BR.md +++ b/documentation/index.pt-BR.md @@ -80,9 +80,9 @@ --- - Recursos de pesquisa como AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, quantização SDNQ Hadamard no estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention e Video CREPA + Recursos de pesquisa como AnyFlow, MixFlow, Explorative Modeling, NextLat, DiffusionBlocks, quantização SDNQ Hadamard no estilo ConvRot, checkpointing segmentado, checkpointing estilo Unsloth, Prompt2Effect, Self-Flow, Self-Transcendence, Flow-DPO, Internal Guidance, iREPA, LayerSync, Diff2Flow, Metal Flash Attention, Video CREPA e clonagem de voz - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.pt-BR.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.pt-BR.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.pt-BR.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.pt-BR.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.pt-BR.md) · [:octicons-arrow-right-24: Clonagem de voz](experimental/VOICE_CLONING.pt-BR.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.pt-BR.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/documentation/index.zh.md b/documentation/index.zh.md index b433416f1..5a365403f 100644 --- a/documentation/index.zh.md +++ b/documentation/index.zh.md @@ -80,9 +80,9 @@ --- - AnyFlow、MixFlow、Explorative Modeling、NextLat、DiffusionBlocks、ConvRot 风格的 SDNQ Hadamard 量化、segmented checkpointing、Unsloth 风格 checkpointing、Prompt2Effect、Self-Flow、Self-Transcendence、Flow-DPO、iREPA、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA 等研究功能 + AnyFlow、MixFlow、Explorative Modeling、NextLat、DiffusionBlocks、ConvRot 风格的 SDNQ Hadamard 量化、segmented checkpointing、Unsloth 风格 checkpointing、Prompt2Effect、Self-Flow、Self-Transcendence、Flow-DPO、iREPA、LayerSync、Diff2Flow、Metal Flash Attention、Video CREPA、语音克隆等研究功能 - [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.zh.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.zh.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.zh.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) + [:octicons-arrow-right-24: AnyFlow](experimental/ANYFLOW.md) · [:octicons-arrow-right-24: MixFlow](experimental/MIXFLOW.md) · [:octicons-arrow-right-24: XM](experimental/EXPLORATION_MODELING.zh.md) · [:octicons-arrow-right-24: NextLat](experimental/NEXTLAT.zh.md) · [:octicons-arrow-right-24: 语音克隆](experimental/VOICE_CLONING.zh.md) · [:octicons-arrow-right-24: DiffusionBlocks](experimental/DIFFUSION_BLOCKS.md) · [:octicons-arrow-right-24: Self-Transcendence](experimental/SELF_TRANSCENDENCE.md) · [:octicons-arrow-right-24: iREPA](experimental/IREPA.zh.md) · [:octicons-arrow-right-24: ConvRot / Hadamard SDNQ](experimental/CONVROT.md) · [:octicons-arrow-right-24: Segmented Checkpointing](experimental/SEGMENTED_CHECKPOINTING.md) · [:octicons-arrow-right-24: Unsloth Checkpointing](experimental/UNSLOTH_CHECKPOINTING.md) · [:octicons-arrow-right-24: Metal Flash Attention](experimental/METAL_FLASH_ATTENTION.md) diff --git a/mkdocs.yml b/mkdocs.yml index 260aaa46d..7e90e0d47 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -379,6 +379,7 @@ nav: - T-LoRA: experimental/T_LORA.md - Unsloth-Style Checkpointing: experimental/UNSLOTH_CHECKPOINTING.md - Video CREPA: experimental/VIDEO_CREPA.md + - Voice Cloning Data Transforms: experimental/VOICE_CLONING.md - Development: - Factory Architecture: development/FACTORY_ARCHITECTURE.md - Web UI Implementation: webui/IMPLEMENTATION.md diff --git a/simpletuner/helpers/data_backend/factory.py b/simpletuner/helpers/data_backend/factory.py index b3214194f..b0aeac2f2 100644 --- a/simpletuner/helpers/data_backend/factory.py +++ b/simpletuner/helpers/data_backend/factory.py @@ -118,6 +118,7 @@ def _coerce_bucket_keys(indices: Dict[Any, Iterable]) -> Dict[Any, list]: normalize_start_step, ) from simpletuner.helpers.data_backend.webshart import WebshartDataBackend +from simpletuner.helpers.data_transforms import process_data_transforms from simpletuner.helpers.distillation.common import DistillationBase from simpletuner.helpers.distillation.composition import resolve_configured_distiller_requirement_profile from simpletuner.helpers.distillation.requirements import ( @@ -3198,8 +3199,10 @@ def _handle_bucket_operations( Args: skip_bucket_split: If True, skip split_buckets_between_processes (used for audio-only datasets). """ + metadata_clone_source_id = backend.get("metadata_clone_source_id") + has_metadata_clone_source = metadata_clone_source_id is not None if ( - not backend.get("auto_generated", False) # auto-generated datasets have duplicate metadata. + not has_metadata_clone_source and "aspect" not in self.args.skip_file_discovery and "aspect" not in backend.get("skip_file_discovery", "") and conditioning_type @@ -3238,7 +3241,7 @@ def _handle_bucket_operations( # serialized cache rather than mixing rank 0's in-memory order with reloaded order. if ( self._is_multi_process() - and not backend.get("auto_generated", False) + and not has_metadata_clone_source and "aspect" not in self.args.skip_file_discovery and "aspect" not in backend.get("skip_file_discovery", "") and conditioning_type @@ -3253,7 +3256,7 @@ def _handle_bucket_operations( init_backend["metadata_backend"].reload_cache() if ( - not backend.get("auto_generated", False) + not has_metadata_clone_source and backend.get("conditioning_type", None) is not None and backend.get("conditioning_type") not in [ @@ -3278,13 +3281,11 @@ def _handle_bucket_operations( apply_padding = not self.args.max_train_steps or self.args.allow_dataset_oversubscription - if backend.get("auto_generated", False): + if has_metadata_clone_source: # when we're duplicating a metadata set, it's already split between processes. - info_log( - f"Duplicating metadata for auto-generated dataset from {backend.get('source_dataset_id', 'unknown_source_dataset_id')}" - ) + info_log(f"Duplicating metadata for generated dataset from {metadata_clone_source_id}") DatasetDuplicator.copy_metadata( - source_backend=StateTracker.get_data_backend(backend.get("source_dataset_id", "unknown_source_dataset_id")), + source_backend=StateTracker.get_data_backend(metadata_clone_source_id), target_backend=init_backend, ) elif backend.get("conditioning_type", None) in ["reference_strict", "mask"]: @@ -3373,6 +3374,7 @@ def _handle_config_versioning(self, backend: Dict[str, Any], init_backend: Dict[ "video", "conditioning_data", "conditioning", + "data_transforms", "hash_filenames", # always enabled, not user-configurable "_s2v_audio_autoinjected", # runtime flag, not user-configurable *runtime_mutable_keys, @@ -3421,6 +3423,7 @@ def _handle_config_versioning(self, backend: Dict[str, Any], init_backend: Dict[ runtime_linkage_keys = ( "conditioning_data", "conditioning", + "data_transforms", "video", "s2v_datasets", "_s2v_audio_autoinjected", @@ -4710,6 +4713,11 @@ def configure(self, data_backend_config: Optional[List[Dict[str, Any]]] = None) data_backend_config = self._inject_grounding_configs(data_backend_config) data_backend_config = self._inject_s2v_audio_configs(data_backend_config) data_backend_config = self.process_conditioning_datasets(data_backend_config) + data_backend_config = process_data_transforms( + global_config=self.args, + data_backend_config=data_backend_config, + accelerator=self.accelerator, + ) self.configure_text_embed_backends(data_backend_config) self.configure_image_embed_backends(data_backend_config) diff --git a/simpletuner/helpers/data_transforms/__init__.py b/simpletuner/helpers/data_transforms/__init__.py new file mode 100644 index 000000000..a6524030b --- /dev/null +++ b/simpletuner/helpers/data_transforms/__init__.py @@ -0,0 +1,10 @@ +"""Dataset transform registry.""" + +from simpletuner.helpers.data_transforms.base import DataTransformTask, process_data_transforms +from simpletuner.helpers.data_transforms.identity_transfer import IdentityTransferTransform + +__all__ = [ + "DataTransformTask", + "IdentityTransferTransform", + "process_data_transforms", +] diff --git a/simpletuner/helpers/data_transforms/base.py b/simpletuner/helpers/data_transforms/base.py new file mode 100644 index 000000000..8c05c7b9f --- /dev/null +++ b/simpletuner/helpers/data_transforms/base.py @@ -0,0 +1,111 @@ +"""Runtime expansion for generated dataset transforms.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Any, Dict, List, Type + + +class DataTransformTask(ABC): + """Base class for data transforms that expand one dataset into another.""" + + TASK: str = "" + SUPPORTED_SOURCE_DATASET_TYPES: tuple[str, ...] = () + REQUIRES_METADATA_CLONE = False + + def __init__( + self, + global_config: Any, + source_backend_config: Dict[str, Any], + transform_config: Dict[str, Any], + accelerator: Any = None, + ) -> None: + self.global_config = global_config + self.source_backend_config = source_backend_config + self.transform_config = deepcopy(transform_config) + self.accelerator = accelerator + + @abstractmethod + def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: + """Materialize any required artifacts and return generated backend configs.""" + + def validate_source_dataset_type(self) -> None: + dataset_type = self.source_backend_config.get("dataset_type", "image") + if self.SUPPORTED_SOURCE_DATASET_TYPES and dataset_type not in self.SUPPORTED_SOURCE_DATASET_TYPES: + supported = ", ".join(self.SUPPORTED_SOURCE_DATASET_TYPES) + raise ValueError( + f"Data transform '{self.TASK}' only supports source dataset_type values: {supported}. " + f"Received dataset_type={dataset_type!r} for backend {self.source_backend_config.get('id')!r}." + ) + + +_TASK_REGISTRY: Dict[str, Type[DataTransformTask]] = {} + + +def register_data_transform(task_cls: Type[DataTransformTask]) -> Type[DataTransformTask]: + if not task_cls.TASK: + raise ValueError("Data transform task classes must define TASK.") + _TASK_REGISTRY[task_cls.TASK] = task_cls + return task_cls + + +def get_data_transform_task(task: str) -> Type[DataTransformTask]: + try: + return _TASK_REGISTRY[task] + except KeyError as exc: + known = ", ".join(sorted(_TASK_REGISTRY)) or "" + raise ValueError(f"Unknown data transform task {task!r}. Known tasks: {known}.") from exc + + +def _normalise_transform_list(source_backend_id: str, transform_block: Any) -> List[Dict[str, Any]]: + if transform_block in (None, [], {}): + return [] + if isinstance(transform_block, dict): + return [transform_block] + if not isinstance(transform_block, list): + raise ValueError(f"data_transforms for backend {source_backend_id!r} must be a dict or list of dicts.") + for transform in transform_block: + if not isinstance(transform, dict): + raise ValueError(f"Every data_transforms entry for backend {source_backend_id!r} must be a dict.") + return transform_block + + +def process_data_transforms( + global_config: Any, + data_backend_config: List[Dict[str, Any]], + accelerator: Any = None, +) -> List[Dict[str, Any]]: + """Expand configured transforms into additional dataset backends.""" + generated_backends: List[Dict[str, Any]] = [] + existing_backend_ids = {backend.get("id") for backend in data_backend_config if backend.get("id")} + + for backend in data_backend_config: + if backend.get("disabled", False) or backend.get("disable", False): + continue + + for transform_config in _normalise_transform_list(backend.get("id", ""), backend.get("data_transforms")): + task_name = transform_config.get("task") + if not task_name: + raise ValueError(f"data_transforms entry for backend {backend.get('id')!r} requires a 'task' value.") + task_cls = get_data_transform_task(str(task_name)) + task = task_cls( + global_config=global_config, + source_backend_config=backend, + transform_config=transform_config, + accelerator=accelerator, + ) + task.validate_source_dataset_type() + new_backends = task.prepare(existing_backend_ids=existing_backend_ids) + for new_backend in new_backends: + backend_id = new_backend.get("id") + if not backend_id: + raise ValueError(f"Data transform {task_name!r} generated a backend without an id.") + if backend_id in existing_backend_ids: + raise ValueError(f"Data transform {task_name!r} generated duplicate backend id {backend_id!r}.") + existing_backend_ids.add(backend_id) + generated_backends.append(new_backend) + + if generated_backends: + data_backend_config.extend(generated_backends) + return data_backend_config diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py new file mode 100644 index 000000000..139eec593 --- /dev/null +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -0,0 +1,491 @@ +"""Audio identity transfer dataset transform plumbing.""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import shutil +from copy import deepcopy +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from simpletuner.helpers.data_transforms.base import DataTransformTask, register_data_transform +from simpletuner.helpers.training.multi_process import should_log + +logger = logging.getLogger(__name__) +if should_log(): + logger.setLevel(os.environ.get("SIMPLETUNER_LOG_LEVEL", "INFO")) +else: + logger.setLevel(logging.ERROR) + + +VOICE_TRANSFORM_FORMAT = "simpletuner-voice-transform" +VOICE_TRANSFORM_FORMAT_VERSION = 1 + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _json_default(value: Any) -> str: + return repr(value) + + +def _stable_json(data: Dict[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":"), default=_json_default) + + +def _sha256_json(data: Dict[str, Any]) -> str: + return hashlib.sha256(_stable_json(data).encode("utf-8")).hexdigest() + + +def _get_arg_value(args: Any, key: str, default: Any = None) -> Any: + if isinstance(args, dict): + return args.get(key, default) + return getattr(args, key, default) + + +class RVCTransformLogger: + """Small local JSON logger for startup voice-transform work.""" + + def __init__(self, output_dir: str, accelerator: Any = None) -> None: + self.accelerator = accelerator + self.enabled = accelerator is None or bool(getattr(accelerator, "is_main_process", True)) + self.log_dir = Path(output_dir) / "logs" / "rvc" + self.events_path = self.log_dir / "training_stats.jsonl" + self.summary_path = self.log_dir / "summary.json" + + def event(self, transform_id: str, event: str, **payload: Any) -> None: + if not self.enabled: + return + self.log_dir.mkdir(parents=True, exist_ok=True) + record = { + "timestamp": _utc_now(), + "transform_id": transform_id, + "event": event, + **payload, + } + with self.events_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True, default=_json_default) + "\n") + + def summary(self, transform_id: str, **payload: Any) -> None: + if not self.enabled: + return + self.log_dir.mkdir(parents=True, exist_ok=True) + record = { + "timestamp": _utc_now(), + "transform_id": transform_id, + **payload, + } + tmp_path = self.summary_path.with_suffix(".json.tmp") + tmp_path.write_text(json.dumps(record, indent=2, sort_keys=True, default=_json_default), encoding="utf-8") + tmp_path.replace(self.summary_path) + + +@dataclass(frozen=True) +class VoiceModelArtifact: + cache_dir: Path + manifest_path: Path + model_path: Path + index_path: Optional[Path] + manifest: Dict[str, Any] + + +class RVCTrainer: + def train( + self, + source_backend_config: Dict[str, Any], + transform_config: Dict[str, Any], + cache_dir: Path, + accelerator: Any = None, + logger: Optional[RVCTransformLogger] = None, + ) -> VoiceModelArtifact: + world_size = int(getattr(accelerator, "num_processes", 1) or 1) + raise NotImplementedError( + "On-demand RVC training is configured, but the native RVC trainer is not implemented yet. " + f"The transform runner reached the trainer with DDP world_size={world_size}." + ) + + +class RVCConverter: + def convert( + self, + source_backend_config: Dict[str, Any], + target_backend_config: Dict[str, Any], + transform_config: Dict[str, Any], + artifact: VoiceModelArtifact, + input_paths: List[str], + accelerator: Any = None, + logger: Optional[RVCTransformLogger] = None, + ) -> None: + raise NotImplementedError( + "RVC identity transfer conversion is configured, but the native converter is not implemented yet." + ) + + +class HubVoiceModelCache: + def __init__(self, hub_model_id: str, token: Optional[str] = None) -> None: + self.hub_model_id = hub_model_id + self.token = token + + def download_if_compatible(self, cache_dir: Path, fingerprint: str) -> Optional[VoiceModelArtifact]: + try: + from huggingface_hub import hf_hub_download + from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError + except ImportError as exc: + raise ImportError("huggingface_hub is required when identity_transfer.reuse_from_hub is enabled.") from exc + + try: + manifest_file = hf_hub_download( + repo_id=self.hub_model_id, + filename="voice_transform/manifest.json", + token=self.token, + ) + except (EntryNotFoundError, RepositoryNotFoundError): + return None + + manifest = json.loads(Path(manifest_file).read_text(encoding="utf-8")) + if not _manifest_matches(manifest, fingerprint): + return None + + model_file = hf_hub_download( + repo_id=self.hub_model_id, + filename="voice_transform/model.pth", + token=self.token, + ) + index_file: Optional[str] + try: + index_file = hf_hub_download( + repo_id=self.hub_model_id, + filename="voice_transform/index.index", + token=self.token, + ) + except EntryNotFoundError: + index_file = None + + cache_dir.mkdir(parents=True, exist_ok=True) + local_manifest = cache_dir / "manifest.json" + local_model = cache_dir / "model.pth" + local_index = cache_dir / "index.index" + shutil.copy2(manifest_file, local_manifest) + shutil.copy2(model_file, local_model) + if index_file: + shutil.copy2(index_file, local_index) + return VoiceModelArtifact( + cache_dir=cache_dir, + manifest_path=local_manifest, + model_path=local_model, + index_path=local_index if index_file else None, + manifest=manifest, + ) + + def upload(self, artifact: VoiceModelArtifact) -> None: + try: + from huggingface_hub import HfApi + except ImportError as exc: + raise ImportError("huggingface_hub is required when identity_transfer.push_to_hub is enabled.") from exc + + api = HfApi(token=self.token) + api.create_repo(repo_id=self.hub_model_id, repo_type="model", exist_ok=True) + api.upload_folder( + repo_id=self.hub_model_id, + repo_type="model", + folder_path=str(artifact.cache_dir), + path_in_repo="voice_transform", + ) + + +def _manifest_matches(manifest: Dict[str, Any], fingerprint: str) -> bool: + return ( + manifest.get("format") == VOICE_TRANSFORM_FORMAT + and manifest.get("format_version") == VOICE_TRANSFORM_FORMAT_VERSION + and manifest.get("task") == "identity_transfer" + and manifest.get("method") == "rvc" + and manifest.get("fingerprint") == fingerprint + ) + + +@register_data_transform +class IdentityTransferTransform(DataTransformTask): + TASK = "identity_transfer" + SUPPORTED_SOURCE_DATASET_TYPES = ("audio",) + REQUIRES_METADATA_CLONE = False + + def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: + source_id = self.source_backend_config.get("id") + if not source_id: + raise ValueError("identity_transfer requires the source backend to have an id.") + + transform = self._normalise_transform_config(existing_backend_ids) + transform_id = transform["id"] + output_dir = self._output_dir() + transform_root = Path(output_dir) / "cache" / "data_transforms" / transform_id + model_cache_dir = Path(transform["model"]["cache_dir"]) + generated_dir = Path(transform["target"]["instance_data_dir"]) + fingerprint = self._fingerprint(transform) + run_logger = RVCTransformLogger(output_dir, accelerator=self.accelerator) + + target_backend_config = self._target_backend_config(transform, generated_dir) + if self._generated_cache_matches(generated_dir, fingerprint): + run_logger.event(transform_id, "generated_cache_reused", path=str(generated_dir)) + run_logger.summary( + transform_id, status="reused_generated_cache", generated_backend_id=target_backend_config["id"] + ) + return [target_backend_config] + + artifact = self._resolve_voice_model(transform, fingerprint, model_cache_dir, run_logger) + if not self._is_main_process(): + self._wait_for_everyone() + run_logger.summary( + transform_id, status="waiting_for_main_process", generated_backend_id=target_backend_config["id"] + ) + return [target_backend_config] + + input_paths = self._rank_shard(self._discover_source_audio_paths()) + run_logger.event( + transform_id, + "conversion_start", + source_count=len(input_paths), + generated_path=str(generated_dir), + world_size=self._world_size(), + ) + RVCConverter().convert( + source_backend_config=self.source_backend_config, + target_backend_config=target_backend_config, + transform_config=transform, + artifact=artifact, + input_paths=input_paths, + accelerator=self.accelerator, + logger=run_logger, + ) + self._write_generated_manifest(generated_dir, fingerprint, transform) + self._wait_for_everyone() + run_logger.summary(transform_id, status="generated", generated_backend_id=target_backend_config["id"]) + return [target_backend_config] + + def _normalise_transform_config(self, existing_backend_ids: set[str]) -> Dict[str, Any]: + source_id = self.source_backend_config["id"] + transform = deepcopy(self.transform_config) + transform.setdefault("task", self.TASK) + transform.setdefault("method", "rvc") + if transform["method"] != "rvc": + raise ValueError("identity_transfer currently supports method='rvc' only.") + + transform_id = transform.get("id") or f"{source_id}_identity_transfer" + if transform_id in existing_backend_ids: + raise ValueError(f"identity_transfer generated backend id {transform_id!r} already exists.") + transform["id"] = transform_id + + output_dir = self._output_dir() + transform_root = Path(output_dir) / "cache" / "data_transforms" / transform_id + model = deepcopy(transform.get("model") or {}) + model.setdefault("train_if_missing", True) + model.setdefault("force_retrain", False) + model.setdefault("build_index", True) + model.setdefault("reuse_from_hub", bool(model.get("hub_model_id"))) + model.setdefault("push_to_hub", False) + model.setdefault("cache_dir", str(transform_root / "rvc_model")) + transform["model"] = model + + conversion = deepcopy(transform.get("conversion") or {}) + conversion.setdefault("audio_mode", "vocal_only") + conversion.setdefault("separation_method", "demucs") + transform["conversion"] = conversion + + target = deepcopy(transform.get("target") or {}) + target.setdefault("id", transform_id) + target.setdefault("type", "local") + target.setdefault("dataset_type", "audio") + target.setdefault("metadata_backend", "discovery") + target.setdefault("caption_strategy", "textfile") + target.setdefault("instance_data_dir", str(transform_root / "generated_audio")) + transform["target"] = target + return transform + + def _target_backend_config(self, transform: Dict[str, Any], generated_dir: Path) -> Dict[str, Any]: + source_cfg = deepcopy(self.source_backend_config) + target = deepcopy(transform["target"]) + + target_cfg: Dict[str, Any] = { + "id": target["id"], + "type": target["type"], + "dataset_type": "audio", + "metadata_backend": target["metadata_backend"], + "caption_strategy": target["caption_strategy"], + "instance_data_dir": str(generated_dir), + "generated_by": "data_transforms", + "data_transform_task": self.TASK, + "source_dataset_id": self.source_backend_config["id"], + "data_transform_source_dataset_id": self.source_backend_config["id"], + "data_transform_config": transform, + } + for key in ( + "audio", + "audio_column", + "huggingface", + "parquet", + "csv", + "cache_dir_vae", + "minimum_image_size", + "repeats", + "train_batch_size", + ): + if key in source_cfg: + target_cfg[key] = deepcopy(source_cfg[key]) + target_cfg.update({key: value for key, value in target.items() if key not in {"id", "type", "dataset_type"}}) + target_cfg["dataset_type"] = "audio" + target_cfg["instance_data_dir"] = str(generated_dir) + target_audio = deepcopy(source_cfg.get("audio") or {}) + target_audio.update(deepcopy(target.get("audio") or {})) + if target_audio: + target_cfg["audio"] = target_audio + return target_cfg + + def _resolve_voice_model( + self, + transform: Dict[str, Any], + fingerprint: str, + model_cache_dir: Path, + run_logger: RVCTransformLogger, + ) -> VoiceModelArtifact: + transform_id = transform["id"] + local_artifact = self._local_artifact(model_cache_dir, fingerprint) + if local_artifact and not transform["model"].get("force_retrain", False): + run_logger.event(transform_id, "voice_model_reused", source="local", path=str(model_cache_dir)) + return local_artifact + + model_cfg = transform["model"] + hub_model_id = model_cfg.get("hub_model_id") + if hub_model_id and model_cfg.get("reuse_from_hub", True) and not model_cfg.get("force_retrain", False): + artifact = HubVoiceModelCache(hub_model_id, token=model_cfg.get("hub_token")).download_if_compatible( + model_cache_dir, + fingerprint, + ) + if artifact: + run_logger.event(transform_id, "voice_model_reused", source="hub", hub_model_id=hub_model_id) + return artifact + + if not model_cfg.get("train_if_missing", False): + raise ValueError( + "identity_transfer could not find a compatible local or Hub voice artifact, and " + "model.train_if_missing is false." + ) + + run_logger.event(transform_id, "voice_model_training_start", world_size=self._world_size()) + artifact = RVCTrainer().train( + source_backend_config=self.source_backend_config, + transform_config=transform, + cache_dir=model_cache_dir, + accelerator=self.accelerator, + logger=run_logger, + ) + if model_cfg.get("push_to_hub", False): + if not hub_model_id: + raise ValueError("identity_transfer.model.push_to_hub requires identity_transfer.model.hub_model_id.") + HubVoiceModelCache(hub_model_id, token=model_cfg.get("hub_token")).upload(artifact) + run_logger.event(transform_id, "voice_model_pushed", hub_model_id=hub_model_id) + return artifact + + def _local_artifact(self, cache_dir: Path, fingerprint: str) -> Optional[VoiceModelArtifact]: + manifest_path = cache_dir / "manifest.json" + model_path = cache_dir / "model.pth" + index_path = cache_dir / "index.index" + if not manifest_path.exists() or not model_path.exists(): + return None + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not _manifest_matches(manifest, fingerprint): + return None + return VoiceModelArtifact( + cache_dir=cache_dir, + manifest_path=manifest_path, + model_path=model_path, + index_path=index_path if index_path.exists() else None, + manifest=manifest, + ) + + def _generated_cache_matches(self, generated_dir: Path, fingerprint: str) -> bool: + manifest_path = generated_dir / ".simpletuner_identity_transfer.json" + if not manifest_path.exists() or not generated_dir.exists(): + return False + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + return _manifest_matches(manifest, fingerprint) + + def _write_generated_manifest(self, generated_dir: Path, fingerprint: str, transform: Dict[str, Any]) -> None: + generated_dir.mkdir(parents=True, exist_ok=True) + manifest = self._manifest(fingerprint, transform) + (generated_dir / ".simpletuner_identity_transfer.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True), + encoding="utf-8", + ) + + def _manifest(self, fingerprint: str, transform: Dict[str, Any]) -> Dict[str, Any]: + return { + "format": VOICE_TRANSFORM_FORMAT, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + "task": self.TASK, + "method": "rvc", + "fingerprint": fingerprint, + "source_dataset_id": self.source_backend_config["id"], + "transform_id": transform["id"], + "created_by": "simpletuner", + "created_at": _utc_now(), + } + + def _fingerprint(self, transform: Dict[str, Any]) -> str: + source_cfg = deepcopy(self.source_backend_config) + source_cfg.pop("data_transforms", None) + transform_for_hash = deepcopy(transform) + model_cfg = transform_for_hash.get("model") or {} + for transient_key in ("hub_token", "push_to_hub", "reuse_from_hub"): + model_cfg.pop(transient_key, None) + transform_for_hash["model"] = model_cfg + return _sha256_json( + { + "source_backend": source_cfg, + "transform": transform_for_hash, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + } + ) + + def _discover_source_audio_paths(self) -> List[str]: + source_type = self.source_backend_config.get("type") + if source_type != "local": + raise NotImplementedError( + "identity_transfer input discovery is currently implemented for local audio source backends only." + ) + instance_data_dir = self.source_backend_config.get("instance_data_dir") + if not instance_data_dir: + raise ValueError("identity_transfer local source backend requires instance_data_dir.") + audio_exts = {".flac", ".wav", ".mp3", ".ogg", ".m4a", ".aac", ".opus"} + root = Path(instance_data_dir) + if not root.exists(): + raise FileNotFoundError(f"identity_transfer source instance_data_dir does not exist: {instance_data_dir}") + paths = [str(path) for path in sorted(root.rglob("*")) if path.suffix.lower() in audio_exts] + if not paths: + raise ValueError(f"identity_transfer found no audio files under {instance_data_dir}.") + return paths + + def _rank_shard(self, items: Iterable[str]) -> List[str]: + rank = int(getattr(self.accelerator, "process_index", 0) or 0) + world_size = self._world_size() + return [item for idx, item in enumerate(items) if idx % world_size == rank] + + def _world_size(self) -> int: + return int(getattr(self.accelerator, "num_processes", 1) or 1) + + def _is_main_process(self) -> bool: + return self.accelerator is None or bool(getattr(self.accelerator, "is_main_process", True)) + + def _wait_for_everyone(self) -> None: + if self.accelerator is not None and hasattr(self.accelerator, "wait_for_everyone"): + self.accelerator.wait_for_everyone() + + def _output_dir(self) -> str: + output_dir = _get_arg_value(self.global_config, "output_dir", None) + if output_dir: + return str(output_dir) + return os.path.join(os.getcwd(), ".simpletuner_output") diff --git a/simpletuner/helpers/metadata/utils/duplicator.py b/simpletuner/helpers/metadata/utils/duplicator.py index 839b13152..dce8e52ae 100644 --- a/simpletuner/helpers/metadata/utils/duplicator.py +++ b/simpletuner/helpers/metadata/utils/duplicator.py @@ -240,6 +240,7 @@ def _create_single_conditioning_config(source_cfg, cond_cfg, source_id, global_c # Set core fields target_cfg["auto_generated"] = True target_cfg["source_dataset_id"] = source_id + target_cfg["metadata_clone_source_id"] = source_id target_cfg["dataset_type"] = "conditioning" target_cfg["conditioning_config"] = cond_cfg target_cfg["conditioning_type"] = cond_cfg.get("conditioning_type", "reference_strict") diff --git a/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py b/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py index fc59cb73c..9b7656f0f 100644 --- a/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py +++ b/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py @@ -592,6 +592,14 @@ class BackendBlueprint(BaseModel): "type": "toggle", "defaultValue": False, }, + { + "id": "data_transforms", + "label": "data transforms", + "description": "optional generated audio split configuration, such as identity_transfer", + "type": "textarea", + "placeholder": '[{"task":"identity_transfer","id":"voice-transfer","model":{"train_if_missing":true}}]', + "advanced": True, + }, { "id": "cache_dir_vae", "label": "vae cache directory", diff --git a/simpletuner/static/js/dataloader-section-component.js b/simpletuner/static/js/dataloader-section-component.js index 47d98effc..53c2039ee 100644 --- a/simpletuner/static/js/dataloader-section-component.js +++ b/simpletuner/static/js/dataloader-section-component.js @@ -381,6 +381,9 @@ function dataloaderSectionComponent() { if (dataset && !this.supportsTrainBatchSize(dataset)) { delete dataset.train_batch_size; } + if (dataset && dataset.dataset_type !== 'audio') { + delete dataset.data_transforms; + } this.markAsUnsaved(); }, @@ -462,6 +465,13 @@ function dataloaderSectionComponent() { dataset.conditioning = []; } } + if (dataset.dataset_type === 'audio' && !Array.isArray(dataset.data_transforms)) { + if (dataset.data_transforms && typeof dataset.data_transforms === 'object') { + dataset.data_transforms = [dataset.data_transforms]; + } else { + dataset.data_transforms = []; + } + } if (!dataset.grounding || typeof dataset.grounding !== 'object') { dataset.grounding = { enabled: false }; } @@ -601,6 +611,65 @@ function dataloaderSectionComponent() { dataset._connectionDetails = null; this.markAsUnsaved(); }, + identityTransferTransform(dataset) { + if (!dataset || dataset.dataset_type !== 'audio') { + return null; + } + if (!Array.isArray(dataset.data_transforms)) { + dataset.data_transforms = []; + } + let transform = dataset.data_transforms.find((entry) => entry && entry.task === 'identity_transfer'); + if (!transform) { + transform = { + task: 'identity_transfer', + method: 'rvc', + id: `${dataset.id || 'audio'}_identity_transfer`, + model: { + train_if_missing: true, + build_index: true, + reuse_from_hub: true, + push_to_hub: false, + force_retrain: false + }, + conversion: { + audio_mode: 'vocal_only', + separation_method: 'demucs' + }, + target: {} + }; + dataset.data_transforms.push(transform); + } + transform.method = transform.method || 'rvc'; + transform.model = transform.model && typeof transform.model === 'object' ? transform.model : {}; + transform.conversion = transform.conversion && typeof transform.conversion === 'object' ? transform.conversion : {}; + transform.target = transform.target && typeof transform.target === 'object' ? transform.target : {}; + if (transform.model.train_if_missing === undefined) transform.model.train_if_missing = true; + if (transform.model.build_index === undefined) transform.model.build_index = true; + if (transform.model.reuse_from_hub === undefined) transform.model.reuse_from_hub = true; + if (transform.model.push_to_hub === undefined) transform.model.push_to_hub = false; + if (transform.model.force_retrain === undefined) transform.model.force_retrain = false; + transform.conversion.audio_mode = transform.conversion.audio_mode || 'vocal_only'; + transform.conversion.separation_method = transform.conversion.separation_method || 'demucs'; + return transform; + }, + identityTransferEnabled(dataset) { + return !!( + dataset + && Array.isArray(dataset.data_transforms) + && dataset.data_transforms.some((entry) => entry && entry.task === 'identity_transfer') + ); + }, + setIdentityTransferEnabled(dataset, enabled) { + if (!dataset) { + return; + } + if (enabled) { + this.identityTransferTransform(dataset); + } else if (Array.isArray(dataset.data_transforms)) { + dataset.data_transforms = dataset.data_transforms.filter((entry) => !(entry && entry.task === 'identity_transfer')); + } + this.markAsUnsaved(); + }, async testDatasetConnection(dataset) { const trainer = Alpine.store('trainer'); if (!dataset || !trainer || typeof trainer.prepareDatasetsForSave !== 'function') { diff --git a/simpletuner/templates/components/dataloader/sections/audio_body.html b/simpletuner/templates/components/dataloader/sections/audio_body.html index ae6fd8cfc..c55935e94 100644 --- a/simpletuner/templates/components/dataloader/sections/audio_body.html +++ b/simpletuner/templates/components/dataloader/sections/audio_body.html @@ -182,6 +182,133 @@
Format
+ +
+
Voice Identity Transfer
+
+
+
+ + +
+
+
+ + +
diff --git a/simpletuner/templates/trainer_htmx.html b/simpletuner/templates/trainer_htmx.html index e578f09e6..db17c36c1 100644 --- a/simpletuner/templates/trainer_htmx.html +++ b/simpletuner/templates/trainer_htmx.html @@ -1743,9 +1743,13 @@ if (dataset.dataset_type === 'audio') { dataset.audio = this.normalizeAudioOptions(dataset.audio); + dataset.data_transforms = this.normalizeDataTransforms(dataset.data_transforms, dataset.id, dataset.dataset_type); } else if (dataset.audio && typeof dataset.audio === 'object') { dataset.audio = this.normalizeAudioOptions(dataset.audio); } + if (dataset.dataset_type !== 'audio') { + delete dataset.data_transforms; + } if (dataset.dataset_type === 'conditioning') { // Default to 'controlnet' if ControlNet training is enabled, otherwise 'reference_strict' @@ -2245,6 +2249,72 @@ return normalized; }, + normalizeDataTransforms(value, datasetId, datasetType) { + if (datasetType !== 'audio') { + return []; + } + let transforms = []; + if (Array.isArray(value)) { + transforms = value; + } else if (value && typeof value === 'object') { + transforms = [value]; + } else if (typeof value === 'string' && value.trim()) { + try { + const parsed = JSON.parse(value); + transforms = Array.isArray(parsed) ? parsed : [parsed]; + } catch (error) { + console.warn('Failed to parse data_transforms JSON string:', error); + transforms = []; + } + } + + return transforms + .filter((entry) => entry && typeof entry === 'object') + .map((entry) => { + if (entry.task !== 'identity_transfer') { + return this.sanitizeValueForSave(entry); + } + const normalized = { + task: 'identity_transfer', + method: 'rvc', + id: this._coerceString(entry.id) || `${datasetId || 'audio'}_identity_transfer`, + model: {}, + conversion: {}, + target: {} + }; + const model = entry.model && typeof entry.model === 'object' ? entry.model : {}; + const conversion = entry.conversion && typeof entry.conversion === 'object' ? entry.conversion : {}; + const target = entry.target && typeof entry.target === 'object' ? entry.target : {}; + + normalized.model.train_if_missing = this.normalizeBoolean(model.train_if_missing); + normalized.model.force_retrain = this.normalizeBoolean(model.force_retrain); + normalized.model.build_index = model.build_index === undefined ? true : this.normalizeBoolean(model.build_index); + normalized.model.reuse_from_hub = model.reuse_from_hub === undefined ? true : this.normalizeBoolean(model.reuse_from_hub); + normalized.model.push_to_hub = this.normalizeBoolean(model.push_to_hub); + + const hubModelId = this._coerceString(model.hub_model_id); + if (hubModelId) normalized.model.hub_model_id = hubModelId; + const cacheDir = this._coerceString(model.cache_dir); + if (cacheDir) normalized.model.cache_dir = cacheDir; + + const audioMode = this._coerceString(conversion.audio_mode) || 'vocal_only'; + if (['vocal_only', 'separate_convert_remix', 'full_mix_convert'].includes(audioMode)) { + normalized.conversion.audio_mode = audioMode; + } else { + normalized.conversion.audio_mode = 'vocal_only'; + } + normalized.conversion.separation_method = this._coerceString(conversion.separation_method) || 'demucs'; + + const targetId = this._coerceString(target.id); + if (targetId) normalized.target.id = targetId; + const targetDir = this._coerceString(target.instance_data_dir); + if (targetDir) normalized.target.instance_data_dir = targetDir; + const captionStrategy = this._coerceString(target.caption_strategy); + if (captionStrategy) normalized.target.caption_strategy = captionStrategy; + + return normalized; + }); + }, safeParseJSON(text) { if (typeof text !== 'string') { return null; @@ -2491,6 +2561,14 @@ if (audioOptions) { cleaned.audio = audioOptions; } + const dataTransforms = this.normalizeDataTransforms(dataset.data_transforms, datasetId, cleaned.dataset_type); + if (dataTransforms.length) { + cleaned.data_transforms = dataTransforms; + } else { + delete cleaned.data_transforms; + } + } else { + delete cleaned.data_transforms; } if (cleaned.type !== 'local') { diff --git a/tests/js/dataloader_audio_capabilities.test.js b/tests/js/dataloader_audio_capabilities.test.js index 3dc4fd8d9..0a53b36fe 100644 --- a/tests/js/dataloader_audio_capabilities.test.js +++ b/tests/js/dataloader_audio_capabilities.test.js @@ -228,4 +228,44 @@ describe('Dataloader Section Audio Capabilities', () => { expect(component.normalizeBoolean(undefined)).toBe(false); }); }); + + describe('identity transfer helpers', () => { + test('adds default identity transfer transform for audio datasets', () => { + const component = createComponentWithContext({}); + component.markAsUnsaved = jest.fn(); + const dataset = { + id: 'voice-source', + dataset_type: 'audio', + data_transforms: [] + }; + + component.setIdentityTransferEnabled(dataset, true); + + expect(dataset.data_transforms).toHaveLength(1); + expect(dataset.data_transforms[0].task).toBe('identity_transfer'); + expect(dataset.data_transforms[0].method).toBe('rvc'); + expect(dataset.data_transforms[0].model.train_if_missing).toBe(true); + expect(dataset.data_transforms[0].model.build_index).toBe(true); + expect(dataset.data_transforms[0].conversion.audio_mode).toBe('vocal_only'); + expect(component.identityTransferEnabled(dataset)).toBe(true); + }); + + test('removes identity transfer transform without touching other transforms', () => { + const component = createComponentWithContext({}); + component.markAsUnsaved = jest.fn(); + const dataset = { + id: 'voice-source', + dataset_type: 'audio', + data_transforms: [ + { task: 'identity_transfer', method: 'rvc' }, + { task: 'other_transform' } + ] + }; + + component.setIdentityTransferEnabled(dataset, false); + + expect(dataset.data_transforms).toEqual([{ task: 'other_transform' }]); + expect(component.identityTransferEnabled(dataset)).toBe(false); + }); + }); }); diff --git a/tests/test_data_transforms_identity_transfer.py b/tests/test_data_transforms_identity_transfer.py new file mode 100644 index 000000000..80cbc42d7 --- /dev/null +++ b/tests/test_data_transforms_identity_transfer.py @@ -0,0 +1,161 @@ +import json +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock + +from simpletuner.helpers.data_transforms import process_data_transforms +from simpletuner.helpers.data_transforms.identity_transfer import ( + VOICE_TRANSFORM_FORMAT, + VOICE_TRANSFORM_FORMAT_VERSION, + IdentityTransferTransform, + RVCTransformLogger, +) + + +class TestIdentityTransferTransform(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.temp_dir) + self.source_dir = Path(self.temp_dir) / "source" + self.source_dir.mkdir() + (self.source_dir / "sample.flac").write_bytes(b"not real audio") + self.args = SimpleNamespace(output_dir=str(Path(self.temp_dir) / "output")) + self.source_backend = { + "id": "artist-source", + "type": "local", + "dataset_type": "audio", + "instance_data_dir": str(self.source_dir), + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "audio": {"sample_rate": 44100, "channels": 2}, + } + + def _transform(self, config=None, accelerator=None): + return IdentityTransferTransform( + global_config=self.args, + source_backend_config=self.source_backend, + transform_config=config or {"task": "identity_transfer"}, + accelerator=accelerator, + ) + + def test_process_data_transforms_reuses_generated_cache_as_primary_audio_dataset(self): + transform = self._transform() + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + fingerprint = transform._fingerprint(normalised) + generated_dir = Path(normalised["target"]["instance_data_dir"]) + generated_dir.mkdir(parents=True) + (generated_dir / ".simpletuner_identity_transfer.json").write_text( + json.dumps( + { + "format": VOICE_TRANSFORM_FORMAT, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + "task": "identity_transfer", + "method": "rvc", + "fingerprint": fingerprint, + } + ), + encoding="utf-8", + ) + + config = process_data_transforms( + global_config=self.args, + data_backend_config=[ + { + **self.source_backend, + "data_transforms": [{"task": "identity_transfer"}], + } + ], + ) + + generated = config[1] + self.assertEqual(generated["id"], "artist-source_identity_transfer") + self.assertEqual(generated["dataset_type"], "audio") + self.assertEqual(generated["generated_by"], "data_transforms") + self.assertEqual(generated["source_dataset_id"], "artist-source") + self.assertNotIn("auto_generated", generated) + self.assertNotIn("metadata_clone_source_id", generated) + self.assertEqual(generated["audio"]["sample_rate"], 44100) + + def test_identity_transfer_rejects_non_audio_source(self): + image_source = {**self.source_backend, "dataset_type": "image"} + with self.assertRaises(ValueError) as context: + process_data_transforms( + global_config=self.args, + data_backend_config=[ + { + **image_source, + "data_transforms": [{"task": "identity_transfer"}], + } + ], + ) + + self.assertIn("only supports source dataset_type", str(context.exception)) + + def test_rank_shard_splits_inputs_by_accelerator_rank(self): + accelerator = MagicMock() + accelerator.process_index = 1 + accelerator.num_processes = 3 + transform = self._transform(accelerator=accelerator) + + self.assertEqual(transform._rank_shard(["a", "b", "c", "d", "e", "f", "g"]), ["b", "e"]) + + def test_local_voice_model_artifact_reuses_matching_manifest(self): + transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + fingerprint = transform._fingerprint(normalised) + cache_dir = Path(normalised["model"]["cache_dir"]) + cache_dir.mkdir(parents=True) + (cache_dir / "model.pth").write_bytes(b"model") + (cache_dir / "manifest.json").write_text( + json.dumps( + { + "format": VOICE_TRANSFORM_FORMAT, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + "task": "identity_transfer", + "method": "rvc", + "fingerprint": fingerprint, + } + ), + encoding="utf-8", + ) + + artifact = transform._resolve_voice_model( + normalised, + fingerprint, + cache_dir, + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + self.assertEqual(artifact.model_path, cache_dir / "model.pth") + + def test_missing_voice_model_requires_explicit_training(self): + transform = self._transform({"task": "identity_transfer", "model": {"train_if_missing": False}}) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + + with self.assertRaises(ValueError) as context: + transform._resolve_voice_model( + normalised, + transform._fingerprint(normalised), + Path(normalised["model"]["cache_dir"]), + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + self.assertIn("train_if_missing is false", str(context.exception)) + + def test_logger_writes_local_json_files(self): + run_logger = RVCTransformLogger(str(Path(self.temp_dir) / "output")) + + run_logger.event("voice-transfer", "voice_model_reused", source="local") + run_logger.summary("voice-transfer", status="reused_generated_cache") + + self.assertTrue((Path(self.temp_dir) / "output" / "logs" / "rvc" / "training_stats.jsonl").exists()) + summary = json.loads((Path(self.temp_dir) / "output" / "logs" / "rvc" / "summary.json").read_text()) + self.assertEqual(summary["status"], "reused_generated_cache") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dataset_blueprints.py b/tests/test_dataset_blueprints.py index 452e7161b..4d1b323dd 100644 --- a/tests/test_dataset_blueprints.py +++ b/tests/test_dataset_blueprints.py @@ -44,5 +44,16 @@ def test_local_caption_blueprint_exposes_train_batch_size(self): self.assertIn("train_batch_size", {field.id for field in blueprint.fields}) +class TestAudioDatasetBlueprints(unittest.TestCase): + def test_local_audio_blueprint_exposes_data_transforms(self): + blueprint = find_blueprint("local", "audio") + + self.assertIsNotNone(blueprint) + fields = {field.id: field for field in blueprint.fields} + self.assertIn("data_transforms", fields) + self.assertEqual(fields["data_transforms"].type, "textarea") + self.assertIn("identity_transfer", fields["data_transforms"].placeholder) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_factory_edge_cases.py b/tests/test_factory_edge_cases.py index 289ddb410..5d58654a3 100644 --- a/tests/test_factory_edge_cases.py +++ b/tests/test_factory_edge_cases.py @@ -824,6 +824,7 @@ def test_inline_conditioning_auto_generation_for_image_dataset(self): self.assertEqual(generated.get("dataset_type"), "conditioning") self.assertTrue(generated.get("auto_generated")) self.assertEqual(generated.get("source_dataset_id"), "primary") + self.assertEqual(generated.get("metadata_clone_source_id"), "primary") conditioning_cfg = generated.get("conditioning_config") or {} self.assertEqual(conditioning_cfg.get("type"), "canny") self.assertEqual(conditioning_cfg.get("conditioning_type"), "controlnet") @@ -865,6 +866,7 @@ def test_i2v_video_without_conditioning_uses_first_frame_generator(self): self.assertEqual(generated.get("type"), "local") self.assertEqual(generated.get("metadata_backend"), "discovery") self.assertEqual(generated.get("source_dataset_id"), "h3_video") + self.assertEqual(generated.get("metadata_clone_source_id"), "h3_video") self.assertNotIn("video", generated) conditioning_cfg = generated.get("conditioning_config") or {} self.assertEqual(conditioning_cfg.get("type"), "i2v_first_frame") @@ -1746,6 +1748,53 @@ def test_main_process_reloads_refreshed_bucket_cache_before_split(self): metadata_backend.reload_cache.assert_called_once_with() + @patch("simpletuner.helpers.data_backend.factory.DatasetDuplicator.copy_metadata") + @patch("simpletuner.helpers.data_backend.factory.StateTracker") + def test_metadata_clone_requires_explicit_source_marker(self, mock_state_tracker, mock_copy_metadata): + from simpletuner.helpers.data_backend.factory import FactoryRegistry + + self.args.skip_file_discovery = "aspect" + self.args.eval_dataset_id = None + self.args.max_train_steps = 100 + self.args.allow_dataset_oversubscription = False + metadata_backend = MagicMock() + init_backend = { + "id": "generated-audio", + "config": {}, + "dataset_type": "audio", + "metadata_backend": metadata_backend, + } + factory = FactoryRegistry( + args=self.args, + accelerator=self.accelerator, + text_encoders=self.text_encoders, + tokenizers=self.tokenizers, + model=self.model, + ) + factory._handle_config_versioning = MagicMock() + + factory._handle_bucket_operations( + backend={"id": "generated-audio", "auto_generated": True, "skip_file_discovery": "aspect"}, + init_backend=init_backend, + conditioning_type=None, + ) + + mock_copy_metadata.assert_not_called() + + factory._handle_bucket_operations( + backend={ + "id": "generated-conditioning", + "auto_generated": True, + "metadata_clone_source_id": "primary", + "skip_file_discovery": "aspect", + }, + init_backend=init_backend, + conditioning_type=None, + ) + + mock_state_tracker.get_data_backend.assert_called_once_with("primary") + mock_copy_metadata.assert_called_once() + def test_image_embeds_backend_configuration(self): """Image embed configuration should not instantiate VAE cache directly.""" from simpletuner.helpers.data_backend.factory import FactoryRegistry From cc544a2be9ff462dcb1c1eb9751c26bf85c125f6 Mon Sep 17 00:00:00 2001 From: bghira Date: Mon, 24 Aug 2026 17:34:20 -0600 Subject: [PATCH 02/12] Address identity transfer review comments --- .../data_transforms/identity_transfer.py | 1 - tests/test_webui_e2e.py | 126 +++++++++++++++++- 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py index 139eec593..74b5f6adf 100644 --- a/simpletuner/helpers/data_transforms/identity_transfer.py +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -223,7 +223,6 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: transform = self._normalise_transform_config(existing_backend_ids) transform_id = transform["id"] output_dir = self._output_dir() - transform_root = Path(output_dir) / "cache" / "data_transforms" / transform_id model_cache_dir = Path(transform["model"]["cache_dir"]) generated_dir = Path(transform["target"]["instance_data_dir"]) fingerprint = self._fingerprint(transform) diff --git a/tests/test_webui_e2e.py b/tests/test_webui_e2e.py index 8271b0325..9e0e5511e 100644 --- a/tests/test_webui_e2e.py +++ b/tests/test_webui_e2e.py @@ -1699,7 +1699,7 @@ def element_is_visible(driver): return el && el.offsetParent !== null && el.offsetHeight > 0; """ ) - except: + except Exception: return False self.assertTrue(wait.until(element_is_visible), f"Tab {tab_name} failed to load") @@ -1786,7 +1786,7 @@ def scenario(driver, _browser): try: overlay = driver.find_element(By.CSS_SELECTOR, ".onboarding-overlay") print(f"DEBUG: Overlay found but not visible, style: {overlay.get_attribute('style')}") - except: + except Exception: print("DEBUG: Overlay element not found at all") self.fail("Onboarding overlay not visible after waiting") @@ -2090,6 +2090,128 @@ def scenario(driver, _browser): self.for_each_browser("test_dataset_vae_cache_options_mark_unsaved", scenario) + def test_audio_identity_transfer_controls_mark_unsaved(self) -> None: + """Audio identity-transfer controls should update Alpine state and enable saving.""" + self.seed_defaults() + + def scenario(driver, _browser): + trainer_page = self._trainer_page(driver) + datasets_tab = DatasetsTab(driver, base_url=self.base_url) + + trainer_page.navigate_to_trainer() + self.dismiss_onboarding(driver) + trainer_page.switch_to_datasets_tab() + trainer_page.wait_for_tab("datasets") + + datasets_tab.add_dataset("audio") + dataset_index = datasets_tab.get_dataset_count() - 1 + driver.execute_script( + "const store = window.Alpine && Alpine.store ? Alpine.store('trainer') : null;" + "if (store) { store.hasUnsavedChanges = false; }" + ) + + dataset_id = driver.execute_script( + "const comp = window.dataloaderSectionComponentInstance;" + "const dataset = comp && comp.datasets ? comp.datasets[arguments[0]] : null;" + "if (!dataset) { return null; }" + "if (typeof comp.expandSectionIfCollapsed === 'function') { comp.expandSectionIfCollapsed(dataset, 'card'); }" + "if (typeof comp.setListTab === 'function') { comp.setListTab(dataset, 'audio'); }" + "comp.parameterFilterQuery = '';" + "return dataset.id;", + dataset_index, + ) + self.assertIsInstance(dataset_id, str) + + checkbox_id = f"audio-identity-transfer-{dataset_id}" + WebDriverWait(driver, 10).until( + lambda d: d.execute_script("return !!document.getElementById(arguments[0]);", checkbox_id) + ) + self.assertTrue( + driver.execute_script( + "const checkbox = document.getElementById(arguments[0]);" + "checkbox.scrollIntoView({ block: 'center' });" + "checkbox.click();" + "return checkbox.checked;", + checkbox_id, + ) + ) + + enabled_state = WebDriverWait(driver, 5).until( + lambda d: d.execute_script( + """ + const comp = window.dataloaderSectionComponentInstance; + const store = window.Alpine && Alpine.store ? Alpine.store('trainer') : null; + const saveButton = document.querySelector('button[aria-label="Save dataset configuration"]'); + const dataset = comp && Array.isArray(comp.datasets) + ? comp.datasets.find((candidate) => candidate && candidate.id === arguments[0]) + : null; + const transform = dataset && Array.isArray(dataset.data_transforms) + ? dataset.data_transforms.find((entry) => entry && entry.task === 'identity_transfer') + : null; + return transform && transform.method === 'rvc' + && store && store.hasUnsavedChanges === true + && saveButton && saveButton.offsetParent !== null + && saveButton.classList.contains('btn-warning') + ? { dirty: store.hasUnsavedChanges, transformId: transform.id } + : null; + """, + dataset_id, + ) + ) + self.assertTrue(enabled_state["dirty"]) + self.assertEqual(enabled_state["transformId"], f"{dataset_id}_identity_transfer") + + driver.execute_script( + "const store = window.Alpine && Alpine.store ? Alpine.store('trainer') : null;" + "if (store) { store.hasUnsavedChanges = false; }" + ) + + WebDriverWait(driver, 10).until( + lambda d: d.execute_script( + """ + return Array.from(document.querySelectorAll('.dataset-list-item-expanded input')) + .some((input) => input.placeholder === 'org/voice-artifact'); + """ + ) + ) + driver.execute_script( + """ + const input = Array.from(document.querySelectorAll('.dataset-list-item-expanded input')) + .find((candidate) => candidate.placeholder === 'org/voice-artifact'); + input.scrollIntoView({ block: 'center' }); + input.value = 'RareConcepts/test-voice'; + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('change', { bubbles: true })); + """ + ) + + edited_state = WebDriverWait(driver, 5).until( + lambda d: d.execute_script( + """ + const comp = window.dataloaderSectionComponentInstance; + const store = window.Alpine && Alpine.store ? Alpine.store('trainer') : null; + const saveButton = document.querySelector('button[aria-label="Save dataset configuration"]'); + const dataset = comp && Array.isArray(comp.datasets) + ? comp.datasets.find((candidate) => candidate && candidate.id === arguments[0]) + : null; + const transform = dataset && Array.isArray(dataset.data_transforms) + ? dataset.data_transforms.find((entry) => entry && entry.task === 'identity_transfer') + : null; + return transform && transform.model && transform.model.hub_model_id === 'RareConcepts/test-voice' + && store && store.hasUnsavedChanges === true + && saveButton && saveButton.offsetParent !== null + && saveButton.classList.contains('btn-warning') + ? { hubModelId: transform.model.hub_model_id, dirty: store.hasUnsavedChanges } + : null; + """, + dataset_id, + ) + ) + self.assertTrue(edited_state["dirty"]) + self.assertEqual(edited_state["hubModelId"], "RareConcepts/test-voice") + + self.for_each_browser("test_audio_identity_transfer_controls_mark_unsaved", scenario) + def test_text_embed_cache_options_mark_unsaved(self) -> None: """Text cache mode controls should update the text_embeds entry and mark datasets dirty.""" self.seed_defaults() From 504d9f0668f19ac3b4a02cdc08beab7265fdae21 Mon Sep 17 00:00:00 2001 From: bghira Date: Tue, 25 Aug 2026 18:11:39 -0600 Subject: [PATCH 03/12] Require huggingface-hub-rvc 0.1.3 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 376b7c44e..1b8eddbb4 100644 --- a/setup.py +++ b/setup.py @@ -320,6 +320,7 @@ def _collect_package_files(*directories: str): "kernels>=0.16.0,<0.17.0", "torch-optimi>=0.2.1", "librosa>=0.10.2", + "huggingface-hub-rvc>=0.1.3", "loguru>=0.7.2", "toml>=0.10.2", "fastapi[standard]>=0.115.0", From 411db4775c42cd6a4fc0f8c2903f2e3bce75fbc9 Mon Sep 17 00:00:00 2001 From: bghira Date: Wed, 26 Aug 2026 16:08:51 -0600 Subject: [PATCH 04/12] Add RVC identity transfer data transform --- setup.py | 11 +- .../data_transforms/identity_transfer.py | 246 ++++++++++++----- simpletuner/helpers/rvc/__init__.py | 1 + simpletuner/helpers/rvc/simple.py | 17 ++ .../test_data_transforms_identity_transfer.py | 252 +++++++++++++++++- tests/test_rvc_smoke.py | 195 ++++++++++++++ 6 files changed, 649 insertions(+), 73 deletions(-) create mode 100644 simpletuner/helpers/rvc/__init__.py create mode 100644 simpletuner/helpers/rvc/simple.py create mode 100644 tests/test_rvc_smoke.py diff --git a/setup.py b/setup.py index 1b8eddbb4..11102a86e 100644 --- a/setup.py +++ b/setup.py @@ -320,7 +320,16 @@ def _collect_package_files(*directories: str): "kernels>=0.16.0,<0.17.0", "torch-optimi>=0.2.1", "librosa>=0.10.2", + "faiss-cpu>=1.13.0,<2", + "onnxruntime>=1.24.4,<2; platform_system != 'Windows'", + "onnxruntime-directml>=1.24.4,<2; platform_system == 'Windows'", + "praat-parselmouth>=0.4.5,<1", + "scikit-learn>=1.6.0,<2", + "soundfile>=0.13.0,<1", "huggingface-hub-rvc>=0.1.3", + "demucs>=4.0.1", + "lyricsgenius>=3.0.1", + "mutagen>=1.47.0", "loguru>=0.7.2", "toml>=0.10.2", "fastapi[standard]>=0.115.0", @@ -432,7 +441,7 @@ def _collect_package_files(*directories: str): "helpers/ramtorch/LICENSE", ], }, - python_requires=">=3.12,<3.14", + python_requires=">=3.12,<3.15", install_requires=base_deps, extras_require=extras_require, entry_points={ diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py index 74b5f6adf..f6afedfdf 100644 --- a/simpletuner/helpers/data_transforms/identity_transfer.py +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -7,6 +7,7 @@ import logging import os import shutil +import tempfile from copy import deepcopy from dataclasses import dataclass from datetime import datetime, timezone @@ -101,13 +102,28 @@ def train( source_backend_config: Dict[str, Any], transform_config: Dict[str, Any], cache_dir: Path, + fingerprint: str, + manifest_base: Dict[str, Any], accelerator: Any = None, logger: Optional[RVCTransformLogger] = None, ) -> VoiceModelArtifact: - world_size = int(getattr(accelerator, "num_processes", 1) or 1) - raise NotImplementedError( - "On-demand RVC training is configured, but the native RVC trainer is not implemented yet. " - f"The transform runner reached the trainer with DDP world_size={world_size}." + from simpletuner.helpers.rvc.simple import SimpleRVCTrainer + + artifact = SimpleRVCTrainer().train( + source_backend_config=source_backend_config, + transform_config=transform_config, + cache_dir=cache_dir, + fingerprint=fingerprint, + manifest_base=manifest_base, + accelerator=accelerator, + run_logger=logger, + ) + return VoiceModelArtifact( + cache_dir=artifact.cache_dir, + manifest_path=artifact.manifest_path, + model_path=artifact.model_path, + index_path=artifact.index_path, + manifest=artifact.manifest, ) @@ -122,81 +138,82 @@ def convert( accelerator: Any = None, logger: Optional[RVCTransformLogger] = None, ) -> None: - raise NotImplementedError( - "RVC identity transfer conversion is configured, but the native converter is not implemented yet." + from simpletuner.helpers.rvc.simple import SimpleRVCArtifact, SimpleRVCConverter + + simple_artifact = SimpleRVCArtifact( + cache_dir=artifact.cache_dir, + manifest_path=artifact.manifest_path, + model_path=artifact.model_path, + index_path=artifact.index_path, + manifest=artifact.manifest, + ) + SimpleRVCConverter().convert( + source_backend_config=source_backend_config, + target_backend_config=target_backend_config, + transform_config=transform_config, + artifact=simple_artifact, + input_paths=input_paths, + accelerator=accelerator, + run_logger=logger, ) class HubVoiceModelCache: - def __init__(self, hub_model_id: str, token: Optional[str] = None) -> None: + def __init__(self, hub_model_id: str, token: Optional[str] = None, public: bool = False) -> None: self.hub_model_id = hub_model_id self.token = token + self.public = public def download_if_compatible(self, cache_dir: Path, fingerprint: str) -> Optional[VoiceModelArtifact]: try: - from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError + from huggingface_hub_rvc import RVCPipeline except ImportError as exc: - raise ImportError("huggingface_hub is required when identity_transfer.reuse_from_hub is enabled.") from exc + raise ImportError( + "huggingface_hub and huggingface_hub_rvc are required when identity_transfer.reuse_from_hub is enabled." + ) from exc try: - manifest_file = hf_hub_download( - repo_id=self.hub_model_id, - filename="voice_transform/manifest.json", + pipeline = RVCPipeline.from_pretrained( + self.hub_model_id, token=self.token, ) except (EntryNotFoundError, RepositoryNotFoundError): return None - manifest = json.loads(Path(manifest_file).read_text(encoding="utf-8")) + manifest = pipeline.artifact.manifest if not _manifest_matches(manifest, fingerprint): return None - model_file = hf_hub_download( - repo_id=self.hub_model_id, - filename="voice_transform/model.pth", - token=self.token, - ) - index_file: Optional[str] - try: - index_file = hf_hub_download( - repo_id=self.hub_model_id, - filename="voice_transform/index.index", - token=self.token, - ) - except EntryNotFoundError: - index_file = None - - cache_dir.mkdir(parents=True, exist_ok=True) - local_manifest = cache_dir / "manifest.json" - local_model = cache_dir / "model.pth" - local_index = cache_dir / "index.index" - shutil.copy2(manifest_file, local_manifest) - shutil.copy2(model_file, local_model) - if index_file: - shutil.copy2(index_file, local_index) - return VoiceModelArtifact( - cache_dir=cache_dir, - manifest_path=local_manifest, - model_path=local_model, - index_path=local_index if index_file else None, - manifest=manifest, - ) + pipeline.save_pretrained(cache_dir) + return _voice_model_artifact_from_hub_pipeline(RVCPipeline.from_pretrained(cache_dir, local_files_only=True)) def upload(self, artifact: VoiceModelArtifact) -> None: try: - from huggingface_hub import HfApi + from huggingface_hub_rvc import RVCConfig, RVCPipeline + from huggingface_hub_rvc._runtime import SimpleRVCArtifact except ImportError as exc: - raise ImportError("huggingface_hub is required when identity_transfer.push_to_hub is enabled.") from exc - - api = HfApi(token=self.token) - api.create_repo(repo_id=self.hub_model_id, repo_type="model", exist_ok=True) - api.upload_folder( - repo_id=self.hub_model_id, - repo_type="model", - folder_path=str(artifact.cache_dir), - path_in_repo="voice_transform", + raise ImportError("huggingface_hub_rvc is required when identity_transfer.push_to_hub is enabled.") from exc + + simple_artifact = SimpleRVCArtifact( + cache_dir=artifact.cache_dir, + manifest_path=artifact.manifest_path, + model_path=artifact.model_path, + index_path=artifact.index_path, + manifest=artifact.manifest, + ) + pipeline = RVCPipeline( + artifact=simple_artifact, + config=RVCConfig(model_name=_voice_model_name(artifact.manifest, self.hub_model_id)), ) + with tempfile.TemporaryDirectory() as temp_dir: + pipeline.save_pretrained( + temp_dir, + push_to_hub=True, + repo_id=self.hub_model_id, + token=self.token, + private=not self.public, + ) def _manifest_matches(manifest: Dict[str, Any], fingerprint: str) -> bool: @@ -209,6 +226,27 @@ def _manifest_matches(manifest: Dict[str, Any], fingerprint: str) -> bool: ) +def _voice_model_artifact_from_hub_pipeline(pipeline: Any) -> VoiceModelArtifact: + artifact = pipeline.artifact + return VoiceModelArtifact( + cache_dir=artifact.cache_dir, + manifest_path=artifact.manifest_path, + model_path=artifact.model_path, + index_path=artifact.index_path, + manifest=artifact.manifest, + ) + + +def _voice_model_name(manifest: Dict[str, Any], hub_model_id: Optional[str] = None) -> str: + voice_model = manifest.get("voice_model") or {} + name = manifest.get("model_name") or voice_model.get("model_name") + if name: + return str(name) + if hub_model_id: + return hub_model_id.rstrip("/").rsplit("/", 1)[-1] + return "RVC Voice Model" + + @register_data_transform class IdentityTransferTransform(DataTransformTask): TASK = "identity_transfer" @@ -225,18 +263,19 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: output_dir = self._output_dir() model_cache_dir = Path(transform["model"]["cache_dir"]) generated_dir = Path(transform["target"]["instance_data_dir"]) - fingerprint = self._fingerprint(transform) + generated_fingerprint = self._generated_fingerprint(transform) + voice_model_fingerprint = self._voice_model_fingerprint(transform) run_logger = RVCTransformLogger(output_dir, accelerator=self.accelerator) target_backend_config = self._target_backend_config(transform, generated_dir) - if self._generated_cache_matches(generated_dir, fingerprint): + if self._generated_cache_matches(generated_dir, generated_fingerprint): run_logger.event(transform_id, "generated_cache_reused", path=str(generated_dir)) run_logger.summary( transform_id, status="reused_generated_cache", generated_backend_id=target_backend_config["id"] ) return [target_backend_config] - artifact = self._resolve_voice_model(transform, fingerprint, model_cache_dir, run_logger) + artifact = self._resolve_voice_model(transform, voice_model_fingerprint, model_cache_dir, run_logger) if not self._is_main_process(): self._wait_for_everyone() run_logger.summary( @@ -261,7 +300,7 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: accelerator=self.accelerator, logger=run_logger, ) - self._write_generated_manifest(generated_dir, fingerprint, transform) + self._write_generated_manifest(generated_dir, generated_fingerprint, transform) self._wait_for_everyone() run_logger.summary(transform_id, status="generated", generated_backend_id=target_backend_config["id"]) return [target_backend_config] @@ -287,12 +326,23 @@ def _normalise_transform_config(self, existing_backend_ids: set[str]) -> Dict[st model.setdefault("build_index", True) model.setdefault("reuse_from_hub", bool(model.get("hub_model_id"))) model.setdefault("push_to_hub", False) + model.setdefault("public", False) model.setdefault("cache_dir", str(transform_root / "rvc_model")) + model.setdefault("asset_hub_model_id", "lj1995/VoiceConversionWebUI") + model.setdefault("model_name", _voice_model_name({}, model.get("hub_model_id") or transform_id)) + model.setdefault("sample_rate", 48000) + model.setdefault("identity_audio_mode", "separate") + model.setdefault("training_steps", 1000) + model.setdefault("batch_size", 4) + model.setdefault("learning_rate", 1e-4) + model.setdefault("max_seconds_per_file", 180.0) transform["model"] = model conversion = deepcopy(transform.get("conversion") or {}) - conversion.setdefault("audio_mode", "vocal_only") + conversion.setdefault("audio_mode", "separate_convert_remix") conversion.setdefault("separation_method", "demucs") + conversion.setdefault("timbre_strength", 1.0) + conversion.setdefault("retrieval_strength", 0.75) transform["conversion"] = conversion target = deepcopy(transform.get("target") or {}) @@ -379,27 +429,36 @@ def _resolve_voice_model( source_backend_config=self.source_backend_config, transform_config=transform, cache_dir=model_cache_dir, + fingerprint=fingerprint, + manifest_base=self._manifest(fingerprint, transform), accelerator=self.accelerator, logger=run_logger, ) if model_cfg.get("push_to_hub", False): if not hub_model_id: raise ValueError("identity_transfer.model.push_to_hub requires identity_transfer.model.hub_model_id.") - HubVoiceModelCache(hub_model_id, token=model_cfg.get("hub_token")).upload(artifact) + HubVoiceModelCache( + hub_model_id, + token=model_cfg.get("hub_token"), + public=bool(model_cfg.get("public", False)), + ).upload(artifact) run_logger.event(transform_id, "voice_model_pushed", hub_model_id=hub_model_id) return artifact def _local_artifact(self, cache_dir: Path, fingerprint: str) -> Optional[VoiceModelArtifact]: - manifest_path = cache_dir / "manifest.json" - model_path = cache_dir / "model.pth" - index_path = cache_dir / "index.index" + artifact_dir = cache_dir / "voice_transform" if (cache_dir / "voice_transform").exists() else cache_dir + manifest_path = artifact_dir / "manifest.json" + model_path = artifact_dir / "model.safetensors" + if not model_path.exists(): + model_path = artifact_dir / "model.pth" + index_path = artifact_dir / "index.index" if not manifest_path.exists() or not model_path.exists(): return None manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if not _manifest_matches(manifest, fingerprint): return None return VoiceModelArtifact( - cache_dir=cache_dir, + cache_dir=artifact_dir, manifest_path=manifest_path, model_path=model_path, index_path=index_path if index_path.exists() else None, @@ -415,32 +474,53 @@ def _generated_cache_matches(self, generated_dir: Path, fingerprint: str) -> boo def _write_generated_manifest(self, generated_dir: Path, fingerprint: str, transform: Dict[str, Any]) -> None: generated_dir.mkdir(parents=True, exist_ok=True) - manifest = self._manifest(fingerprint, transform) + manifest = self._manifest(fingerprint, transform, source_dataset_id=self.source_backend_config["id"]) (generated_dir / ".simpletuner_identity_transfer.json").write_text( json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8", ) - def _manifest(self, fingerprint: str, transform: Dict[str, Any]) -> Dict[str, Any]: - return { + def _manifest( + self, + fingerprint: str, + transform: Dict[str, Any], + source_dataset_id: Optional[str] = None, + ) -> Dict[str, Any]: + manifest = { "format": VOICE_TRANSFORM_FORMAT, "format_version": VOICE_TRANSFORM_FORMAT_VERSION, "task": self.TASK, "method": "rvc", "fingerprint": fingerprint, - "source_dataset_id": self.source_backend_config["id"], "transform_id": transform["id"], "created_by": "simpletuner", "created_at": _utc_now(), } + if source_dataset_id is not None: + manifest["source_dataset_id"] = source_dataset_id + return manifest - def _fingerprint(self, transform: Dict[str, Any]) -> str: + def _generated_fingerprint(self, transform: Dict[str, Any]) -> str: source_cfg = deepcopy(self.source_backend_config) source_cfg.pop("data_transforms", None) transform_for_hash = deepcopy(transform) model_cfg = transform_for_hash.get("model") or {} - for transient_key in ("hub_token", "push_to_hub", "reuse_from_hub"): + for transient_key in ( + "asset_hub_token", + "cache_dir", + "device", + "demucs_device", + "force_retrain", + "hub_model_id", + "hub_token", + "model_name", + "push_to_hub", + "public", + "reuse_from_hub", + "train_if_missing", + ): model_cfg.pop(transient_key, None) + model_cfg["voice_model_fingerprint"] = self._voice_model_fingerprint(transform) transform_for_hash["model"] = model_cfg return _sha256_json( { @@ -450,6 +530,34 @@ def _fingerprint(self, transform: Dict[str, Any]) -> str: } ) + def _voice_model_fingerprint(self, transform: Dict[str, Any]) -> str: + model_cfg = deepcopy(transform.get("model") or {}) + for transient_key in ( + "asset_hub_token", + "cache_dir", + "device", + "demucs_device", + "force_retrain", + "hub_model_id", + "hub_token", + "model_name", + "push_to_hub", + "public", + "reuse_from_hub", + "train_if_missing", + ): + model_cfg.pop(transient_key, None) + return _sha256_json( + { + "method": transform.get("method", "rvc"), + "model": model_cfg, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + } + ) + + def _fingerprint(self, transform: Dict[str, Any]) -> str: + return self._generated_fingerprint(transform) + def _discover_source_audio_paths(self) -> List[str]: source_type = self.source_backend_config.get("type") if source_type != "local": diff --git a/simpletuner/helpers/rvc/__init__.py b/simpletuner/helpers/rvc/__init__.py new file mode 100644 index 000000000..67556db26 --- /dev/null +++ b/simpletuner/helpers/rvc/__init__.py @@ -0,0 +1 @@ +"""Small voice-transfer helpers used by audio data transforms.""" diff --git a/simpletuner/helpers/rvc/simple.py b/simpletuner/helpers/rvc/simple.py new file mode 100644 index 000000000..97f06a978 --- /dev/null +++ b/simpletuner/helpers/rvc/simple.py @@ -0,0 +1,17 @@ +"""Compatibility imports for the SimpleTuner RVC transform API.""" + +from huggingface_hub_rvc._runtime import ( + RVCRecord, + SimpleRVCArtifact, + SimpleRVCConverter, + SimpleRVCTrainer, + _load_model_payload, +) + +__all__ = [ + "RVCRecord", + "SimpleRVCArtifact", + "SimpleRVCConverter", + "SimpleRVCTrainer", + "_load_model_payload", +] diff --git a/tests/test_data_transforms_identity_transfer.py b/tests/test_data_transforms_identity_transfer.py index 80cbc42d7..2395d5990 100644 --- a/tests/test_data_transforms_identity_transfer.py +++ b/tests/test_data_transforms_identity_transfer.py @@ -1,11 +1,15 @@ import json +import math import os import shutil import tempfile import unittest +import wave from pathlib import Path from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch + +import torch from simpletuner.helpers.data_transforms import process_data_transforms from simpletuner.helpers.data_transforms.identity_transfer import ( @@ -13,6 +17,7 @@ VOICE_TRANSFORM_FORMAT_VERSION, IdentityTransferTransform, RVCTransformLogger, + VoiceModelArtifact, ) @@ -34,6 +39,19 @@ def setUp(self): "audio": {"sample_rate": 44100, "channels": 2}, } + def _write_wav(self, path: Path, frequency: float = 220.0, sample_rate: int = 16000, seconds: float = 0.25): + sample_count = int(sample_rate * seconds) + path.parent.mkdir(parents=True, exist_ok=True) + with wave.open(str(path), "w") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(sample_rate) + frames = bytearray() + for idx in range(sample_count): + value = int(math.sin(2.0 * math.pi * frequency * idx / sample_rate) * 12000) + frames.extend(value.to_bytes(2, byteorder="little", signed=True)) + handle.writeframes(bytes(frames)) + def _transform(self, config=None, accelerator=None): return IdentityTransferTransform( global_config=self.args, @@ -106,7 +124,40 @@ def test_rank_shard_splits_inputs_by_accelerator_rank(self): def test_local_voice_model_artifact_reuses_matching_manifest(self): transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) - fingerprint = transform._fingerprint(normalised) + + self.assertFalse(normalised["model"]["push_to_hub"]) + self.assertFalse(normalised["model"]["public"]) + + fingerprint = transform._voice_model_fingerprint(normalised) + cache_dir = Path(normalised["model"]["cache_dir"]) + cache_dir.mkdir(parents=True) + (cache_dir / "model.safetensors").write_bytes(b"model") + (cache_dir / "manifest.json").write_text( + json.dumps( + { + "format": VOICE_TRANSFORM_FORMAT, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + "task": "identity_transfer", + "method": "rvc", + "fingerprint": fingerprint, + } + ), + encoding="utf-8", + ) + + artifact = transform._resolve_voice_model( + normalised, + fingerprint, + cache_dir, + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + self.assertEqual(artifact.model_path, cache_dir / "model.safetensors") + + def test_local_voice_model_artifact_loads_legacy_pth_manifest(self): + transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + fingerprint = transform._voice_model_fingerprint(normalised) cache_dir = Path(normalised["model"]["cache_dir"]) cache_dir.mkdir(parents=True) (cache_dir / "model.pth").write_bytes(b"model") @@ -132,6 +183,39 @@ def test_local_voice_model_artifact_reuses_matching_manifest(self): self.assertEqual(artifact.model_path, cache_dir / "model.pth") + def test_local_voice_model_artifact_reuses_hub_rvc_layout(self): + transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + fingerprint = transform._voice_model_fingerprint(normalised) + cache_dir = Path(normalised["model"]["cache_dir"]) + voice_dir = cache_dir / "voice_transform" + voice_dir.mkdir(parents=True) + (cache_dir / "config.json").write_text('{"model_name": "Test Voice"}', encoding="utf-8") + (voice_dir / "model.safetensors").write_bytes(b"model") + (voice_dir / "manifest.json").write_text( + json.dumps( + { + "format": VOICE_TRANSFORM_FORMAT, + "format_version": VOICE_TRANSFORM_FORMAT_VERSION, + "task": "identity_transfer", + "method": "rvc", + "fingerprint": fingerprint, + "model_name": "Test Voice", + } + ), + encoding="utf-8", + ) + + artifact = transform._resolve_voice_model( + normalised, + fingerprint, + cache_dir, + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + self.assertEqual(artifact.cache_dir, voice_dir) + self.assertEqual(artifact.model_path, voice_dir / "model.safetensors") + def test_missing_voice_model_requires_explicit_training(self): transform = self._transform({"task": "identity_transfer", "model": {"train_if_missing": False}}) normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) @@ -139,13 +223,175 @@ def test_missing_voice_model_requires_explicit_training(self): with self.assertRaises(ValueError) as context: transform._resolve_voice_model( normalised, - transform._fingerprint(normalised), + transform._voice_model_fingerprint(normalised), Path(normalised["model"]["cache_dir"]), RVCTransformLogger(str(Path(self.temp_dir) / "output")), ) self.assertIn("train_if_missing is false", str(context.exception)) + def test_push_to_hub_passes_public_flag_to_hub_cache(self): + transform = self._transform( + { + "task": "identity_transfer", + "id": "voice-transfer", + "model": { + "identity_data_dir": str(Path(self.temp_dir) / "identity"), + "hub_model_id": "org/target-voice-rvc", + "reuse_from_hub": False, + "push_to_hub": True, + "public": True, + }, + } + ) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + fingerprint = transform._voice_model_fingerprint(normalised) + cache_dir = Path(normalised["model"]["cache_dir"]) + + def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): + cache_dir.mkdir(parents=True) + model_path = cache_dir / "model.pth" + manifest_path = cache_dir / "manifest.json" + model_path.write_bytes(b"rvc") + manifest = {**manifest_base, "fingerprint": fingerprint, "voice_model": {"kind": "simpletuner-rvc-v2-f0"}} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return VoiceModelArtifact(cache_dir, manifest_path, model_path, None, manifest) + + with ( + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCTrainer.train", new=fake_train), + patch("simpletuner.helpers.data_transforms.identity_transfer.HubVoiceModelCache") as hub_cache, + ): + transform._resolve_voice_model( + normalised, + fingerprint, + cache_dir, + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + hub_cache.assert_called_once_with("org/target-voice-rvc", token=None, public=True) + hub_cache.return_value.upload.assert_called_once() + + def test_voice_model_fingerprint_is_independent_of_conversion_source(self): + transform = self._transform( + { + "task": "identity_transfer", + "id": "voice-transfer", + "model": {"identity_data_dir": str(Path(self.temp_dir) / "identity")}, + "target": {"instance_data_dir": str(Path(self.temp_dir) / "generated-a")}, + } + ) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + + other_source = { + **self.source_backend, + "id": "other-source", + "instance_data_dir": str(Path(self.temp_dir) / "other-source"), + } + other_transform = IdentityTransferTransform( + global_config=self.args, + source_backend_config=other_source, + transform_config={ + "task": "identity_transfer", + "id": "voice-transfer-other", + "model": {"identity_data_dir": str(Path(self.temp_dir) / "identity")}, + "target": {"instance_data_dir": str(Path(self.temp_dir) / "generated-b")}, + }, + ) + other_normalised = other_transform._normalise_transform_config(existing_backend_ids={"other-source"}) + + self.assertEqual( + transform._voice_model_fingerprint(normalised), + other_transform._voice_model_fingerprint(other_normalised), + ) + self.assertNotEqual( + transform._generated_fingerprint(normalised), + other_transform._generated_fingerprint(other_normalised), + ) + + def test_process_data_transforms_trains_and_converts_with_rvc_artifact(self): + source_dir = Path(self.temp_dir) / "convert-source" + identity_dir = Path(self.temp_dir) / "identity" + generated_dir = Path(self.temp_dir) / "generated" + self._write_wav(source_dir / "source.wav", frequency=180.0) + self._write_wav(identity_dir / "voice.wav", frequency=320.0) + (source_dir / "source.txt").write_text("rock vocal test", encoding="utf-8") + + def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): + cache_dir.mkdir(parents=True) + model_path = cache_dir / "model.pth" + manifest_path = cache_dir / "manifest.json" + model_path.write_bytes(b"rvc") + manifest = {**manifest_base, "fingerprint": fingerprint, "voice_model": {"kind": "simpletuner-rvc-v2-f0"}} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return VoiceModelArtifact(cache_dir, manifest_path, model_path, None, manifest) + + def fake_convert(_self, source_backend_config, target_backend_config, *_args, **_kwargs): + output_dir = Path(target_backend_config["instance_data_dir"]) + output_dir.mkdir(parents=True) + shutil.copy2(Path(source_backend_config["instance_data_dir"]) / "source.wav", output_dir / "source.wav") + shutil.copy2(Path(source_backend_config["instance_data_dir"]) / "source.txt", output_dir / "source.txt") + + with ( + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCTrainer.train", new=fake_train), + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCConverter.convert", new=fake_convert), + ): + config = process_data_transforms( + global_config=self.args, + data_backend_config=[ + { + "id": "convert-source", + "type": "local", + "dataset_type": "audio", + "instance_data_dir": str(source_dir), + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "audio": {"sample_rate": 48000, "channels": 1}, + "data_transforms": [ + { + "task": "identity_transfer", + "id": "voice-transfer", + "model": { + "identity_data_dir": str(identity_dir), + "sample_rate": 48000, + "identity_audio_mode": "vocal_only", + "training_steps": 4, + "batch_size": 1, + "learning_rate": 1e-4, + "device": "cpu", + }, + "conversion": { + "audio_mode": "vocal_only", + "timbre_strength": 0.5, + "device": "cpu", + }, + "target": {"instance_data_dir": str(generated_dir)}, + } + ], + } + ], + ) + + generated = config[1] + self.assertEqual(generated["id"], "voice-transfer") + self.assertTrue((generated_dir / "source.wav").exists()) + self.assertEqual((generated_dir / "source.txt").read_text(encoding="utf-8"), "rock vocal test") + model_path = Path(self.args.output_dir) / "cache" / "data_transforms" / "voice-transfer" / "rvc_model" / "model.pth" + self.assertTrue(model_path.exists()) + + def test_training_requires_identity_data_dir(self): + transform = self._transform({"task": "identity_transfer"}) + normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + + with self.assertRaises(ValueError) as context: + transform._resolve_voice_model( + normalised, + transform._fingerprint(normalised), + Path(normalised["model"]["cache_dir"]), + RVCTransformLogger(str(Path(self.temp_dir) / "output")), + ) + + self.assertIn("identity_data_dir is required", str(context.exception)) + def test_logger_writes_local_json_files(self): run_logger = RVCTransformLogger(str(Path(self.temp_dir) / "output")) diff --git a/tests/test_rvc_smoke.py b/tests/test_rvc_smoke.py new file mode 100644 index 000000000..217dbcce5 --- /dev/null +++ b/tests/test_rvc_smoke.py @@ -0,0 +1,195 @@ +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import torch +from safetensors.torch import save_file as save_safetensors_file + +from simpletuner.helpers.data_transforms import identity_transfer # noqa: F401 +from simpletuner.helpers.data_transforms.base import process_data_transforms +from simpletuner.helpers.rvc.simple import RVCRecord, SimpleRVCConverter, SimpleRVCTrainer, _load_model_payload + + +class RVCIndexSmokeTests(unittest.TestCase): + def _record(self, frames: int, dims: int = 768) -> RVCRecord: + return RVCRecord( + phone=torch.randn(frames, dims), + pitch=torch.full((frames,), 128, dtype=torch.long), + pitchf=torch.full((frames,), 220.0, dtype=torch.float32), + spec=torch.randn(1025, frames), + wave=torch.randn(1, frames * 320), + ) + + def test_build_index_writes_flat_index_and_feature_cache(self): + with tempfile.TemporaryDirectory() as temp_dir: + cache_dir = Path(temp_dir) + records = [self._record(4), self._record(3)] + + index_path = SimpleRVCTrainer()._build_index(records, cache_dir, {}) + + self.assertIsNotNone(index_path) + self.assertTrue(index_path.exists()) + self.assertTrue((cache_dir / "features.safetensors").exists()) + + import faiss + + index = faiss.read_index(str(index_path)) + self.assertEqual(index.ntotal, 7) + self.assertEqual(index.d, 768) + + def test_rvc_model_loader_supports_safetensors_and_legacy_pth(self): + state_dict = {"emb_g.weight": torch.randn(1, 2)} + metadata = { + "kind": "simpletuner-rvc-v2-f0", + "version": "v2", + "f0": "true", + "sample_rate": "48000", + "config_json": "{}", + "training_json": '{"steps": 1.0}', + } + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + safetensors_path = root / "model.safetensors" + pth_path = root / "model.pth" + save_safetensors_file(state_dict, str(safetensors_path), metadata=metadata) + torch.save( + { + "kind": "simpletuner-rvc-v2-f0", + "version": "v2", + "f0": True, + "sample_rate": 48000, + "config": {}, + "generator_state_dict": state_dict, + "training": {"steps": 1.0}, + }, + pth_path, + ) + + safetensors_payload = _load_model_payload(safetensors_path) + pth_payload = _load_model_payload(pth_path) + + self.assertEqual(safetensors_payload["kind"], "simpletuner-rvc-v2-f0") + self.assertEqual(pth_payload["kind"], "simpletuner-rvc-v2-f0") + self.assertIn("emb_g.weight", safetensors_payload["generator_state_dict"]) + self.assertIn("emb_g.weight", pth_payload["generator_state_dict"]) + + def test_torch_retrieve_uses_nearest_feature_vectors(self): + converter = SimpleRVCConverter() + index_vectors = np.array( + [ + [0.0, 0.0], + [1.0, 0.0], + [0.0, 1.0], + ], + dtype=np.float32, + ) + query = np.array([[0.95, 0.05]], dtype=np.float32) + + retrieved = converter._torch_retrieve(query, index_vectors, torch.device("cpu")) + + self.assertEqual(retrieved.shape, query.shape) + self.assertGreater(retrieved[0, 0], retrieved[0, 1]) + + +@unittest.skipUnless( + os.environ.get("SIMPLETUNER_RUN_RVC_SMOKE") == "1", + "set SIMPLETUNER_RUN_RVC_SMOKE=1 to run the real RVC training/conversion smoke test", +) +class RVCIdentityTransferEndToEndSmokeTest(unittest.TestCase): + def test_identity_transfer_trains_rvc_and_writes_audio(self): + source_env = os.environ.get("SIMPLETUNER_RVC_SMOKE_SOURCE") + identity_env = os.environ.get("SIMPLETUNER_RVC_SMOKE_IDENTITY") + if not source_env or not identity_env: + self.skipTest("SIMPLETUNER_RVC_SMOKE_SOURCE and SIMPLETUNER_RVC_SMOKE_IDENTITY are required") + + source_dir = Path(source_env).expanduser() + identity_dir = Path(identity_env).expanduser() + self.assertTrue(source_dir.exists()) + self.assertTrue(identity_dir.exists()) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_root = Path(temp_dir) + generated_dir = temp_root / "generated" + output_dir = temp_root / "output" + config = [ + { + "id": "rvc-smoke-source", + "type": "local", + "dataset_type": "audio", + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "instance_data_dir": str(source_dir), + "audio": {"sample_rate": 48000, "channels": 2, "audio_only": True}, + "data_transforms": [ + { + "id": "rvc-smoke-generated", + "task": "identity_transfer", + "method": "rvc", + "model": { + "identity_data_dir": str(identity_dir), + "cache_dir": str(output_dir / "cache" / "rvc_model"), + "train_if_missing": True, + "force_retrain": True, + "sample_rate": 48000, + "identity_audio_mode": os.environ.get("SIMPLETUNER_RVC_SMOKE_IDENTITY_MODE", "separate"), + "training_steps": int(os.environ.get("SIMPLETUNER_RVC_SMOKE_STEPS", "1")), + "batch_size": 1, + "learning_rate": 1e-4, + "max_seconds_per_file": float(os.environ.get("SIMPLETUNER_RVC_SMOKE_MAX_SECONDS", "5.0")), + "build_index": True, + "flat_index_threshold": 1000000, + "device": os.environ.get("SIMPLETUNER_RVC_SMOKE_DEVICE", "cpu"), + "demucs_device": os.environ.get("SIMPLETUNER_RVC_SMOKE_DEMUCS_DEVICE", "cpu"), + }, + "conversion": { + "audio_mode": os.environ.get("SIMPLETUNER_RVC_SMOKE_AUDIO_MODE", "separate_convert_remix"), + "separation_method": "demucs", + "demucs_device": os.environ.get("SIMPLETUNER_RVC_SMOKE_DEMUCS_DEVICE", "cpu"), + "timbre_strength": 1.0, + "retrieval_strength": 0.75, + "torch_retrieval": True, + "device": os.environ.get("SIMPLETUNER_RVC_SMOKE_DEVICE", "cpu"), + }, + "target": { + "id": "rvc-smoke-generated", + "type": "local", + "dataset_type": "audio", + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "instance_data_dir": str(generated_dir), + "audio": {"sample_rate": 48000, "channels": 2, "audio_only": True}, + }, + } + ], + } + ] + + result = process_data_transforms(SimpleNamespace(output_dir=str(output_dir)), config) + generated_audio = sorted(generated_dir.glob("*.wav")) + + self.assertEqual(len(result), 2) + self.assertGreaterEqual(len(generated_audio), 1) + self.assertTrue((generated_dir / ".simpletuner_identity_transfer.json").exists()) + self.assertTrue((output_dir / "cache" / "rvc_model" / "model.safetensors").exists()) + self.assertTrue((output_dir / "cache" / "rvc_model" / "manifest.json").exists()) + self.assertTrue((output_dir / "cache" / "rvc_model" / "index.index").exists()) + self.assertTrue((output_dir / "cache" / "rvc_model" / "features.safetensors").exists()) + + for path in generated_audio: + self.assertGreater(path.stat().st_size, 1024) + for suffix in (".txt", ".lyrics"): + source_sidecar = source_dir / path.with_suffix(suffix).name + if source_sidecar.exists(): + self.assertTrue(path.with_suffix(suffix).exists()) + + if os.environ.get("SIMPLETUNER_RVC_SMOKE_KEEP_OUTPUT"): + keep_dir = Path(os.environ["SIMPLETUNER_RVC_SMOKE_KEEP_OUTPUT"]).expanduser() + shutil.copytree(temp_root, keep_dir, dirs_exist_ok=True) + + +if __name__ == "__main__": + unittest.main() From 2998f34a0f31f9db6e6c2d50e5accde93debd13b Mon Sep 17 00:00:00 2001 From: bghira Date: Wed, 26 Aug 2026 16:09:13 -0600 Subject: [PATCH 05/12] Add RVC identity transfer utility script --- scripts/run_rvc_model.py | 218 +++++++++++++++++++++++++++++ tests/test_run_rvc_model_script.py | 72 ++++++++++ 2 files changed, 290 insertions(+) create mode 100755 scripts/run_rvc_model.py create mode 100644 tests/test_run_rvc_model_script.py diff --git a/scripts/run_rvc_model.py b/scripts/run_rvc_model.py new file mode 100755 index 000000000..e739bd621 --- /dev/null +++ b/scripts/run_rvc_model.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Train or reuse a SimpleTuner RVC identity model and convert an audio directory.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from simpletuner.helpers.data_transforms import process_data_transforms + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run SimpleTuner's RVC identity-transfer transform outside a training dataloader.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--source-dir", type=Path, required=True, help="Directory of audio files to convert.") + parser.add_argument( + "--identity-dir", + type=Path, + default=None, + help="Directory of target identity audio used to train an RVC artifact when no compatible artifact exists.", + ) + parser.add_argument("--generated-dir", type=Path, required=True, help="Directory where converted audio is written.") + parser.add_argument( + "--output-dir", + type=Path, + required=True, + help="SimpleTuner output directory used for RVC logs and default cache paths.", + ) + parser.add_argument("--transform-id", default="rvc-identity-transfer", help="Generated backend/transform id.") + parser.add_argument("--source-id", default="rvc-source", help="Synthetic source backend id.") + parser.add_argument( + "--model-cache-dir", type=Path, default=None, help="Directory containing or receiving the RVC artifact." + ) + parser.add_argument("--model-name", default=None, help="Human-readable name stored in the RVC artifact metadata.") + + parser.add_argument("--sample-rate", type=int, default=48000, help="RVC sample rate. Only 48000 is supported today.") + parser.add_argument("--channels", type=int, default=2, help="Generated dataset channel count metadata.") + parser.add_argument( + "--asset-hub-model-id", default="lj1995/VoiceConversionWebUI", help="HF repo with default RVC assets." + ) + parser.add_argument("--asset-hub-token", default=None, help="Optional token for the RVC asset repo.") + parser.add_argument( + "--hub-model-id", default=None, help="Optional HF model repo for a reusable SimpleTuner RVC artifact." + ) + parser.add_argument("--hub-token", default=None, help="Optional token for artifact hub reuse or upload.") + parser.add_argument("--reuse-from-hub", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--push-to-hub", action=argparse.BooleanOptionalAction, default=False) + parser.add_argument( + "--public", + action=argparse.BooleanOptionalAction, + default=False, + help="Publish the reusable RVC artifact as a public Hub repo.", + ) + parser.add_argument("--train-if-missing", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--force-retrain", action=argparse.BooleanOptionalAction, default=False) + parser.add_argument("--build-index", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument( + "--flat-index-threshold", type=int, default=100000, help="Use a flat FAISS index below this frame count." + ) + + parser.add_argument( + "--identity-audio-mode", + choices=("separate", "vocal_only"), + default="separate", + help="How identity clips are prepared before RVC feature extraction.", + ) + parser.add_argument("--training-steps", type=int, default=1000, help="RVC generator/discriminator training steps.") + parser.add_argument("--batch-size", type=int, default=4, help="RVC training batch size.") + parser.add_argument("--learning-rate", type=float, default=1e-4, help="RVC AdamW learning rate.") + parser.add_argument("--max-seconds-per-file", type=float, default=180.0, help="Maximum identity audio seconds per file.") + parser.add_argument("--pretrained-generator-path", type=Path, default=None) + parser.add_argument("--pretrained-discriminator-path", type=Path, default=None) + parser.add_argument("--rmvpe-model-path", type=Path, default=None) + parser.add_argument("--hubert-model-path", type=Path, default=None) + + parser.add_argument( + "--audio-mode", + choices=("separate_convert_remix", "vocal_only", "full_mix_convert"), + default="separate_convert_remix", + help="How source audio is converted.", + ) + parser.add_argument("--separation-method", choices=("demucs",), default="demucs") + parser.add_argument("--demucs-model", default="htdemucs", help="Demucs model used for two-stem vocal separation.") + parser.add_argument("--device", default=None, help="RVC train/convert device, e.g. cpu, cuda, mps.") + parser.add_argument("--demucs-device", default=None, help="Demucs device, e.g. cpu, cuda, mps.") + parser.add_argument("--retrieval-strength", type=float, default=0.75, help="RVC retrieval/index blend strength.") + parser.add_argument( + "--timbre-strength", type=float, default=1.0, help="Blend between source waveform and converted waveform." + ) + parser.add_argument("--torch-retrieval", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--is-half", action=argparse.BooleanOptionalAction, default=None) + + parser.add_argument("--dry-run", action="store_true", help="Print the generated transform config without running it.") + parser.add_argument("--print-config", action="store_true", help="Print the generated transform config before running.") + return parser + + +def _set_if_present(config: dict[str, Any], key: str, value: Any) -> None: + if value is not None: + config[key] = str(value) if isinstance(value, Path) else value + + +def build_data_backend_config(args: argparse.Namespace) -> list[dict[str, Any]]: + model_cache_dir = args.model_cache_dir or args.output_dir / "cache" / "data_transforms" / args.transform_id / "rvc_model" + model: dict[str, Any] = { + "cache_dir": str(model_cache_dir), + "train_if_missing": args.train_if_missing, + "force_retrain": args.force_retrain, + "build_index": args.build_index, + "reuse_from_hub": args.reuse_from_hub, + "push_to_hub": args.push_to_hub, + "public": args.public, + "asset_hub_model_id": args.asset_hub_model_id, + "model_name": args.model_name + or (args.hub_model_id.rstrip("/").rsplit("/", 1)[-1] if args.hub_model_id else args.transform_id), + "sample_rate": args.sample_rate, + "identity_audio_mode": args.identity_audio_mode, + "training_steps": args.training_steps, + "batch_size": args.batch_size, + "learning_rate": args.learning_rate, + "max_seconds_per_file": args.max_seconds_per_file, + "flat_index_threshold": args.flat_index_threshold, + } + _set_if_present(model, "identity_data_dir", args.identity_dir) + _set_if_present(model, "asset_hub_token", args.asset_hub_token) + _set_if_present(model, "hub_model_id", args.hub_model_id) + _set_if_present(model, "hub_token", args.hub_token) + _set_if_present(model, "device", args.device) + _set_if_present(model, "demucs_device", args.demucs_device) + _set_if_present(model, "demucs_model", args.demucs_model) + _set_if_present(model, "pretrained_generator_path", args.pretrained_generator_path) + _set_if_present(model, "pretrained_discriminator_path", args.pretrained_discriminator_path) + _set_if_present(model, "rmvpe_model_path", args.rmvpe_model_path) + _set_if_present(model, "hubert_model_path", args.hubert_model_path) + _set_if_present(model, "is_half", args.is_half) + + conversion: dict[str, Any] = { + "audio_mode": args.audio_mode, + "separation_method": args.separation_method, + "demucs_model": args.demucs_model, + "retrieval_strength": args.retrieval_strength, + "timbre_strength": args.timbre_strength, + "torch_retrieval": args.torch_retrieval, + } + _set_if_present(conversion, "device", args.device) + _set_if_present(conversion, "demucs_device", args.demucs_device) + _set_if_present(conversion, "is_half", args.is_half) + + return [ + { + "id": args.source_id, + "type": "local", + "dataset_type": "audio", + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "instance_data_dir": str(args.source_dir), + "audio": {"sample_rate": args.sample_rate, "channels": args.channels, "audio_only": True}, + "data_transforms": [ + { + "id": args.transform_id, + "task": "identity_transfer", + "method": "rvc", + "model": model, + "conversion": conversion, + "target": { + "id": args.transform_id, + "type": "local", + "dataset_type": "audio", + "metadata_backend": "discovery", + "caption_strategy": "textfile", + "instance_data_dir": str(args.generated_dir), + "audio": {"sample_rate": args.sample_rate, "channels": args.channels, "audio_only": True}, + }, + } + ], + } + ] + + +def validate_args(args: argparse.Namespace) -> None: + if not args.source_dir.exists(): + raise FileNotFoundError(f"--source-dir does not exist: {args.source_dir}") + if not args.source_dir.is_dir(): + raise NotADirectoryError(f"--source-dir is not a directory: {args.source_dir}") + if args.sample_rate != 48000: + raise ValueError("SimpleTuner RVC currently supports --sample-rate 48000 only.") + if args.train_if_missing and args.identity_dir is None: + raise ValueError("--identity-dir is required when --train-if-missing is enabled.") + if args.identity_dir is not None and not args.identity_dir.exists(): + raise FileNotFoundError(f"--identity-dir does not exist: {args.identity_dir}") + + +def run(args: argparse.Namespace) -> list[dict[str, Any]]: + validate_args(args) + config = build_data_backend_config(args) + if args.print_config or args.dry_run: + print(json.dumps(config, indent=2, sort_keys=True)) + if args.dry_run: + return config + result = process_data_transforms(SimpleNamespace(output_dir=str(args.output_dir)), config) + generated = sorted(args.generated_dir.rglob("*.wav")) + print(json.dumps({"backend_count": len(result), "generated_dir": str(args.generated_dir)}, indent=2)) + for path in generated: + print(path.relative_to(args.generated_dir)) + return result + + +def main() -> None: + run(build_parser().parse_args()) + + +if __name__ == "__main__": + main() diff --git a/tests/test_run_rvc_model_script.py b/tests/test_run_rvc_model_script.py new file mode 100644 index 000000000..a30514cc8 --- /dev/null +++ b/tests/test_run_rvc_model_script.py @@ -0,0 +1,72 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +def _load_script_module(): + script_path = Path(__file__).resolve().parents[1] / "scripts" / "run_rvc_model.py" + spec = importlib.util.spec_from_file_location("run_rvc_model", script_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class TestRunRVCModelScript(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.script = _load_script_module() + + def test_dry_run_builds_identity_transfer_backend_config(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + source_dir = root / "source" + identity_dir = root / "identity" + generated_dir = root / "generated" + output_dir = root / "output" + source_dir.mkdir() + identity_dir.mkdir() + + args = self.script.build_parser().parse_args( + [ + "--source-dir", + str(source_dir), + "--identity-dir", + str(identity_dir), + "--generated-dir", + str(generated_dir), + "--output-dir", + str(output_dir), + "--training-steps", + "123", + "--batch-size", + "1", + "--model-name", + "Test Voice", + "--public", + "--device", + "cpu", + "--dry-run", + ] + ) + + config = self.script.run(args) + + source_backend = config[0] + transform = source_backend["data_transforms"][0] + self.assertEqual(source_backend["dataset_type"], "audio") + self.assertEqual(transform["task"], "identity_transfer") + self.assertEqual(transform["method"], "rvc") + self.assertEqual(transform["model"]["identity_audio_mode"], "separate") + self.assertEqual(transform["model"]["model_name"], "Test Voice") + self.assertTrue(transform["model"]["public"]) + self.assertEqual(transform["conversion"]["audio_mode"], "separate_convert_remix") + self.assertEqual(transform["model"]["training_steps"], 123) + self.assertEqual(transform["model"]["batch_size"], 1) + self.assertEqual(transform["model"]["device"], "cpu") + self.assertEqual(transform["target"]["instance_data_dir"], str(generated_dir)) + + +if __name__ == "__main__": + unittest.main() From c24a9925c113635d6976d11311c5c806e9fd3245 Mon Sep 17 00:00:00 2001 From: bghira Date: Wed, 26 Aug 2026 16:09:48 -0600 Subject: [PATCH 06/12] Improve audio lyrics fetching utility --- scripts/datasets/fetch_lyrics.py | 26 ++++++++++++++++++++++- tests/test_fetch_lyrics.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tests/test_fetch_lyrics.py diff --git a/scripts/datasets/fetch_lyrics.py b/scripts/datasets/fetch_lyrics.py index 49403cc72..3128af4d8 100644 --- a/scripts/datasets/fetch_lyrics.py +++ b/scripts/datasets/fetch_lyrics.py @@ -52,6 +52,7 @@ AUDIO_EXTENSIONS = {".mp3", ".flac", ".wav", ".ogg", ".m4a", ".aiff", ".opus"} TITLE_VERSION_JUNK = re.compile(r"\s*\((?:[^)]*version|live[^)]*)\)\s*$", re.IGNORECASE) +NUMBERED_ARTIST_TITLE = re.compile(r"^\s*\d+\s*-\s*(?P.+?)\s*-\s*(?P.+?)\s*$") def normalize_title(title): @@ -63,6 +64,28 @@ def normalize_title(title): return title.strip() or None +def metadata_from_numbered_filename(filepath): + """Parse filenames like '001 - Artist - Title.mp3' when embedded tags are generic.""" + match = NUMBERED_ARTIST_TITLE.match(Path(filepath).stem) + if not match: + return None, None + artist = match.group("artist").strip() + title = normalize_title(match.group("title").strip()) + return artist or None, title + + +def prefer_filename_metadata(filepath, artist, title): + """Use filename metadata when tags contain compilation placeholders.""" + filename_artist, filename_title = metadata_from_numbered_filename(filepath) + if not filename_artist or not filename_title: + return artist, title + generic_artist = not artist or artist.strip().lower() in {"various artists", "various", "unknown artist"} + numbered_title = bool(title and NUMBERED_ARTIST_TITLE.match(title.strip())) + if generic_artist or numbered_title: + return filename_artist, filename_title + return artist, title + + def scrape_genius_tokenless(artist, title): """ Scrape Genius without an API token. @@ -238,7 +261,7 @@ def main(): genius.verbose = False # Quieter output genius.remove_section_headers = False # Keep [Verse], [Chorus] structure - ACE-Step likes this! elif lyricsgenius: - print("Notice: No Genius Token provided. Only local ID3 tags will be checked.") + print("Notice: No Genius token provided. Local tags and tokenless Genius scraping will be checked.") files = [p for p in root_dir.rglob("*") if p.suffix.lower() in AUDIO_EXTENSIONS] logger.info(f"Found {len(files)} audio files.") @@ -251,6 +274,7 @@ def main(): continue artist, title, local_lyrics = extract_metadata(filepath) + artist, title = prefer_filename_metadata(filepath, artist, title) if title: title = normalize_title(title) diff --git a/tests/test_fetch_lyrics.py b/tests/test_fetch_lyrics.py new file mode 100644 index 000000000..c0768cb06 --- /dev/null +++ b/tests/test_fetch_lyrics.py @@ -0,0 +1,36 @@ +import unittest +from pathlib import Path + +from scripts.datasets.fetch_lyrics import metadata_from_numbered_filename, prefer_filename_metadata + + +class TestFetchLyricsMetadata(unittest.TestCase): + def test_metadata_from_numbered_filename(self): + artist, title = metadata_from_numbered_filename(Path("001 - Led Zeppelin - Stairway To Heaven.mp3")) + + self.assertEqual("Led Zeppelin", artist) + self.assertEqual("Stairway To Heaven", title) + + def test_filename_metadata_replaces_compilation_tags(self): + artist, title = prefer_filename_metadata( + Path("001 - Led Zeppelin - Stairway To Heaven.mp3"), + "Various Artists", + "001 - Led Zeppelin - Stairway To Heaven", + ) + + self.assertEqual("Led Zeppelin", artist) + self.assertEqual("Stairway To Heaven", title) + + def test_filename_metadata_does_not_replace_specific_tags(self): + artist, title = prefer_filename_metadata( + Path("001 - Led Zeppelin - Stairway To Heaven.mp3"), + "Led Zeppelin", + "Stairway To Heaven", + ) + + self.assertEqual("Led Zeppelin", artist) + self.assertEqual("Stairway To Heaven", title) + + +if __name__ == "__main__": + unittest.main() From 22513c43ee9e75417f625278381e0afc21e0dcbf Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Wed, 26 Aug 2026 16:09:59 -0600 Subject: [PATCH 07/12] Expose audio identity transfer in WebUI --- .../server/data/dataset_blueprints.py | 2 +- .../static/js/dataloader-section-component.js | 28 ++++- .../dataloader/sections/audio_body.html | 100 ++++++++++++++++++ simpletuner/templates/trainer_htmx.html | 45 +++++++- .../js/dataloader_audio_capabilities.test.js | 13 ++- 5 files changed, 180 insertions(+), 8 deletions(-) diff --git a/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py b/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py index 9b7656f0f..0a62bc028 100644 --- a/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py +++ b/simpletuner/simpletuner_sdk/server/data/dataset_blueprints.py @@ -597,7 +597,7 @@ class BackendBlueprint(BaseModel): "label": "data transforms", "description": "optional generated audio split configuration, such as identity_transfer", "type": "textarea", - "placeholder": '[{"task":"identity_transfer","id":"voice-transfer","model":{"train_if_missing":true}}]', + "placeholder": '[{"task":"identity_transfer","id":"voice-transfer","model":{"train_if_missing":true,"model_name":"Target voice RVC","hub_model_id":"org/target-voice-rvc","push_to_hub":false,"public":false}}]', "advanced": True, }, { diff --git a/simpletuner/static/js/dataloader-section-component.js b/simpletuner/static/js/dataloader-section-component.js index 53c2039ee..d5c6401c8 100644 --- a/simpletuner/static/js/dataloader-section-component.js +++ b/simpletuner/static/js/dataloader-section-component.js @@ -629,11 +629,21 @@ function dataloaderSectionComponent() { build_index: true, reuse_from_hub: true, push_to_hub: false, - force_retrain: false + public: false, + force_retrain: false, + asset_hub_model_id: 'lj1995/VoiceConversionWebUI', + model_name: `${dataset.id || 'audio'} RVC voice`, + sample_rate: 48000, + identity_audio_mode: 'separate', + training_steps: 1000, + batch_size: 4, + learning_rate: 0.0001 }, conversion: { - audio_mode: 'vocal_only', - separation_method: 'demucs' + audio_mode: 'separate_convert_remix', + separation_method: 'demucs', + timbre_strength: 1.0, + retrieval_strength: 0.75 }, target: {} }; @@ -647,9 +657,19 @@ function dataloaderSectionComponent() { if (transform.model.build_index === undefined) transform.model.build_index = true; if (transform.model.reuse_from_hub === undefined) transform.model.reuse_from_hub = true; if (transform.model.push_to_hub === undefined) transform.model.push_to_hub = false; + if (transform.model.public === undefined) transform.model.public = false; if (transform.model.force_retrain === undefined) transform.model.force_retrain = false; - transform.conversion.audio_mode = transform.conversion.audio_mode || 'vocal_only'; + transform.model.asset_hub_model_id = transform.model.asset_hub_model_id || 'lj1995/VoiceConversionWebUI'; + transform.model.model_name = transform.model.model_name || `${dataset.id || 'audio'} RVC voice`; + if (transform.model.sample_rate === undefined) transform.model.sample_rate = 48000; + transform.model.identity_audio_mode = transform.model.identity_audio_mode || 'separate'; + if (transform.model.training_steps === undefined) transform.model.training_steps = 1000; + if (transform.model.batch_size === undefined) transform.model.batch_size = 4; + if (transform.model.learning_rate === undefined) transform.model.learning_rate = 0.0001; + transform.conversion.audio_mode = transform.conversion.audio_mode || 'separate_convert_remix'; transform.conversion.separation_method = transform.conversion.separation_method || 'demucs'; + if (transform.conversion.timbre_strength === undefined) transform.conversion.timbre_strength = 1.0; + if (transform.conversion.retrieval_strength === undefined) transform.conversion.retrieval_strength = 0.75; return transform; }, identityTransferEnabled(dataset) { diff --git a/simpletuner/templates/components/dataloader/sections/audio_body.html b/simpletuner/templates/components/dataloader/sections/audio_body.html index c55935e94..0d6e2b32c 100644 --- a/simpletuner/templates/components/dataloader/sections/audio_body.html +++ b/simpletuner/templates/components/dataloader/sections/audio_body.html @@ -219,6 +219,22 @@ <h6 class="modal-section-title">Voice Identity Transfer</h6> @input="markAsUnsaved()" placeholder="org/voice-artifact"> </div> + <div class="col-md-4"> + <label class="form-label small">Model Name</label> + <input type="text" + class="form-control form-control-sm" + x-model.lazy="identityTransferTransform(editingDataset).model.model_name" + @input="markAsUnsaved()" + placeholder="Target voice RVC"> + </div> + <div class="col-md-4"> + <label class="form-label small">RVC Asset Repo</label> + <input type="text" + class="form-control form-control-sm" + x-model.lazy="identityTransferTransform(editingDataset).model.asset_hub_model_id" + @input="markAsUnsaved()" + placeholder="lj1995/VoiceConversionWebUI"> + </div> <div class="col-md-4"> <label class="form-label small">Model Cache Dir</label> <input type="text" @@ -227,6 +243,60 @@ <h6 class="modal-section-title">Voice Identity Transfer</h6> @input="markAsUnsaved()" placeholder="output/cache/data_transforms/.../rvc_model"> </div> + <div class="col-md-4"> + <label class="form-label small">Identity Data Dir</label> + <input type="text" + class="form-control form-control-sm" + x-model.lazy="identityTransferTransform(editingDataset).model.identity_data_dir" + @input="markAsUnsaved()" + placeholder="/path/to/target-vocal-stems"> + </div> + <div class="col-md-4"> + <label class="form-label small">Sample Rate</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).model.sample_rate" + @input="markAsUnsaved()" + min="48000" + max="48000" + step="1"> + </div> + <div class="col-md-4"> + <label class="form-label small">Identity Audio</label> + <select class="form-select form-select-sm" + x-model="identityTransferTransform(editingDataset).model.identity_audio_mode" + @change="markAsUnsaved()"> + <option value="separate">Separate vocals</option> + <option value="vocal_only">Already vocal stems</option> + </select> + </div> + <div class="col-md-4"> + <label class="form-label small">Training Steps</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).model.training_steps" + @input="markAsUnsaved()" + min="1" + step="100"> + </div> + <div class="col-md-4"> + <label class="form-label small">RVC Batch Size</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).model.batch_size" + @input="markAsUnsaved()" + min="1" + step="1"> + </div> + <div class="col-md-4"> + <label class="form-label small">RVC Learning Rate</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).model.learning_rate" + @input="markAsUnsaved()" + min="0" + step="0.00001"> + </div> <div class="col-md-4"> <label class="form-label small">Audio Mode</label> @@ -255,6 +325,26 @@ <h6 class="modal-section-title">Voice Identity Transfer</h6> @input="markAsUnsaved()" placeholder="output/cache/data_transforms/.../generated_audio"> </div> + <div class="col-md-4"> + <label class="form-label small">Transfer Strength</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).conversion.timbre_strength" + @input="markAsUnsaved()" + min="0" + max="1" + step="0.05"> + </div> + <div class="col-md-4"> + <label class="form-label small">Retrieval Strength</label> + <input type="number" + class="form-control form-control-sm" + x-model.number="identityTransferTransform(editingDataset).conversion.retrieval_strength" + @input="markAsUnsaved()" + min="0" + max="1" + step="0.05"> + </div> <div class="col-md-4"> <div class="form-check form-switch mt-4"> @@ -306,6 +396,16 @@ <h6 class="modal-section-title">Voice Identity Transfer</h6> <label class="form-check-label small" :for="'identity-push-hub-' + editingDataset.id">Push to Hub</label> </div> </div> + <div class="col-md-4"> + <div class="form-check form-switch mt-2"> + <input type="checkbox" + class="form-check-input" + :id="'identity-public-hub-' + editingDataset.id" + x-model="identityTransferTransform(editingDataset).model.public" + @change="markAsUnsaved()"> + <label class="form-check-label small" :for="'identity-public-hub-' + editingDataset.id">Public Hub repo</label> + </div> + </div> </div> </template> </div> diff --git a/simpletuner/templates/trainer_htmx.html b/simpletuner/templates/trainer_htmx.html index db17c36c1..8d01e1c25 100644 --- a/simpletuner/templates/trainer_htmx.html +++ b/simpletuner/templates/trainer_htmx.html @@ -2291,19 +2291,60 @@ normalized.model.build_index = model.build_index === undefined ? true : this.normalizeBoolean(model.build_index); normalized.model.reuse_from_hub = model.reuse_from_hub === undefined ? true : this.normalizeBoolean(model.reuse_from_hub); normalized.model.push_to_hub = this.normalizeBoolean(model.push_to_hub); + normalized.model.public = this.normalizeBoolean(model.public); const hubModelId = this._coerceString(model.hub_model_id); if (hubModelId) normalized.model.hub_model_id = hubModelId; + const modelName = this._coerceString(model.model_name); + if (modelName) normalized.model.model_name = modelName; const cacheDir = this._coerceString(model.cache_dir); if (cacheDir) normalized.model.cache_dir = cacheDir; + const identityDataDir = this._coerceString(model.identity_data_dir); + if (identityDataDir) normalized.model.identity_data_dir = identityDataDir; + const assetHubModelId = this._coerceString(model.asset_hub_model_id); + if (assetHubModelId) normalized.model.asset_hub_model_id = assetHubModelId; + const pretrainedGeneratorPath = this._coerceString(model.pretrained_generator_path); + if (pretrainedGeneratorPath) normalized.model.pretrained_generator_path = pretrainedGeneratorPath; + const pretrainedDiscriminatorPath = this._coerceString(model.pretrained_discriminator_path); + if (pretrainedDiscriminatorPath) normalized.model.pretrained_discriminator_path = pretrainedDiscriminatorPath; + const rmvpeModelPath = this._coerceString(model.rmvpe_model_path); + if (rmvpeModelPath) normalized.model.rmvpe_model_path = rmvpeModelPath; + const hubertModelPath = this._coerceString(model.hubert_model_path); + if (hubertModelPath) normalized.model.hubert_model_path = hubertModelPath; + const sampleRate = parseInt(model.sample_rate, 10); + if (Number.isFinite(sampleRate) && sampleRate > 0) normalized.model.sample_rate = sampleRate; + const identityAudioMode = this._coerceString(model.identity_audio_mode) || 'separate'; + if (['separate', 'vocal_only'].includes(identityAudioMode)) { + normalized.model.identity_audio_mode = identityAudioMode; + } + const trainingSteps = parseInt(model.training_steps, 10); + if (Number.isFinite(trainingSteps) && trainingSteps > 0) { + normalized.model.training_steps = trainingSteps; + } + const batchSize = parseInt(model.batch_size, 10); + if (Number.isFinite(batchSize) && batchSize > 0) { + normalized.model.batch_size = batchSize; + } + const learningRate = parseFloat(model.learning_rate); + if (Number.isFinite(learningRate) && learningRate > 0) { + normalized.model.learning_rate = learningRate; + } - const audioMode = this._coerceString(conversion.audio_mode) || 'vocal_only'; + const audioMode = this._coerceString(conversion.audio_mode) || 'separate_convert_remix'; if (['vocal_only', 'separate_convert_remix', 'full_mix_convert'].includes(audioMode)) { normalized.conversion.audio_mode = audioMode; } else { - normalized.conversion.audio_mode = 'vocal_only'; + normalized.conversion.audio_mode = 'separate_convert_remix'; } normalized.conversion.separation_method = this._coerceString(conversion.separation_method) || 'demucs'; + const timbreStrength = parseFloat(conversion.timbre_strength); + if (Number.isFinite(timbreStrength)) { + normalized.conversion.timbre_strength = Math.max(0, Math.min(1, timbreStrength)); + } + const retrievalStrength = parseFloat(conversion.retrieval_strength); + if (Number.isFinite(retrievalStrength)) { + normalized.conversion.retrieval_strength = Math.max(0, Math.min(1, retrievalStrength)); + } const targetId = this._coerceString(target.id); if (targetId) normalized.target.id = targetId; diff --git a/tests/js/dataloader_audio_capabilities.test.js b/tests/js/dataloader_audio_capabilities.test.js index 0a53b36fe..b6e2dc6da 100644 --- a/tests/js/dataloader_audio_capabilities.test.js +++ b/tests/js/dataloader_audio_capabilities.test.js @@ -246,7 +246,18 @@ describe('Dataloader Section Audio Capabilities', () => { expect(dataset.data_transforms[0].method).toBe('rvc'); expect(dataset.data_transforms[0].model.train_if_missing).toBe(true); expect(dataset.data_transforms[0].model.build_index).toBe(true); - expect(dataset.data_transforms[0].conversion.audio_mode).toBe('vocal_only'); + expect(dataset.data_transforms[0].model.push_to_hub).toBe(false); + expect(dataset.data_transforms[0].model.public).toBe(false); + expect(dataset.data_transforms[0].model.asset_hub_model_id).toBe('lj1995/VoiceConversionWebUI'); + expect(dataset.data_transforms[0].model.model_name).toBe('voice-source RVC voice'); + expect(dataset.data_transforms[0].model.sample_rate).toBe(48000); + expect(dataset.data_transforms[0].model.identity_audio_mode).toBe('separate'); + expect(dataset.data_transforms[0].model.training_steps).toBe(1000); + expect(dataset.data_transforms[0].model.batch_size).toBe(4); + expect(dataset.data_transforms[0].model.learning_rate).toBe(0.0001); + expect(dataset.data_transforms[0].conversion.audio_mode).toBe('separate_convert_remix'); + expect(dataset.data_transforms[0].conversion.timbre_strength).toBe(1.0); + expect(dataset.data_transforms[0].conversion.retrieval_strength).toBe(0.75); expect(component.identityTransferEnabled(dataset)).toBe(true); }); From ac45e23cd69b4c9e5fbd1627829805f3004a7792 Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Wed, 26 Aug 2026 16:10:13 -0600 Subject: [PATCH 08/12] Document voice cloning data transforms --- documentation/DATALOADER.es.md | 3 +- documentation/DATALOADER.hi.md | 3 +- documentation/DATALOADER.ja.md | 3 +- documentation/DATALOADER.md | 3 +- documentation/DATALOADER.pt-BR.md | 3 +- documentation/DATALOADER.zh.md | 3 +- .../experimental/VOICE_CLONING.es.md | 35 +++++++++++--- .../experimental/VOICE_CLONING.hi.md | 37 ++++++++++++--- .../experimental/VOICE_CLONING.ja.md | 37 ++++++++++++--- documentation/experimental/VOICE_CLONING.md | 46 +++++++++++++++---- .../experimental/VOICE_CLONING.pt-BR.md | 35 +++++++++++--- .../experimental/VOICE_CLONING.zh.md | 37 ++++++++++++--- 12 files changed, 197 insertions(+), 48 deletions(-) diff --git a/documentation/DATALOADER.es.md b/documentation/DATALOADER.es.md index a6ae307d7..a775105c0 100644 --- a/documentation/DATALOADER.es.md +++ b/documentation/DATALOADER.es.md @@ -263,7 +263,8 @@ Los backends de memoria requieren Linux o macOS y suficiente RAM o swap para la - **Valores:** un objeto de transform o un arreglo de objetos de transform - **Descripción:** Expande un dataset fuente en uno o más datasets de entrenamiento generados antes de que empiece la configuración normal del dataloader. Los datasets generados se tratan como datasets primarios normales salvo que el transform pida explícitamente clonar metadatos. - **Identity transfer de audio:** `{"task": "identity_transfer", "method": "rvc"}` está disponible para backends con `dataset_type: "audio"`. Prepara un split de audio generado para transferencia de identidad vocal y usa el directorio de salida para artefactos de voz cacheados y archivos generados. Consulta [Voice Cloning Data Transforms](experimental/VOICE_CLONING.es.md). -- **Estado:** La primera implementación aporta el registro de transforms, comprobaciones de manifiestos de caché, reutilización/subida de artefactos a Hub, hooks de sharding de inicio compatibles con DDP y logs locales de RVC. El entrenamiento/conversión RVC nativo sigue siendo experimental y fallará de forma explícita salvo que ya exista un split generado compatible en la caché. +- **Identity data:** Pon la musica a convertir en el `instance_data_dir` del backend de audio, los ejemplos de la voz objetivo en `model.identity_data_dir`, y la ruta del split generado en `target.instance_data_dir`. +- **Estado:** La implementación experimental incluye comprobaciones de manifiestos de caché, reutilización/subida de artefactos a Hub mediante el layout `huggingface-hub-rvc`, sharding de inicio compatible con DDP, logs locales de RVC y un trainer/converter compacto de voice-transfer de SimpleTuner. El modo full-song remix usa Demucs para separar voces; el modo vocal-stem no necesita separación. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.hi.md b/documentation/DATALOADER.hi.md index 137dea0de..0a4aec85a 100644 --- a/documentation/DATALOADER.hi.md +++ b/documentation/DATALOADER.hi.md @@ -263,7 +263,8 @@ Memory backends के लिए Linux या macOS और मौजूदा ca - **Values:** एक transform object या transform objects का array - **Description:** Normal dataloader setup शुरू होने से पहले source dataset को एक या अधिक generated training datasets में expand करता है। Generated datasets को regular primary datasets की तरह treat किया जाता है, जब तक transform explicitly metadata clone न मांगे। - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` `dataset_type: "audio"` backends के लिए उपलब्ध है। यह voice identity transfer के लिए generated audio split तैयार करता है और output directory में cached voice artifacts तथा generated files रखता है। देखें [Voice Cloning Data Transforms](experimental/VOICE_CLONING.hi.md)। -- **Status:** पहली implementation transform registry, cache manifest checks, Hub artifact reuse/push plumbing, DDP-aware startup sharding hooks, और local RVC logs देती है। Native RVC training/conversion अभी experimental है और cache में matching generated split न होने पर साफ error देगा। +- **Identity data:** Convert की जाने वाली music audio backend के `instance_data_dir` में रखें, target voice examples `model.identity_data_dir` में रखें, और generated split path `target.instance_data_dir` में रखें। +- **Status:** Experimental implementation cache manifest checks, Hub artifact reuse/push through the `huggingface-hub-rvc` artifact layout, DDP-aware startup sharding, local RVC logs, और compact SimpleTuner voice-transfer trainer/converter देती है। Full-song remix mode vocal separation के लिए Demucs इस्तेमाल करता है; vocal-stem mode में separation की जरूरत नहीं है। ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.ja.md b/documentation/DATALOADER.ja.md index cf98745cf..097e3cfa0 100644 --- a/documentation/DATALOADER.ja.md +++ b/documentation/DATALOADER.ja.md @@ -263,7 +263,8 @@ Hugging Face の音声データセットでは、キャプション（プロン - **Values:** transform オブジェクト、または transform オブジェクトの配列 - **Description:** 通常の dataloader 設定が始まる前に、ソース dataset から 1 つ以上の生成済み training dataset を展開します。transform が明示的に metadata clone を要求しない限り、生成 dataset は通常の primary dataset として扱われます。 - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` は `dataset_type: "audio"` の backend で利用できます。音声 identity transfer 用の生成 audio split を準備し、output directory に voice artifact と生成ファイルをキャッシュします。詳しくは [Voice Cloning Data Transforms](experimental/VOICE_CLONING.ja.md) を参照してください。 -- **Status:** 最初の実装では transform registry、cache manifest の確認、Hub artifact の reuse/push、DDP 対応の startup sharding hook、ローカル RVC log を提供します。ネイティブ RVC training/conversion はまだ experimental であり、互換性のある生成 split が cache に存在しない場合は明示的に失敗します。 +- **Identity data:** 変換したい音楽は audio backend の `instance_data_dir`、対象 voice examples は `model.identity_data_dir`、生成 split path は `target.instance_data_dir` に置きます。 +- **Status:** 実験的な実装は、cache manifest の確認、`huggingface-hub-rvc` artifact layout による Hub artifact の reuse/push、DDP 対応の startup sharding、ローカル RVC log、そして小さな SimpleTuner voice-transfer trainer/converter を提供します。Full-song remix mode は Demucs で vocal separation を行い、vocal-stem mode では separation は不要です。 ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.md b/documentation/DATALOADER.md index 716b411b1..d5b563528 100644 --- a/documentation/DATALOADER.md +++ b/documentation/DATALOADER.md @@ -265,7 +265,8 @@ Memory backends require Linux or macOS and enough RAM or swap for the existing c - **Values:** a transform object or an array of transform objects - **Description:** Expands a source dataset into one or more generated training datasets before normal dataloader setup begins. Generated datasets are treated as regular primary datasets unless the transform explicitly asks to clone metadata. - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` is available for `dataset_type: "audio"` backends. It prepares a generated audio split for voice identity transfer and uses the output directory for cached voice artifacts and generated files. See [Voice Cloning Data Transforms](experimental/VOICE_CLONING.md). -- **Status:** The first implementation provides the transform registry, cache manifest checks, Hub artifact reuse/push plumbing, DDP-aware startup sharding hooks, and local RVC logs. Native RVC training/conversion is still experimental and will fail loudly unless a matching generated split already exists in the cache. +- **Identity data:** Put the music to be converted in the audio backend's `instance_data_dir`, put the target voice examples in `model.identity_data_dir`, and put the generated split path in `target.instance_data_dir`. +- **Status:** The experimental implementation includes cache manifest checks, Hub artifact reuse/push through the `huggingface-hub-rvc` artifact layout, DDP-aware startup sharding, local RVC logs, and a compact SimpleTuner voice-transfer trainer/converter. Full-song remix mode uses Demucs for vocal separation; vocal-stem mode does not need separation. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.pt-BR.md b/documentation/DATALOADER.pt-BR.md index 1d6d95154..554bb9125 100644 --- a/documentation/DATALOADER.pt-BR.md +++ b/documentation/DATALOADER.pt-BR.md @@ -263,7 +263,8 @@ Backends de memória exigem Linux ou macOS e RAM ou swap suficiente para o cache - **Valores:** um objeto de transform ou um array de objetos de transform - **Descrição:** Expande um dataset fonte em um ou mais datasets de treinamento gerados antes da configuração normal do dataloader. Datasets gerados são tratados como datasets primários normais, a menos que o transform peça explicitamente clonagem de metadados. - **Identity transfer de áudio:** `{"task": "identity_transfer", "method": "rvc"}` está disponível para backends com `dataset_type: "audio"`. Ele prepara um split de áudio gerado para transferência de identidade vocal e usa o diretório de saída para artefatos de voz em cache e arquivos gerados. Consulte [Voice Cloning Data Transforms](experimental/VOICE_CLONING.pt-BR.md). -- **Status:** A primeira implementação fornece o registro de transforms, verificações de manifest de cache, reutilização/envio de artefatos para o Hub, hooks de sharding de inicialização compatíveis com DDP e logs locais de RVC. O treinamento/conversão RVC nativo ainda é experimental e falhará explicitamente salvo se já existir um split gerado compatível no cache. +- **Identity data:** Coloque a musica a converter no `instance_data_dir` do backend de audio, os exemplos da voz alvo em `model.identity_data_dir`, e o caminho do split gerado em `target.instance_data_dir`. +- **Status:** A implementação experimental inclui verificações de manifest de cache, reutilização/envio de artefatos para o Hub pelo layout `huggingface-hub-rvc`, sharding de inicialização compatível com DDP, logs locais de RVC e um trainer/converter compacto de voice-transfer do SimpleTuner. O modo full-song remix usa Demucs para separar vocais; o modo vocal-stem não precisa de separação. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.zh.md b/documentation/DATALOADER.zh.md index fbd83c339..a112cfbc8 100644 --- a/documentation/DATALOADER.zh.md +++ b/documentation/DATALOADER.zh.md @@ -262,7 +262,8 @@ LTX-2 使用原生纯音频分支；MiniMax-H3 在打包序列中为每个 laten - **取值:** 一个 transform 对象，或 transform 对象数组 - **说明:** 在常规 dataloader 设置开始前，把源数据集展开为一个或多个生成的训练数据集。除非 transform 明确要求克隆元数据，生成的数据集会被当作普通主训练数据集处理。 - **音频 identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` 可用于 `dataset_type: "audio"` 后端。它会为声音身份迁移准备生成音频 split，并在输出目录中缓存 voice artifacts 和生成文件。参见 [Voice Cloning Data Transforms](experimental/VOICE_CLONING.zh.md)。 -- **状态:** 第一版实现包含 transform registry、缓存 manifest 检查、Hub artifact 复用/上传流程、DDP 感知的启动 sharding hook，以及本地 RVC 日志。原生 RVC 训练/转换仍为实验性质；除非缓存中已经存在匹配的生成 split，否则会明确失败。 +- **Identity data:** 要转换的音乐放在音频 backend 的 `instance_data_dir`，目标声音示例放在 `model.identity_data_dir`，生成 split 路径放在 `target.instance_data_dir`。 +- **状态:** 实验性实现包含缓存 manifest 检查、通过 `huggingface-hub-rvc` artifact layout 进行 Hub artifact 复用/上传、DDP 感知的启动 sharding、本地 RVC 日志，以及一个紧凑的 SimpleTuner voice-transfer 训练/转换器。Full-song remix 模式使用 Demucs 分离人声；vocal-stem 模式不需要分离。 ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/experimental/VOICE_CLONING.es.md b/documentation/experimental/VOICE_CLONING.es.md index ea0182516..400e23cff 100644 --- a/documentation/experimental/VOICE_CLONING.es.md +++ b/documentation/experimental/VOICE_CLONING.es.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## Transferencia de Identidad Estilo RVC -La primera implementacion prevista es conversion de voz estilo RVC. +La primera implementacion es conversion de voz estilo RVC, usando features HuBERT, pitch RMVPE, generator NSF/VITS, multi-period discriminator, losses mel/adversarial e indice de retrieval opcional. En este contexto, el "modelo RVC" es especifico de la voz. Se entrena desde el dataset de identidad objetivo. El indice de recuperacion tambien es especifico de la voz y se construye desde features de esa misma voz. Componentes preentrenados amplios, como features de contenido, extraccion de pitch o modelos de separacion, son infraestructura reutilizable; el modelo de conversion y el indice son artifacts especificos del cantante o hablante. @@ -93,7 +93,7 @@ SimpleTuner deberia poder: ## Comportamiento Por Defecto -Los defaults planificados son conservadores: +Los defaults son conservadores. En este workflow, el backend de audio es la musica de expansion que se convertira, `model.identity_data_dir` es el dataset de la voz objetivo, y `target.instance_data_dir` es solo la ruta del split generado. | Setting | Default | Por que | | --- | --- | --- | @@ -102,11 +102,22 @@ Los defaults planificados son conservadores: | `train_if_missing` | `true` | SimpleTuner debe poder bootstrapear el modelo vocal desde el dataset objetivo. | | `force_retrain` | `false` | Reutiliza un modelo cacheado valido cuando sea posible. | | `build_index` | `true` | Retrieval suele mejorar estabilidad de identidad y reducir leakage. | +| `identity_data_dir` | obligatorio en entrenamiento bajo demanda | Apunta a ejemplos vocales limpios de la voz que se transferira a las canciones de expansion. | +| `identity_audio_mode` | `separate` | Ejecuta Demucs en los clips de identidad antes del entrenamiento. Usa `vocal_only` si el dataset de identidad ya contiene vocal stems. | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs RVC generator/discriminator fine-tuning during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Standard RVC AdamW default. | | `hub_model_id` | sin definir | No se usa cache remoto de modelo vocal sin opt-in del usuario. | | `reuse_from_hub` | `true` cuando `hub_model_id` esta definido | Revisa el Hub antes de gastar tiempo entrenando un modelo bajo demanda. | | `push_to_hub` | `false` | Subir un modelo vocal debe ser explicito porque el artifact representa una identidad vocal. | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | `separate_convert_remix` para canciones completas, `vocal_only` para vocal stems | Las mezclas completas necesitan separacion; los stems no. | | `separation_method` | `demucs` cuando se necesita separacion | Demucs es el stem separator default esperado. | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target voice replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the retrieval index into the generator input. | | tipo del split generado | dataset primario `audio` | Los datos generados entrenan como audio normal, no como conditioning. | | ubicacion de cache | dentro de `output_dir` | Mantiene artifacts ligados al entrenamiento y reutilizables al reiniciar. | | captions | copia captions fuente salvo configuracion diferente | El nuevo split debe preservar letras y contexto de arreglo. | @@ -138,11 +149,12 @@ else: El repositorio del Hub debe usar un layout especifico de SimpleTuner, no una coleccion suelta de archivos: ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` El manifest es el contrato. Debe registrar el fingerprint del dataset de identidad objetivo, settings de entrenamiento RVC, settings del indice, sample rate esperado, versiones de herramientas y version del formato voice-transform de SimpleTuner. SimpleTuner no debe reutilizar un artifact del Hub si no tiene este manifest o si el manifest no coincide con el transform actual. Eso evita aplicar silenciosamente el modelo vocal equivocado a un dataset nuevo. @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` Para identidades privadas, mantén el repositorio del Hub privado salvo permiso explicito para publicar el modelo vocal. El audio generado y los artifacts del modelo pueden tener derechos diferentes, asi que trata sus settings de upload por separado. @@ -324,6 +337,16 @@ El manifest debe registrar fingerprint del dataset de identidad, settings del tr ## Consejos Practicos +Para la voz objetivo en `model.identity_data_dir`, la duracion importa menos que tener cobertura vocal limpia. + +- **Prueba rapida:** 30-60 segundos de audio vocal limpio pueden demostrar que el pipeline funciona, pero la voz convertida normalmente sera rudimentaria. +- **Inicio usable:** 5-10 minutos de voz aislada limpia es un primer objetivo razonable para un dataset de voz personal. +- **Identidad cantada:** 10-30 minutos es mejor cuando necesitas rango de pitch, vocales, dinamica, articulacion y fraseo expresivo. + +Usa muchos clips cortos en vez de un solo archivo largo. Clips de 5-20 segundos son mas faciles de revisar, separar y reutilizar. El trainer RVC actual remuestrea audio de identidad a 48 kHz y trunca cada archivo de identidad a `max_seconds_per_file`, que por defecto es `180`. Si un usuario proporciona un archivo de 30 minutos, por defecto solo se usan los primeros tres minutos. Dividir el dataset evita descartar por accidente cobertura vocal util. + +El proyecto standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) puede entrenar, guardar, cargar y publicar el artifact RVC sin ejecutar un job completo de SimpleTuner. Dentro de SimpleTuner, `scripts/run_rvc_model.py` ofrece una entrada directa para experimentar con la parte de entrenamiento y conversion RVC del pipeline. Usalo cuando quieras ajustar el dataset de identidad, modo Demucs, retrieval strength, transfer strength o reutilizacion de artifacts del Hub antes de gastar tiempo en el entrenamiento LoRA principal. + - Mantén una voz objetivo por LoRA cuando importa el control de identidad. - Prefiere ejemplos vocales limpios y secos para entrenar el modelo de conversion. - Evita duetos salvo que el objetivo sea aprender la mezcla del dueto. @@ -356,4 +379,4 @@ Tratalos como controles separados: ## Estado -Esta pagina describe el comportamiento previsto para un workflow experimental `data_transforms`. La restriccion importante de diseno es que identity transfer sea una funcion de audio de primera clase en SimpleTuner: entrenar o reutilizar el modelo de voice conversion, construir o reutilizar el indice, generar el split expandido, cachear resultados y continuar al entrenamiento normal sin exigir una segunda etapa manual de preprocessing. +Esta pagina describe un workflow experimental `data_transforms`. La implementacion actual entrena o reutiliza un artifact RVC v2 F0 de SimpleTuner, extrae features HuBERT y pitch RMVPE de los clips de identidad, ajusta el generador/discriminador RVC preentrenado, genera el split expandido, cachea los resultados y continua al entrenamiento normal sin exigir una segunda etapa manual de preprocessing. diff --git a/documentation/experimental/VOICE_CLONING.hi.md b/documentation/experimental/VOICE_CLONING.hi.md index 349bf803e..23f9934e8 100644 --- a/documentation/experimental/VOICE_CLONING.hi.md +++ b/documentation/experimental/VOICE_CLONING.hi.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## RVC-Style Identity Transfer -पहली intended implementation RVC-style voice conversion है। +पहली implementation RVC-style voice conversion है: HuBERT content features, RMVPE pitch extraction, NSF/VITS generator, multi-period discriminator, mel/adversarial losses, और optional retrieval index। इस context में "RVC model" voice-specific है। यह target identity dataset से train होता है। Retrieval index भी voice-specific है और उसी target voice की features से बनता है। Content features, pitch extraction, या separation models जैसे broad pretrained components reusable infrastructure हैं; conversion model और index singer या speaker-specific artifacts हैं। @@ -93,7 +93,7 @@ SimpleTuner को ये कर पाना चाहिए: ## Default Behavior -Planned defaults conservative हैं: +Defaults conservative हैं। इस workflow में audio backend वह expansion music है जिसे convert करना है, `model.identity_data_dir` target voice dataset है, और `target.instance_data_dir` सिर्फ generated output split का path है। | Setting | Default | Why | | --- | --- | --- | @@ -102,11 +102,22 @@ Planned defaults conservative हैं: | `train_if_missing` | `true` | SimpleTuner target dataset से voice model bootstrap कर सके। | | `force_retrain` | `false` | Valid cached voice model reuse हो सके। | | `build_index` | `true` | Retrieval identity stability improve कर सकता है और leakage घटा सकता है। | +| `identity_data_dir` | on-demand training में required | उस clean target voice के vocal examples की ओर point करता है जिसे expansion songs में transfer करना है। | +| `identity_audio_mode` | `separate` | Training से पहले identity clips पर Demucs चलाता है। अगर identity dataset में पहले से vocal stems हैं तो `vocal_only` use करें। | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs RVC generator/discriminator fine-tuning during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Standard RVC AdamW default. | | `hub_model_id` | unset | User opt-in न करे तो remote voice-model cache use नहीं होती। | | `reuse_from_hub` | `hub_model_id` set हो तो `true` | On-demand model training से पहले Hub check करता है। | | `push_to_hub` | `false` | Voice model upload explicit होना चाहिए क्योंकि artifact voice identity represent करता है। | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | full songs के लिए `separate_convert_remix`, vocal stems के लिए `vocal_only` | Full mixes को separation चाहिए; stems को नहीं। | | `separation_method` | separation चाहिए तो `demucs` | Demucs expected default stem separator है। | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target voice replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the retrieval index into the generator input. | | generated split type | primary `audio` dataset | Generated data normal audio की तरह train होता है, conditioning नहीं। | | cache location | `output_dir` के अंदर | Artifacts training run से जुड़े रहते हैं और restart पर reuse हो सकते हैं। | | captions | configured न हो तो source captions copy | नया split lyrics और arrangement context preserve करे। | @@ -138,11 +149,12 @@ else: Hub repository को loose files की जगह SimpleTuner-specific layout use करना चाहिए: ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` Manifest contract है। इसमें target identity dataset fingerprint, RVC training settings, index settings, expected sample rate, tool versions, और SimpleTuner voice-transform format version record होने चाहिए। SimpleTuner को ऐसा Hub artifact reuse नहीं करना चाहिए जिसमें manifest न हो या manifest current transform से match न करे। इससे गलत voice model को नए dataset पर silently apply करने से बचते हैं। @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` Private identities के लिए Hub repository private रखें जब तक voice model publish करने की explicit permission न हो। Generated audio और model artifacts के sharing rights अलग हो सकते हैं, इसलिए upload settings अलग-अलग treat करें। @@ -235,7 +248,7 @@ output_dir/ summary.json ``` -Useful local stats में RVC training loss, pitch loss अगर enabled हो, reconstruction या discriminator loss जब लागू हो, samples processed, elapsed time, DDP world size, cache hit या miss reason, और final model local cache, Hub cache, या on-demand training से आया या नहीं शामिल हैं। +Useful local stats में generator loss, discriminator loss, mel loss, KL loss, samples processed, elapsed time, DDP world size, cache hit या miss reason, और final model local cache, Hub cache, या on-demand training से आया या नहीं शामिल हैं। जब तक future implementation RVC transforms के लिए external logger integration explicitly add न करे, ये stats local-only हैं। @@ -324,6 +337,16 @@ Manifest में identity dataset fingerprint, transform settings, source expa ## Practical Dataset Advice +`model.identity_data_dir` में target voice के लिए duration से ज्यादा clean voiced coverage मायने रखती है। + +- **Smoke test:** 30-60 seconds का clean vocal audio pipeline चलने की पुष्टि कर सकता है, लेकिन converted voice आम तौर पर rough होगी। +- **Usable starter:** Personal voice dataset के लिए 5-10 minutes की clean isolated voice एक reasonable first target है। +- **Singing identity:** Pitch range, vowels, dynamics, articulation, और expressive phrasing चाहिए तो 10-30 minutes बेहतर है। + +एक लंबे file की जगह कई short clips use करें। 5-20 seconds के clips inspect, separate, और reuse करना आसान बनाते हैं। Current RVC trainer identity audio को 48 kHz पर resample करता है और हर identity file को `max_seconds_per_file` तक truncate करता है, जिसका default `180` है। अगर user एक 30-minute file देता है, default में सिर्फ पहले तीन minutes use होंगे। Dataset split करने से useful vocal coverage गलती से discard नहीं होती। + +Standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) project full SimpleTuner training job चलाए बिना RVC artifact train, save, load, और publish कर सकता है। SimpleTuner के अंदर `scripts/run_rvc_model.py` pipeline के RVC training और conversion हिस्से के साथ सीधे experiment करने का entrypoint देता है। Main LoRA training पर समय खर्च करने से पहले identity dataset, Demucs mode, retrieval strength, transfer strength, या Hub artifact reuse tune करने के लिए इसका उपयोग करें। + - Identity control important हो तो एक LoRA में एक target vocalist रखें। - Voice-conversion model के लिए clean, dry vocal examples prefer करें। - Duets avoid करें, जब तक goal duet blend सीखना न हो। @@ -356,4 +379,4 @@ Regularisation data usually LoRA को base model behavior preserve करन ## Status -यह page experimental `data_transforms` workflow के intended user-facing behavior को describe करता है। Important design constraint यह है कि identity transfer SimpleTuner audio training की first-class feature बने: voice-conversion model train या reuse करना, retrieval index build या reuse करना, expanded split generate करना, results cache करना, और separate manual preprocessing stage के बिना normal training में continue करना। +यह page experimental `data_transforms` workflow describe करता है। Current implementation SimpleTuner RVC v2 F0 artifact train या reuse करती है, identity clips से HuBERT content features और RMVPE pitch extract करती है, pretrained RVC generator/discriminator fine-tune करती है, retrieval index build करती है, expanded split generate करती है, results cache करती है, और separate manual preprocessing stage के बिना normal training में continue करती है। diff --git a/documentation/experimental/VOICE_CLONING.ja.md b/documentation/experimental/VOICE_CLONING.ja.md index febea5683..fcd582b33 100644 --- a/documentation/experimental/VOICE_CLONING.ja.md +++ b/documentation/experimental/VOICE_CLONING.ja.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## RVC 形式の Identity Transfer -最初に想定している実装は RVC 形式の voice conversion です。 +最初の実装は RVC 形式の voice conversion で、HuBERT content features、RMVPE pitch extraction、NSF/VITS generator、multi-period discriminator、mel/adversarial losses、optional retrieval index を使います。 ここでいう「RVC model」は voice-specific です。対象 identity dataset から学習されます。retrieval index も voice-specific で、同じ対象声の特徴から構築されます。content feature、pitch 抽出、separation model などの広い pretrained component は再利用される基盤です。一方、conversion model と index は歌手または話者固有の artifact です。 @@ -93,7 +93,7 @@ SimpleTuner は以下を行えるべきです。 ## Default Behavior -計画中の default は保守的です。 +default は保守的です。この workflow では、audio backend は変換したい拡張用の楽曲、`model.identity_data_dir` は対象 voice dataset、`target.instance_data_dir` は生成 output split の path です。 | Setting | Default | 理由 | | --- | --- | --- | @@ -102,11 +102,22 @@ SimpleTuner は以下を行えるべきです。 | `train_if_missing` | `true` | SimpleTuner が対象 dataset から voice model を bootstrap できるようにします。 | | `force_retrain` | `false` | 有効な cache model をできるだけ再利用します。 | | `build_index` | `true` | retrieval は identity の安定性を上げ、漏れを減らしやすいです。 | +| `identity_data_dir` | on-demand training では必須 | 拡張楽曲へ移したい対象 voice の clean vocal examples を指します。 | +| `identity_audio_mode` | `separate` | training 前に identity clips へ Demucs を実行します。identity dataset が既に vocal stems の場合は `vocal_only` を使います。 | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs RVC generator/discriminator fine-tuning during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Standard RVC AdamW default. | | `hub_model_id` | unset | user が opt-in しない限り remote voice-model cache は使いません。 | | `reuse_from_hub` | `hub_model_id` が設定されている場合は `true` | on-demand model を学習する前に Hub を確認します。 | | `push_to_hub` | `false` | voice model は声の identity を表すため、upload は明示的であるべきです。 | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | full song は `separate_convert_remix`、vocal stem は `vocal_only` | full mix には分離が必要で、stem には不要です。 | | `separation_method` | 分離が必要なら `demucs` | Demucs が想定 default の stem separator です。 | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target voice replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the retrieval index into the generator input. | | generated split type | primary `audio` dataset | 生成データは conditioning ではなく通常音声として学習します。 | | cache location | `output_dir` 内 | artifact を training run に結び付け、restart で再利用しやすくします。 | | captions | 設定がなければ source captions を copy | 新しい split は歌詞と編曲文脈を保持すべきです。 | @@ -138,11 +149,12 @@ else: Hub repository は、単なる file collection ではなく SimpleTuner-specific layout を使うべきです。 ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` manifest が contract です。target identity dataset fingerprint、RVC training settings、index settings、expected sample rate、tool versions、SimpleTuner voice-transform format version を記録します。manifest がない、または current transform と一致しない Hub artifact を SimpleTuner は再利用すべきではありません。これにより、間違った voice model を新しい dataset に黙って適用する事故を避けられます。 @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` private identity の場合、明示的な許可がない限り Hub repository は private のままにしてください。generated audio と model artifact は共有権利が異なる場合があるため、upload settings は別々に扱います。 @@ -235,7 +248,7 @@ output_dir/ summary.json ``` -有用な local stats には、RVC training loss、pitch loss が有効な場合の値、該当する reconstruction/discriminator loss、processed samples、elapsed time、DDP world size、cache hit/miss reason、final model が local cache、Hub cache、on-demand training のどれから来たかが含まれます。 +有用な local stats には、generator loss、discriminator loss、mel loss、KL loss、processed samples、elapsed time、DDP world size、cache hit/miss reason、final model が local cache、Hub cache、on-demand training のどれから来たかが含まれます。 これらの stats は、将来 RVC transforms 向けの external logger integration が明示的に追加されるまでは local-only です。 @@ -324,6 +337,16 @@ manifest には identity dataset fingerprint、transform settings、source expan ## Dataset Advice +`model.identity_data_dir` の target voice では、長さそのものより clean voiced coverage が重要です。 + +- **Smoke test:** 30-60 秒の clean vocal audio で pipeline が動くことは確認できますが、converted voice は通常かなり粗くなります。 +- **Usable starter:** 個人 voice dataset の最初の目標としては、5-10 分の clean isolated voice が現実的です。 +- **Singing identity:** pitch range、vowels、dynamics、articulation、expressive phrasing が必要な場合は 10-30 分の方が良いです。 + +1 つの長い file ではなく、多数の短い clip を使ってください。5-20 秒程度の clip は確認、分離、再利用がしやすくなります。現在の RVC trainer は identity audio を 48 kHz に resample し、各 identity file を `max_seconds_per_file` で truncate します。default は `180` です。ユーザーが 30 分の file を 1 つ渡した場合、default では最初の 3 分だけが使われます。dataset を分割すると、有用な vocal coverage を誤って捨てることを避けられます。 + +standalone の [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) project は、SimpleTuner の full training job を実行せずに RVC artifact を train、save、load、publish できます。SimpleTuner 内では `scripts/run_rvc_model.py` が、pipeline の RVC training/conversion 部分を直接試す entrypoint です。main LoRA training に時間を使う前に、identity dataset、Demucs mode、retrieval strength、transfer strength、Hub artifact reuse を調整したい場合に使ってください。 + - identity control が重要なら、1 つの LoRA に target vocalist は 1 人だけにします。 - voice-conversion model にはクリーンで dry な vocal examples を優先します。 - duet blend を学びたい場合以外は duet を避けます。 @@ -356,4 +379,4 @@ regularisation data は通常、LoRA に base model の挙動を保たせるた ## Status -このページは、実験的な `data_transforms` workflow の意図した user-facing behavior を説明しています。重要な設計制約は、identity transfer を SimpleTuner の first-class audio training feature にすることです。voice-conversion model の学習または再利用、retrieval index の構築または再利用、expanded split の生成、結果の cache、そして別の手動 preprocessing stage なしで通常 training へ進むことを目指します。 +このページは、実験的な `data_transforms` workflow を説明します。現在の実装は SimpleTuner RVC v2 F0 artifact を学習または再利用し、identity clips から HuBERT content features と RMVPE pitch を抽出し、pretrained RVC generator/discriminator を fine-tune し、expanded split を生成して cache し、別の手動 preprocessing stage なしで通常 training へ進みます。 diff --git a/documentation/experimental/VOICE_CLONING.md b/documentation/experimental/VOICE_CLONING.md index 661b50b4d..144b21f75 100644 --- a/documentation/experimental/VOICE_CLONING.md +++ b/documentation/experimental/VOICE_CLONING.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## RVC-Style Identity Transfer -The first intended implementation is RVC-style voice conversion. +The first implementation is RVC-style voice conversion using the same core ingredients that make community RVC trainers work: HuBERT/ContentVec-style content features, RMVPE pitch extraction, an NSF/VITS generator, a multi-period discriminator, adversarial and mel reconstruction losses, and an optional nearest-neighbor retrieval index. In this context, the "RVC model" is voice-specific. It is trained from the target identity dataset. The retrieval index is also voice-specific and is built from features from the same target voice. Broad pretrained components, such as content features, pitch extraction, or separation models, are reusable infrastructure; the conversion model and index are the artist- or speaker-specific artifacts. @@ -93,7 +93,7 @@ SimpleTuner should be able to: ## Default Behavior -The planned defaults are conservative: +The defaults are conservative. In this workflow, the audio backend is the expansion music that should be converted, `model.identity_data_dir` is the target voice dataset, and `target.instance_data_dir` is the generated output split. | Setting | Default | Why | | --- | --- | --- | @@ -102,11 +102,22 @@ The planned defaults are conservative: | `train_if_missing` | `true` | SimpleTuner should be able to bootstrap the voice model from the target dataset. | | `force_retrain` | `false` | Reuse a valid cached voice model when possible. | | `build_index` | `true` | Retrieval usually improves identity stability and reduces leakage. | +| `identity_data_dir` | required when training on demand | Points to clean vocal examples of the voice to transfer into the expansion songs. | +| `identity_audio_mode` | `separate` | Runs Demucs on identity clips before training. Use `vocal_only` when the identity dataset already contains vocal stems. | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Provides the default MIT-licensed RVC assets: HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | The current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs the RVC generator/discriminator fine-tuning stage during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Matches the standard RVC AdamW default. | | `hub_model_id` | unset | No remote voice-model cache is used unless the user opts in. | | `reuse_from_hub` | `true` when `hub_model_id` is set | Check the Hub before spending time training an on-demand model. | | `push_to_hub` | `false` | Uploading a voice model should be explicit because the artifact represents a voice identity. | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | `separate_convert_remix` for full songs, `vocal_only` for vocal stems | Full mixes need separation; stems do not. | | `separation_method` | `demucs` when separation is needed | Demucs is the expected default stem separator. | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target-voice vocal replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the saved retrieval index into the generator input. | | generated split type | primary `audio` dataset | The generated data is trained like normal audio, not used as conditioning. | | cache location | inside `output_dir` | Keeps generated artifacts tied to the training run and reusable on restart. | | captions | copy source captions unless configured otherwise | The new split should preserve lyrics and arrangement context. | @@ -135,17 +146,18 @@ else: stop and ask for a model path or a reusable cache ``` -The Hub repository should use a SimpleTuner-specific layout rather than a loose collection of files: +The Hub repository uses the `huggingface-hub-rvc` layout rather than a loose collection of files: ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` -The manifest is the contract. It should record the target identity dataset fingerprint, RVC training settings, index settings, expected sample rate, tool versions, and the SimpleTuner voice-transform format version. SimpleTuner should not reuse a Hub artifact that lacks this manifest or whose manifest does not match the current transform. That avoids silently applying the wrong voice model to a new dataset. +`config.json` records the package-level RVC metadata, including `model_name`. The voice manifest is the SimpleTuner contract. It should record the target identity dataset fingerprint, RVC training settings, index settings, expected sample rate, tool versions, and the SimpleTuner voice-transform format version. SimpleTuner should not reuse a Hub artifact that lacks this manifest or whose manifest does not match the current transform. That avoids silently applying the wrong voice model to a new dataset. Publishing should be opt-in. A reasonable pseudo config is: @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` For private identities, keep the Hub repository private unless you have explicit permission to publish the voice model. Generated audio and model artifacts may have different sharing rights, so treat their upload settings separately. @@ -178,6 +191,10 @@ Audio dataset Force retrain: off Build retrieval index: on Hub model id: optional + RVC asset repo: lj1995/VoiceConversionWebUI + Training steps: 1000 + Batch size: 4 + Learning rate: 1e-4 Reuse from Hub: on when Hub model id is set Push RVC model to Hub: off by default Hub repo privacy: private by default @@ -235,7 +252,7 @@ output_dir/ summary.json ``` -Useful local stats include RVC training loss, pitch loss if enabled, reconstruction or discriminator loss when applicable, samples processed, elapsed time, DDP world size, cache hit or miss reason, and whether the final model came from local cache, Hub cache, or on-demand training. +Useful local stats include generator loss, discriminator loss, mel loss, KL loss, samples processed, elapsed time, DDP world size, cache hit or miss reason, and whether the final model came from local cache, Hub cache, or on-demand training. These stats are local-only unless a future implementation explicitly adds external logger integration for RVC transforms. @@ -330,6 +347,16 @@ This is important because voice-conversion training and full-song separation can ## Practical Dataset Advice +For the target voice in `model.identity_data_dir`, duration matters less than clean voiced coverage. + +- **Smoke test:** 30-60 seconds of clean vocal audio can prove the pipeline runs, but the converted voice will usually be rough. +- **Usable starter:** 5-10 minutes of clean isolated voice is a reasonable first target for a personal voice dataset. +- **Singing identity:** 10-30 minutes is better when you need pitch range, vowels, dynamics, articulation, and expressive phrasing. + +Use many short clips rather than one long file. Clips around 5-20 seconds are easier to inspect, separate, and reuse. The current RVC trainer resamples identity audio to 48 kHz and truncates each identity file to `max_seconds_per_file`, which defaults to `180`. If a user provides one 30-minute file, only the first three minutes are used by default. Splitting the dataset avoids accidentally throwing away useful vocal coverage. + +The standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) project can train, save, load, and publish the RVC artifact without running a full SimpleTuner training job. Inside SimpleTuner, `scripts/run_rvc_model.py` provides a direct entrypoint for experimenting with the RVC training and conversion portion of the pipeline. Use it when you want to tune the identity dataset, Demucs mode, retrieval strength, transfer strength, or Hub artifact reuse before spending time on the main LoRA training run. + - Keep one target vocalist per generated LoRA when identity control matters. - Prefer clean, dry vocal examples for training the voice-conversion model. - Avoid duets unless the goal is specifically to learn the duet blend. @@ -348,6 +375,7 @@ This is important because voice-conversion training and full-song separation can | Vocal identity is weak | Voice-conversion model needs cleaner target data, more target data, or a stronger retrieval index. | | Captions do not control the voice | Captions still mention source-vocal identity or omit the target identity. | | The main model learns artifacts | Generated audio quality is too low or too dominant in the train mix. | +| Converted vocals are monotonic or robotic | The RVC path is missing proper F0 extraction, pretrained generator/discriminator initialization, adversarial training, or enough clean target vocal data. | ## Relationship To Regularisation Data @@ -363,4 +391,4 @@ Treat these as separate levers: ## Status -This page describes the intended user-facing behavior for an experimental `data_transforms` workflow. The important design constraint is that identity transfer should become a first-class SimpleTuner audio training feature: train or reuse the voice-conversion model, build or reuse the retrieval index, generate the expanded split, cache the results, and then continue into normal training without requiring a separate manual preprocessing stage. +This page describes an experimental `data_transforms` workflow. The current implementation trains or reuses a SimpleTuner RVC v2 F0 artifact, extracts HuBERT content features and RMVPE pitch from identity clips, fine-tunes the pretrained RVC generator/discriminator, builds a retrieval index, generates the expanded split, caches the results, and then continues into normal training without requiring a separate manual preprocessing stage. diff --git a/documentation/experimental/VOICE_CLONING.pt-BR.md b/documentation/experimental/VOICE_CLONING.pt-BR.md index 0ccbd11a4..f8c56638e 100644 --- a/documentation/experimental/VOICE_CLONING.pt-BR.md +++ b/documentation/experimental/VOICE_CLONING.pt-BR.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## Transferencia de Identidade Estilo RVC -A primeira implementacao pretendida e conversao de voz estilo RVC. +A primeira implementacao e conversao de voz estilo RVC, usando features HuBERT, pitch RMVPE, generator NSF/VITS, multi-period discriminator, losses mel/adversarial e indice de retrieval opcional. Neste contexto, o "modelo RVC" e especifico da voz. Ele e treinado a partir do dataset de identidade alvo. O indice de recuperacao tambem e especifico da voz e e construido com features da mesma voz alvo. Componentes pre-treinados amplos, como features de conteudo, extracao de pitch ou modelos de separacao, sao infraestrutura reutilizavel; o modelo de conversao e o indice sao artifacts especificos da cantora, cantor ou locutor. @@ -93,7 +93,7 @@ O SimpleTuner deve conseguir: ## Comportamento Padrao -Os defaults planejados sao conservadores: +Os defaults sao conservadores. Neste workflow, o backend de audio e a musica de expansao que sera convertida, `model.identity_data_dir` e o dataset da voz alvo, e `target.instance_data_dir` e apenas o caminho do split gerado. | Setting | Default | Por que | | --- | --- | --- | @@ -102,11 +102,22 @@ Os defaults planejados sao conservadores: | `train_if_missing` | `true` | O SimpleTuner deve bootstrapar o modelo vocal a partir do dataset alvo. | | `force_retrain` | `false` | Reutiliza um modelo em cache valido quando possivel. | | `build_index` | `true` | Retrieval costuma melhorar estabilidade de identidade e reduzir vazamento. | +| `identity_data_dir` | obrigatorio no treino sob demanda | Aponta para exemplos vocais limpos da voz que sera transferida para as musicas de expansao. | +| `identity_audio_mode` | `separate` | Executa Demucs nos clips de identidade antes do treino. Use `vocal_only` se o dataset de identidade ja contem vocal stems. | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs RVC generator/discriminator fine-tuning during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Standard RVC AdamW default. | | `hub_model_id` | nao definido | Nenhum cache remoto de modelo vocal e usado sem opt-in do usuario. | | `reuse_from_hub` | `true` quando `hub_model_id` esta definido | Verifica o Hub antes de gastar tempo treinando um modelo sob demanda. | | `push_to_hub` | `false` | Upload de modelo vocal deve ser explicito porque o artifact representa uma identidade vocal. | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | `separate_convert_remix` para musicas completas, `vocal_only` para vocal stems | Mix completo precisa de separacao; stems nao. | | `separation_method` | `demucs` quando separacao e necessaria | Demucs e o stem separator default esperado. | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target voice replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the retrieval index into the generator input. | | tipo do split gerado | dataset primario `audio` | Dados gerados treinam como audio normal, nao conditioning. | | local de cache | dentro de `output_dir` | Mantem artifacts ligados ao treino e reutilizaveis no restart. | | captions | copia captions da fonte salvo configuracao diferente | O novo split deve preservar letras e contexto de arranjo. | @@ -138,11 +149,12 @@ else: O repositorio no Hub deve usar um layout especifico do SimpleTuner, nao apenas arquivos soltos: ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` O manifest e o contrato. Ele deve registrar fingerprint do dataset de identidade alvo, settings de treino RVC, settings do indice, sample rate esperado, versoes das ferramentas e versao do formato voice-transform do SimpleTuner. O SimpleTuner nao deve reutilizar um artifact do Hub sem esse manifest ou com manifest que nao corresponde ao transform atual. Isso evita aplicar silenciosamente o modelo vocal errado a um novo dataset. @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` Para identidades privadas, mantenha o repositorio do Hub privado salvo permissao explicita para publicar o modelo vocal. Audio gerado e artifacts de modelo podem ter direitos diferentes, entao trate seus settings de upload separadamente. @@ -324,6 +337,16 @@ O manifest deve registrar fingerprint do dataset de identidade, settings do tran ## Conselhos Praticos +Para a voz alvo em `model.identity_data_dir`, a duracao importa menos do que cobertura vocal limpa. + +- **Teste rapido:** 30-60 segundos de audio vocal limpo podem provar que o pipeline roda, mas a voz convertida normalmente ficara rudimentar. +- **Inicio utilizavel:** 5-10 minutos de voz isolada limpa e um primeiro alvo razoavel para um dataset de voz pessoal. +- **Identidade cantada:** 10-30 minutos e melhor quando voce precisa de faixa de pitch, vogais, dinamica, articulacao e fraseado expressivo. + +Use muitos clips curtos em vez de um unico arquivo longo. Clips de 5-20 segundos sao mais faceis de revisar, separar e reutilizar. O trainer RVC atual reamostra o audio de identidade para 48 kHz e trunca cada arquivo de identidade para `max_seconds_per_file`, que por default e `180`. Se um usuario fornece um arquivo de 30 minutos, por default apenas os primeiros tres minutos sao usados. Dividir o dataset evita descartar cobertura vocal util por acidente. + +O projeto standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) pode treinar, salvar, carregar e publicar o artifact RVC sem executar um job completo do SimpleTuner. Dentro do SimpleTuner, `scripts/run_rvc_model.py` oferece uma entrada direta para experimentar com a parte de treinamento e conversao RVC do pipeline. Use quando quiser ajustar o dataset de identidade, modo Demucs, retrieval strength, transfer strength ou reutilizacao de artifacts do Hub antes de gastar tempo no treino LoRA principal. + - Mantenha uma voz alvo por LoRA quando controle de identidade importa. - Prefira exemplos vocais limpos e secos para treinar o modelo de voice conversion. - Evite duetos, a menos que o objetivo seja aprender o blend do dueto. @@ -356,4 +379,4 @@ Trate como controles separados: ## Status -Esta pagina descreve o comportamento pretendido para um workflow experimental `data_transforms`. A restricao central de design e que identity transfer seja uma funcionalidade de audio de primeira classe no SimpleTuner: treinar ou reutilizar o modelo de voice conversion, construir ou reutilizar o indice, gerar o split expandido, cachear resultados e seguir para o treino normal sem exigir uma segunda etapa manual de preprocessing. +Esta pagina descreve um workflow experimental `data_transforms`. A implementacao atual treina ou reutiliza um artifact RVC v2 F0 do SimpleTuner, extrai features HuBERT e pitch RMVPE dos clips de identidade, ajusta o gerador/discriminador RVC pretreinado, gera o split expandido, cacheia os resultados e continua para o treino normal sem exigir uma segunda etapa manual de preprocessing. diff --git a/documentation/experimental/VOICE_CLONING.zh.md b/documentation/experimental/VOICE_CLONING.zh.md index d72c06b98..57d31d47d 100644 --- a/documentation/experimental/VOICE_CLONING.zh.md +++ b/documentation/experimental/VOICE_CLONING.zh.md @@ -78,7 +78,7 @@ continue with normal metadata discovery, bucketing, caching, and training ## RVC 风格的身份迁移 -第一个计划支持的实现是 RVC 风格的语音转换。 +第一个实现是 RVC 风格的语音转换，使用 HuBERT 内容特征、RMVPE 音高提取、NSF/VITS generator、multi-period discriminator、mel/adversarial losses，以及可选 retrieval index。 这里的“RVC 模型”是声音专用的。它从目标身份数据集中训练得到。检索索引也是声音专用的，并由同一目标声音的特征构建。内容特征、音高提取、分离模型等预训练组件是可复用基础设施；转换模型和索引才是歌手或说话人专用的工件。 @@ -93,7 +93,7 @@ SimpleTuner 应该能够： ## 默认行为 -计划默认值比较保守： +默认值比较保守。在这个 workflow 中，音频 backend 是要被转换的扩展音乐，`model.identity_data_dir` 是目标声音数据集，`target.instance_data_dir` 只是生成输出 split 的路径。 | 设置 | 默认值 | 原因 | | --- | --- | --- | @@ -102,11 +102,22 @@ SimpleTuner 应该能够： | `train_if_missing` | `true` | SimpleTuner 应能从目标数据集启动训练声音模型。 | | `force_retrain` | `false` | 尽量复用有效缓存模型。 | | `build_index` | `true` | 检索通常能提高身份稳定性并减少泄漏。 | +| `identity_data_dir` | 按需训练时必填 | 指向要迁移到扩展歌曲中的干净目标声音示例。 | +| `identity_audio_mode` | `separate` | 训练前对身份音频运行 Demucs。若身份数据集已是人声 stems，请用 `vocal_only`。 | +| `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | +| `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | +| `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | +| `training_steps` | `1000` | Runs RVC generator/discriminator fine-tuning during startup. Increase for larger or more varied identity datasets. | +| `batch_size` | `4` | RVC training batch size before distributed sharding. Lower it for memory pressure. | +| `learning_rate` | `1e-4` | Standard RVC AdamW default. | | `hub_model_id` | 未设置 | 用户未显式启用时，不使用远程声音模型缓存。 | | `reuse_from_hub` | 设置 `hub_model_id` 时为 `true` | 在花时间按需训练前先检查 Hub。 | | `push_to_hub` | `false` | 声音模型代表一个声音身份，上传必须显式开启。 | +| `public` | `false` | Hub uploads are private by default. Set this to `true` only when the voice artifact can be published publicly. | | `audio_mode` | 完整歌曲默认 `separate_convert_remix`，人声 stem 默认 `vocal_only` | 完整混音需要分离；stem 不需要。 | | `separation_method` | 需要分离时使用 `demucs` | Demucs 是预期默认 stem 分离器。 | +| `timbre_strength` | `1.0` | Controls how strongly the synthesized target voice replaces the source vocal. Lower values blend source and converted vocals. | +| `retrieval_strength` | `0.75` | Blends nearest target-voice content frames from the retrieval index into the generator input. | | 生成 split 类型 | 主 `audio` 数据集 | 生成数据像普通音频一样训练，不作为 conditioning。 | | 缓存位置 | `output_dir` 内 | 让生成工件绑定训练运行，并能重启复用。 | | captions | 默认复制源 captions，除非另有配置 | 新 split 应保留歌词和编曲上下文。 | @@ -138,11 +149,12 @@ else: Hub 仓库应使用 SimpleTuner 专用布局，而不是松散文件集合： ```text +config.json voice_transform/ manifest.json - model.pth + model.safetensors + features.safetensors index.index - README.md ``` Manifest 是契约。它应记录目标身份数据集指纹、RVC 训练设置、索引设置、预期采样率、工具版本和 SimpleTuner voice-transform 格式版本。缺少 manifest 或 manifest 与当前 transform 不匹配时，SimpleTuner 不应复用该 Hub 工件。这样可以避免把错误声音模型静默应用到新数据集。 @@ -154,10 +166,11 @@ identity_transfer: method: rvc model: train_if_missing: true + model_name: Target voice RVC hub_model_id: org/target-voice-rvc reuse_from_hub: true push_to_hub: true - private: true + public: false ``` 对于私有声音身份，除非有明确授权，否则 Hub 仓库应保持私有。生成音频和模型工件可能有不同共享权利，因此应分别处理上传设置。 @@ -235,7 +248,7 @@ output_dir/ summary.json ``` -有用的本地统计包括 RVC 训练 loss、启用时的 pitch loss、适用时的 reconstruction 或 discriminator loss、已处理样本数、耗时、DDP world size、缓存命中或未命中原因，以及最终模型来自本地缓存、Hub 缓存还是按需训练。 +有用的本地统计包括 generator loss、discriminator loss、mel loss、KL loss、已处理样本数、耗时、DDP world size、缓存命中或未命中原因，以及最终模型来自本地缓存、Hub 缓存还是按需训练。 除非未来实现明确为 RVC transforms 添加外部 logger 集成，否则这些统计仅保存在本地。 @@ -324,6 +337,16 @@ Manifest 应记录身份数据集指纹、变换设置、扩展源数据指纹 ## 实用数据集建议 +对于 `model.identity_data_dir` 里的目标声音，干净的有声覆盖比单纯时长更重要。 + +- **快速冒烟测试：** 30-60 秒干净人声可以证明 pipeline 能跑通，但转换声音通常会比较粗糙。 +- **可用起点：** 对个人声音数据集来说，5-10 分钟干净、独立的人声是合理的第一目标。 +- **歌唱身份：** 如果需要覆盖音高范围、元音、动态、咬字和表现性 phrasing，10-30 分钟会更好。 + +请使用许多短 clip，而不是一个很长的文件。5-20 秒左右的 clip 更容易检查、分离和复用。当前 RVC trainer 会把身份音频重采样到 48 kHz，并把每个身份文件截断到 `max_seconds_per_file`，默认值是 `180`。如果用户提供一个 30 分钟文件，默认只会使用前三分钟。拆分数据集可以避免意外丢掉有用的人声覆盖。 + +独立的 [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface-hub-rvc) 项目可以在不运行完整 SimpleTuner 训练任务的情况下训练、保存、加载和发布 RVC artifact。在 SimpleTuner 中，`scripts/run_rvc_model.py` 提供了一个直接入口，用于更深入地实验 pipeline 的 RVC 训练和转换部分。若想在主 LoRA 训练前调试身份数据集、Demucs 模式、retrieval strength、transfer strength 或 Hub artifact 复用，请先使用它。 + - 身份控制重要时，每个 LoRA 保持一个目标歌手。 - 用干净、较干的人声样本训练语音转换模型。 - 除非目标就是学习合唱混合，否则避免二重唱。 @@ -356,4 +379,4 @@ Manifest 应记录身份数据集指纹、变换设置、扩展源数据指纹 ## 状态 -本文描述计划中的实验性 `data_transforms` 工作流的用户体验。核心设计约束是：身份迁移应成为 SimpleTuner 的一等音频训练功能，能够训练或复用语音转换模型、构建或复用检索索引、生成扩展 split、缓存结果，然后直接进入正常训练，而不要求用户手动执行第二个预处理阶段。 +本文描述实验性的 `data_transforms` workflow。当前实现会训练或复用 SimpleTuner RVC v2 F0 artifact，从身份音频中提取 HuBERT content features 和 RMVPE pitch，微调预训练 RVC generator/discriminator，构建 retrieval index，生成扩展 split，缓存结果，然后直接进入正常训练，不要求用户手动执行第二个 preprocessing 阶段。 From cccdd195aa7968a0242bdc896d881e50011ca4bc Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Wed, 26 Aug 2026 18:13:25 -0600 Subject: [PATCH 09/12] ignore output artifacts from testing --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f63b768bb..920538ef6 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ package-lock.json site/ tunableop_results*.csv .cog/ +artifacts/ From a60aec183b82c6c30f596962d497a3776b2b4bf8 Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Thu, 27 Aug 2026 12:20:04 -0600 Subject: [PATCH 10/12] fix selection of demucs by default --- documentation/DATALOADER.es.md | 1 + documentation/DATALOADER.hi.md | 1 + documentation/DATALOADER.ja.md | 1 + documentation/DATALOADER.md | 1 + documentation/DATALOADER.pt-BR.md | 1 + documentation/DATALOADER.zh.md | 1 + .../experimental/VOICE_CLONING.es.md | 2 ++ .../experimental/VOICE_CLONING.hi.md | 2 ++ .../experimental/VOICE_CLONING.ja.md | 2 ++ documentation/experimental/VOICE_CLONING.md | 2 ++ .../experimental/VOICE_CLONING.pt-BR.md | 2 ++ .../experimental/VOICE_CLONING.zh.md | 2 ++ scripts/run_rvc_model.py | 8 +++++++ setup.py | 2 +- .../data_transforms/identity_transfer.py | 3 +++ .../static/js/dataloader-section-component.js | 2 ++ simpletuner/templates/trainer_htmx.html | 1 + .../js/dataloader_audio_capabilities.test.js | 1 + .../test_data_transforms_identity_transfer.py | 13 +++++++++++ tests/test_run_rvc_model_script.py | 23 +++++++++++++++++++ 20 files changed, 70 insertions(+), 1 deletion(-) diff --git a/documentation/DATALOADER.es.md b/documentation/DATALOADER.es.md index a775105c0..33d68e266 100644 --- a/documentation/DATALOADER.es.md +++ b/documentation/DATALOADER.es.md @@ -264,6 +264,7 @@ Los backends de memoria requieren Linux o macOS y suficiente RAM o swap para la - **Descripción:** Expande un dataset fuente en uno o más datasets de entrenamiento generados antes de que empiece la configuración normal del dataloader. Los datasets generados se tratan como datasets primarios normales salvo que el transform pida explícitamente clonar metadatos. - **Identity transfer de audio:** `{"task": "identity_transfer", "method": "rvc"}` está disponible para backends con `dataset_type: "audio"`. Prepara un split de audio generado para transferencia de identidad vocal y usa el directorio de salida para artefactos de voz cacheados y archivos generados. Consulta [Voice Cloning Data Transforms](experimental/VOICE_CLONING.es.md). - **Identity data:** Pon la musica a convertir en el `instance_data_dir` del backend de audio, los ejemplos de la voz objetivo en `model.identity_data_dir`, y la ruta del split generado en `target.instance_data_dir`. +- **Debug de stems:** Define `model.identity_stem_debug_dir` para conservar previews `vocals.wav` y `no_vocals.wav` de la identidad separada que realmente usa el entrenamiento RVC. Sirve para revisar casos donde la salida suena como si los instrumentos se hubieran aprendido como parte de la voz. - **Estado:** La implementación experimental incluye comprobaciones de manifiestos de caché, reutilización/subida de artefactos a Hub mediante el layout `huggingface-hub-rvc`, sharding de inicio compatible con DDP, logs locales de RVC y un trainer/converter compacto de voice-transfer de SimpleTuner. El modo full-song remix usa Demucs para separar voces; el modo vocal-stem no necesita separación. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.hi.md b/documentation/DATALOADER.hi.md index 0a4aec85a..a0f87dfe7 100644 --- a/documentation/DATALOADER.hi.md +++ b/documentation/DATALOADER.hi.md @@ -264,6 +264,7 @@ Memory backends के लिए Linux या macOS और मौजूदा ca - **Description:** Normal dataloader setup शुरू होने से पहले source dataset को एक या अधिक generated training datasets में expand करता है। Generated datasets को regular primary datasets की तरह treat किया जाता है, जब तक transform explicitly metadata clone न मांगे। - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` `dataset_type: "audio"` backends के लिए उपलब्ध है। यह voice identity transfer के लिए generated audio split तैयार करता है और output directory में cached voice artifacts तथा generated files रखता है। देखें [Voice Cloning Data Transforms](experimental/VOICE_CLONING.hi.md)। - **Identity data:** Convert की जाने वाली music audio backend के `instance_data_dir` में रखें, target voice examples `model.identity_data_dir` में रखें, और generated split path `target.instance_data_dir` में रखें। +- **Stem debugging:** `model.identity_stem_debug_dir` set करने पर RVC training में सच में इस्तेमाल हुए separated identity `vocals.wav` और `no_vocals.wav` previews बचाए जाते हैं। Output में instruments voice का हिस्सा लगें तो separation result जांचने के लिए इसका उपयोग करें। - **Status:** Experimental implementation cache manifest checks, Hub artifact reuse/push through the `huggingface-hub-rvc` artifact layout, DDP-aware startup sharding, local RVC logs, और compact SimpleTuner voice-transfer trainer/converter देती है। Full-song remix mode vocal separation के लिए Demucs इस्तेमाल करता है; vocal-stem mode में separation की जरूरत नहीं है। ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.ja.md b/documentation/DATALOADER.ja.md index 097e3cfa0..4790d0827 100644 --- a/documentation/DATALOADER.ja.md +++ b/documentation/DATALOADER.ja.md @@ -264,6 +264,7 @@ Hugging Face の音声データセットでは、キャプション（プロン - **Description:** 通常の dataloader 設定が始まる前に、ソース dataset から 1 つ以上の生成済み training dataset を展開します。transform が明示的に metadata clone を要求しない限り、生成 dataset は通常の primary dataset として扱われます。 - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` は `dataset_type: "audio"` の backend で利用できます。音声 identity transfer 用の生成 audio split を準備し、output directory に voice artifact と生成ファイルをキャッシュします。詳しくは [Voice Cloning Data Transforms](experimental/VOICE_CLONING.ja.md) を参照してください。 - **Identity data:** 変換したい音楽は audio backend の `instance_data_dir`、対象 voice examples は `model.identity_data_dir`、生成 split path は `target.instance_data_dir` に置きます。 +- **Stem debugging:** `model.identity_stem_debug_dir` を設定すると、RVC training が実際に使う分離後の identity `vocals.wav` と `no_vocals.wav` preview を保存できます。出力で楽器まで声として学習されたように聞こえる場合の確認に使います。 - **Status:** 実験的な実装は、cache manifest の確認、`huggingface-hub-rvc` artifact layout による Hub artifact の reuse/push、DDP 対応の startup sharding、ローカル RVC log、そして小さな SimpleTuner voice-transfer trainer/converter を提供します。Full-song remix mode は Demucs で vocal separation を行い、vocal-stem mode では separation は不要です。 ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.md b/documentation/DATALOADER.md index d5b563528..307eff08c 100644 --- a/documentation/DATALOADER.md +++ b/documentation/DATALOADER.md @@ -266,6 +266,7 @@ Memory backends require Linux or macOS and enough RAM or swap for the existing c - **Description:** Expands a source dataset into one or more generated training datasets before normal dataloader setup begins. Generated datasets are treated as regular primary datasets unless the transform explicitly asks to clone metadata. - **Audio identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` is available for `dataset_type: "audio"` backends. It prepares a generated audio split for voice identity transfer and uses the output directory for cached voice artifacts and generated files. See [Voice Cloning Data Transforms](experimental/VOICE_CLONING.md). - **Identity data:** Put the music to be converted in the audio backend's `instance_data_dir`, put the target voice examples in `model.identity_data_dir`, and put the generated split path in `target.instance_data_dir`. +- **Stem debugging:** Set `model.identity_stem_debug_dir` to preserve the separated identity `vocals.wav` and `no_vocals.wav` previews used by RVC training. This is useful when outputs sound like instruments were learned as part of the voice. - **Status:** The experimental implementation includes cache manifest checks, Hub artifact reuse/push through the `huggingface-hub-rvc` artifact layout, DDP-aware startup sharding, local RVC logs, and a compact SimpleTuner voice-transfer trainer/converter. Full-song remix mode uses Demucs for vocal separation; vocal-stem mode does not need separation. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.pt-BR.md b/documentation/DATALOADER.pt-BR.md index 554bb9125..41e873f39 100644 --- a/documentation/DATALOADER.pt-BR.md +++ b/documentation/DATALOADER.pt-BR.md @@ -264,6 +264,7 @@ Backends de memória exigem Linux ou macOS e RAM ou swap suficiente para o cache - **Descrição:** Expande um dataset fonte em um ou mais datasets de treinamento gerados antes da configuração normal do dataloader. Datasets gerados são tratados como datasets primários normais, a menos que o transform peça explicitamente clonagem de metadados. - **Identity transfer de áudio:** `{"task": "identity_transfer", "method": "rvc"}` está disponível para backends com `dataset_type: "audio"`. Ele prepara um split de áudio gerado para transferência de identidade vocal e usa o diretório de saída para artefatos de voz em cache e arquivos gerados. Consulte [Voice Cloning Data Transforms](experimental/VOICE_CLONING.pt-BR.md). - **Identity data:** Coloque a musica a converter no `instance_data_dir` do backend de audio, os exemplos da voz alvo em `model.identity_data_dir`, e o caminho do split gerado em `target.instance_data_dir`. +- **Debug de stems:** Defina `model.identity_stem_debug_dir` para preservar previews `vocals.wav` e `no_vocals.wav` da identidade separada que o treino RVC realmente usou. Isso ajuda quando a saída soa como se instrumentos tivessem sido aprendidos como parte da voz. - **Status:** A implementação experimental inclui verificações de manifest de cache, reutilização/envio de artefatos para o Hub pelo layout `huggingface-hub-rvc`, sharding de inicialização compatível com DDP, logs locais de RVC e um trainer/converter compacto de voice-transfer do SimpleTuner. O modo full-song remix usa Demucs para separar vocais; o modo vocal-stem não precisa de separação. ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/DATALOADER.zh.md b/documentation/DATALOADER.zh.md index a112cfbc8..125ed197e 100644 --- a/documentation/DATALOADER.zh.md +++ b/documentation/DATALOADER.zh.md @@ -263,6 +263,7 @@ LTX-2 使用原生纯音频分支；MiniMax-H3 在打包序列中为每个 laten - **说明:** 在常规 dataloader 设置开始前，把源数据集展开为一个或多个生成的训练数据集。除非 transform 明确要求克隆元数据，生成的数据集会被当作普通主训练数据集处理。 - **音频 identity transfer:** `{"task": "identity_transfer", "method": "rvc"}` 可用于 `dataset_type: "audio"` 后端。它会为声音身份迁移准备生成音频 split，并在输出目录中缓存 voice artifacts 和生成文件。参见 [Voice Cloning Data Transforms](experimental/VOICE_CLONING.zh.md)。 - **Identity data:** 要转换的音乐放在音频 backend 的 `instance_data_dir`，目标声音示例放在 `model.identity_data_dir`，生成 split 路径放在 `target.instance_data_dir`。 +- **Stem 调试:** 设置 `model.identity_stem_debug_dir` 可以保留 RVC 训练实际使用的分离后 identity `vocals.wav` 和 `no_vocals.wav` 预览。当输出听起来像把乐器也学进声音时，用它检查分离结果。 - **状态:** 实验性实现包含缓存 manifest 检查、通过 `huggingface-hub-rvc` artifact layout 进行 Hub artifact 复用/上传、DDP 感知的启动 sharding、本地 RVC 日志，以及一个紧凑的 SimpleTuner voice-transfer 训练/转换器。Full-song remix 模式使用 Demucs 分离人声；vocal-stem 模式不需要分离。 ### `instance_data_dir` / `aws_data_prefix` diff --git a/documentation/experimental/VOICE_CLONING.es.md b/documentation/experimental/VOICE_CLONING.es.md index 400e23cff..b4af6e858 100644 --- a/documentation/experimental/VOICE_CLONING.es.md +++ b/documentation/experimental/VOICE_CLONING.es.md @@ -104,6 +104,7 @@ Los defaults son conservadores. En este workflow, el backend de audio es la musi | `build_index` | `true` | Retrieval suele mejorar estabilidad de identidad y reducir leakage. | | `identity_data_dir` | obligatorio en entrenamiento bajo demanda | Apunta a ejemplos vocales limpios de la voz que se transferira a las canciones de expansion. | | `identity_audio_mode` | `separate` | Ejecuta Demucs en los clips de identidad antes del entrenamiento. Usa `vocal_only` si el dataset de identidad ya contiene vocal stems. | +| `identity_stem_debug_dir` | sin definir | Directorio opcional para guardar previews `vocals.wav` y `no_vocals.wav` de la identidad. Usalo para confirmar que RVC entrena con voces aisladas, no con leakage de instrumentos. | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -361,6 +362,7 @@ El proyecto standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io | La LoRA solo funciona en un genero | La identidad vocal sigue entrelazada con captions de arreglo o datos fuente. | | El split generado suena hueco o con fase rara | Artifacts de separacion/remix en procesamiento de canciones completas. | | Los instrumentos suenan convertidos como voz | Se uso `full_mix_convert` cuando hacia falta separacion. | +| El modelo vocal parece aprender instrumentos | La separacion de identidad produjo vocal stems con demasiado leakage de acompanamiento. Define `model.identity_stem_debug_dir`, inspecciona los stems guardados, o preprocesa stems vocales mas limpios y usa `identity_audio_mode=vocal_only`. | | La identidad vocal es debil | Faltan datos objetivo mas limpios, mas datos o mejor indice. | | Las captions no controlan la voz | Las captions aun mencionan la voz fuente u omiten la identidad objetivo. | | El modelo principal aprende artifacts | El audio generado tiene baja calidad o demasiado peso en el mix de entrenamiento. | diff --git a/documentation/experimental/VOICE_CLONING.hi.md b/documentation/experimental/VOICE_CLONING.hi.md index 23f9934e8..a7161ab5e 100644 --- a/documentation/experimental/VOICE_CLONING.hi.md +++ b/documentation/experimental/VOICE_CLONING.hi.md @@ -104,6 +104,7 @@ Defaults conservative हैं। इस workflow में audio backend व | `build_index` | `true` | Retrieval identity stability improve कर सकता है और leakage घटा सकता है। | | `identity_data_dir` | on-demand training में required | उस clean target voice के vocal examples की ओर point करता है जिसे expansion songs में transfer करना है। | | `identity_audio_mode` | `separate` | Training से पहले identity clips पर Demucs चलाता है। अगर identity dataset में पहले से vocal stems हैं तो `vocal_only` use करें। | +| `identity_stem_debug_dir` | unset | Identity `vocals.wav` और `no_vocals.wav` previews save करने की optional directory। इससे verify करें कि RVC isolated vocals से train कर रहा है, instrument bleed से नहीं। | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -361,6 +362,7 @@ Standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/huggingface | LoRA सिर्फ एक genre में काम करती है | Voice identity अभी भी arrangement captions या source data से entangled है। | | Generated split hollow या phasey सुनाई देता है | Full-song processing में separation/remix artifacts। | | Instruments voice-converted जैसे सुनते हैं | Separation चाहिए थी लेकिन `full_mix_convert` use हुआ। | +| Voice model instruments सीखता हुआ लगता है | Identity separation के vocal stems में accompaniment bleed बहुत ज्यादा है। `model.identity_stem_debug_dir` set करके saved stems inspect करें, या cleaner vocal stems preprocess करके `identity_audio_mode=vocal_only` use करें। | | Vocal identity weak है | Voice-conversion model को cleaner target data, more data, या stronger retrieval index चाहिए। | | Captions voice control नहीं करतीं | Captions में source-vocal identity बची है या target identity missing है। | | Main model artifacts सीखता है | Generated audio low quality है या train mix में बहुत dominant है। | diff --git a/documentation/experimental/VOICE_CLONING.ja.md b/documentation/experimental/VOICE_CLONING.ja.md index fcd582b33..8b6580bfd 100644 --- a/documentation/experimental/VOICE_CLONING.ja.md +++ b/documentation/experimental/VOICE_CLONING.ja.md @@ -104,6 +104,7 @@ default は保守的です。この workflow では、audio backend は変換し | `build_index` | `true` | retrieval は identity の安定性を上げ、漏れを減らしやすいです。 | | `identity_data_dir` | on-demand training では必須 | 拡張楽曲へ移したい対象 voice の clean vocal examples を指します。 | | `identity_audio_mode` | `separate` | training 前に identity clips へ Demucs を実行します。identity dataset が既に vocal stems の場合は `vocal_only` を使います。 | +| `identity_stem_debug_dir` | unset | identity の `vocals.wav` と `no_vocals.wav` preview を保存する任意の directory です。RVC が楽器 bleed ではなく分離済み vocal から training しているか確認できます。 | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -361,6 +362,7 @@ standalone の [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/hugging | LoRA が 1 ジャンルでしか効かない | 声 identity が arrangement caption や source data とまだ絡んでいます。 | | generated split が phasey または hollow に聞こえる | full-song 処理の separation/remix artifact。 | | 楽器まで変声されたように聞こえる | 分離が必要なのに `full_mix_convert` を使っています。 | +| voice model が楽器まで学習したように聞こえる | identity separation の vocal stem に accompaniment bleed が多すぎます。`model.identity_stem_debug_dir` で保存された stems を確認するか、より clean な vocal stems を前処理して `identity_audio_mode=vocal_only` を使ってください。 | | vocal identity が弱い | target data の品質、量、または retrieval index が不足しています。 | | caption が声を制御しない | source vocal identity が caption に残っている、または target identity がありません。 | | main model が artifact を学ぶ | generated audio の品質が低い、または train mix 内で強すぎます。 | diff --git a/documentation/experimental/VOICE_CLONING.md b/documentation/experimental/VOICE_CLONING.md index 144b21f75..3cba1c581 100644 --- a/documentation/experimental/VOICE_CLONING.md +++ b/documentation/experimental/VOICE_CLONING.md @@ -104,6 +104,7 @@ The defaults are conservative. In this workflow, the audio backend is the expans | `build_index` | `true` | Retrieval usually improves identity stability and reduces leakage. | | `identity_data_dir` | required when training on demand | Points to clean vocal examples of the voice to transfer into the expansion songs. | | `identity_audio_mode` | `separate` | Runs Demucs on identity clips before training. Use `vocal_only` when the identity dataset already contains vocal stems. | +| `identity_stem_debug_dir` | unset | Optional directory for saved identity `vocals.wav` and `no_vocals.wav` previews. Use it to verify that RVC is training from isolated vocals rather than instrument bleed. | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Provides the default MIT-licensed RVC assets: HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | The current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -372,6 +373,7 @@ The standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/hugging | The LoRA only works in one genre | Voice identity is still entangled with arrangement captions or source data. | | The generated split sounds phasey or hollow | Separation/remix artifacts from full-song processing. | | Instruments sound like they were voice-converted | `full_mix_convert` was used where separation was needed. | +| The voice model seems to learn instruments | Identity separation produced vocal stems with too much accompaniment bleed. Set `model.identity_stem_debug_dir`, inspect the saved stems, or preprocess cleaner vocal stems and use `identity_audio_mode=vocal_only`. | | Vocal identity is weak | Voice-conversion model needs cleaner target data, more target data, or a stronger retrieval index. | | Captions do not control the voice | Captions still mention source-vocal identity or omit the target identity. | | The main model learns artifacts | Generated audio quality is too low or too dominant in the train mix. | diff --git a/documentation/experimental/VOICE_CLONING.pt-BR.md b/documentation/experimental/VOICE_CLONING.pt-BR.md index f8c56638e..7107ce806 100644 --- a/documentation/experimental/VOICE_CLONING.pt-BR.md +++ b/documentation/experimental/VOICE_CLONING.pt-BR.md @@ -104,6 +104,7 @@ Os defaults sao conservadores. Neste workflow, o backend de audio e a musica de | `build_index` | `true` | Retrieval costuma melhorar estabilidade de identidade e reduzir vazamento. | | `identity_data_dir` | obrigatorio no treino sob demanda | Aponta para exemplos vocais limpos da voz que sera transferida para as musicas de expansao. | | `identity_audio_mode` | `separate` | Executa Demucs nos clips de identidade antes do treino. Use `vocal_only` se o dataset de identidade ja contem vocal stems. | +| `identity_stem_debug_dir` | nao definido | Diretorio opcional para salvar previews `vocals.wav` e `no_vocals.wav` da identidade. Use para confirmar que o RVC esta treinando com vocais isolados, nao com vazamento de instrumentos. | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -361,6 +362,7 @@ O projeto standalone [`huggingface-hub-rvc`](https://github.com/SimpleTuner-io/h | A LoRA so funciona em um genero | A identidade vocal ainda esta entrelacada com captions de arranjo ou dados fonte. | | O split gerado soa oco ou com fase estranha | Artifacts de separacao/remix em processamento de musica completa. | | Instrumentos parecem convertidos como voz | `full_mix_convert` foi usado quando separacao era necessaria. | +| O modelo vocal parece aprender instrumentos | A separacao de identidade gerou vocal stems com vazamento demais de acompanhamento. Defina `model.identity_stem_debug_dir`, inspecione os stems salvos, ou preprocesse stems vocais mais limpos e use `identity_audio_mode=vocal_only`. | | Identidade vocal fraca | O modelo precisa de dados alvo mais limpos, mais dados ou indice melhor. | | Captions nao controlam a voz | Captions ainda mencionam a voz fonte ou omitem a identidade alvo. | | O modelo principal aprende artifacts | Audio gerado tem baixa qualidade ou peso demais no mix de treino. | diff --git a/documentation/experimental/VOICE_CLONING.zh.md b/documentation/experimental/VOICE_CLONING.zh.md index 57d31d47d..f98e1a7d5 100644 --- a/documentation/experimental/VOICE_CLONING.zh.md +++ b/documentation/experimental/VOICE_CLONING.zh.md @@ -104,6 +104,7 @@ SimpleTuner 应该能够： | `build_index` | `true` | 检索通常能提高身份稳定性并减少泄漏。 | | `identity_data_dir` | 按需训练时必填 | 指向要迁移到扩展歌曲中的干净目标声音示例。 | | `identity_audio_mode` | `separate` | 训练前对身份音频运行 Demucs。若身份数据集已是人声 stems，请用 `vocal_only`。 | +| `identity_stem_debug_dir` | 未设置 | 可选目录，用来保存 identity `vocals.wav` 和 `no_vocals.wav` 预览。用它确认 RVC 是否从隔离人声训练，而不是从乐器泄漏训练。 | | `asset_hub_model_id` | `lj1995/VoiceConversionWebUI` | Default RVC asset repository for HuBERT, RMVPE, and v2 48k pretrained generator/discriminator checkpoints. | | `model_name` | transform or Hub repo name | Human-readable name saved into the RVC artifact so downloaded caches are identifiable outside their folder name. | | `sample_rate` | `48000` | Current implementation targets RVC v2 48k assets. Other rates need matching pretrained assets and configs. | @@ -361,6 +362,7 @@ Manifest 应记录身份数据集指纹、变换设置、扩展源数据指纹 | LoRA 只在一种流派里有效 | 声音身份仍与编曲 caption 或源数据纠缠。 | | 生成 split 听起来空、相位怪 | 完整歌曲处理中的分离或 remix 伪影。 | | 乐器像被变声了一样 | 需要分离时使用了 `full_mix_convert`。 | +| 声音模型像学进了乐器 | Identity separation 的人声 stem 混入太多伴奏。设置 `model.identity_stem_debug_dir` 检查保存的 stems，或先预处理更干净的人声 stems 并使用 `identity_audio_mode=vocal_only`。 | | 人声身份很弱 | 目标数据需要更干净、更多样，或检索索引更强。 | | Caption 控制不了声音 | Caption 仍含源声音身份，或没有目标身份。 | | 主模型学到伪影 | 生成音频质量太低或在训练 mix 中占比太高。 | diff --git a/scripts/run_rvc_model.py b/scripts/run_rvc_model.py index e739bd621..beaca895e 100755 --- a/scripts/run_rvc_model.py +++ b/scripts/run_rvc_model.py @@ -73,6 +73,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--batch-size", type=int, default=4, help="RVC training batch size.") parser.add_argument("--learning-rate", type=float, default=1e-4, help="RVC AdamW learning rate.") parser.add_argument("--max-seconds-per-file", type=float, default=180.0, help="Maximum identity audio seconds per file.") + parser.add_argument( + "--identity-stem-debug-dir", + type=Path, + default=None, + help="Optional directory for identity vocal/accompaniment stem previews.", + ) parser.add_argument("--pretrained-generator-path", type=Path, default=None) parser.add_argument("--pretrained-discriminator-path", type=Path, default=None) parser.add_argument("--rmvpe-model-path", type=Path, default=None) @@ -120,6 +126,7 @@ def build_data_backend_config(args: argparse.Namespace) -> list[dict[str, Any]]: or (args.hub_model_id.rstrip("/").rsplit("/", 1)[-1] if args.hub_model_id else args.transform_id), "sample_rate": args.sample_rate, "identity_audio_mode": args.identity_audio_mode, + "separation_method": args.separation_method, "training_steps": args.training_steps, "batch_size": args.batch_size, "learning_rate": args.learning_rate, @@ -133,6 +140,7 @@ def build_data_backend_config(args: argparse.Namespace) -> list[dict[str, Any]]: _set_if_present(model, "device", args.device) _set_if_present(model, "demucs_device", args.demucs_device) _set_if_present(model, "demucs_model", args.demucs_model) + _set_if_present(model, "identity_stem_debug_dir", args.identity_stem_debug_dir) _set_if_present(model, "pretrained_generator_path", args.pretrained_generator_path) _set_if_present(model, "pretrained_discriminator_path", args.pretrained_discriminator_path) _set_if_present(model, "rmvpe_model_path", args.rmvpe_model_path) diff --git a/setup.py b/setup.py index 11102a86e..5bf2b8509 100644 --- a/setup.py +++ b/setup.py @@ -326,7 +326,7 @@ def _collect_package_files(*directories: str): "praat-parselmouth>=0.4.5,<1", "scikit-learn>=1.6.0,<2", "soundfile>=0.13.0,<1", - "huggingface-hub-rvc>=0.1.3", + "huggingface-hub-rvc>=0.1.4", "demucs>=4.0.1", "lyricsgenius>=3.0.1", "mutagen>=1.47.0", diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py index f6afedfdf..363289f20 100644 --- a/simpletuner/helpers/data_transforms/identity_transfer.py +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -344,6 +344,7 @@ def _normalise_transform_config(self, existing_backend_ids: set[str]) -> Dict[st conversion.setdefault("timbre_strength", 1.0) conversion.setdefault("retrieval_strength", 0.75) transform["conversion"] = conversion + model.setdefault("separation_method", conversion["separation_method"]) target = deepcopy(transform.get("target") or {}) target.setdefault("id", transform_id) @@ -513,6 +514,7 @@ def _generated_fingerprint(self, transform: Dict[str, Any]) -> str: "force_retrain", "hub_model_id", "hub_token", + "identity_stem_debug_dir", "model_name", "push_to_hub", "public", @@ -540,6 +542,7 @@ def _voice_model_fingerprint(self, transform: Dict[str, Any]) -> str: "force_retrain", "hub_model_id", "hub_token", + "identity_stem_debug_dir", "model_name", "push_to_hub", "public", diff --git a/simpletuner/static/js/dataloader-section-component.js b/simpletuner/static/js/dataloader-section-component.js index d5c6401c8..094c5993d 100644 --- a/simpletuner/static/js/dataloader-section-component.js +++ b/simpletuner/static/js/dataloader-section-component.js @@ -635,6 +635,7 @@ function dataloaderSectionComponent() { model_name: `${dataset.id || 'audio'} RVC voice`, sample_rate: 48000, identity_audio_mode: 'separate', + separation_method: 'demucs', training_steps: 1000, batch_size: 4, learning_rate: 0.0001 @@ -668,6 +669,7 @@ function dataloaderSectionComponent() { if (transform.model.learning_rate === undefined) transform.model.learning_rate = 0.0001; transform.conversion.audio_mode = transform.conversion.audio_mode || 'separate_convert_remix'; transform.conversion.separation_method = transform.conversion.separation_method || 'demucs'; + transform.model.separation_method = transform.model.separation_method || transform.conversion.separation_method; if (transform.conversion.timbre_strength === undefined) transform.conversion.timbre_strength = 1.0; if (transform.conversion.retrieval_strength === undefined) transform.conversion.retrieval_strength = 0.75; return transform; diff --git a/simpletuner/templates/trainer_htmx.html b/simpletuner/templates/trainer_htmx.html index 8d01e1c25..966e25d0d 100644 --- a/simpletuner/templates/trainer_htmx.html +++ b/simpletuner/templates/trainer_htmx.html @@ -2337,6 +2337,7 @@ normalized.conversion.audio_mode = 'separate_convert_remix'; } normalized.conversion.separation_method = this._coerceString(conversion.separation_method) || 'demucs'; + normalized.model.separation_method = this._coerceString(model.separation_method) || normalized.conversion.separation_method; const timbreStrength = parseFloat(conversion.timbre_strength); if (Number.isFinite(timbreStrength)) { normalized.conversion.timbre_strength = Math.max(0, Math.min(1, timbreStrength)); diff --git a/tests/js/dataloader_audio_capabilities.test.js b/tests/js/dataloader_audio_capabilities.test.js index b6e2dc6da..1fd424111 100644 --- a/tests/js/dataloader_audio_capabilities.test.js +++ b/tests/js/dataloader_audio_capabilities.test.js @@ -252,6 +252,7 @@ describe('Dataloader Section Audio Capabilities', () => { expect(dataset.data_transforms[0].model.model_name).toBe('voice-source RVC voice'); expect(dataset.data_transforms[0].model.sample_rate).toBe(48000); expect(dataset.data_transforms[0].model.identity_audio_mode).toBe('separate'); + expect(dataset.data_transforms[0].model.separation_method).toBe('demucs'); expect(dataset.data_transforms[0].model.training_steps).toBe(1000); expect(dataset.data_transforms[0].model.batch_size).toBe(4); expect(dataset.data_transforms[0].model.learning_rate).toBe(0.0001); diff --git a/tests/test_data_transforms_identity_transfer.py b/tests/test_data_transforms_identity_transfer.py index 2395d5990..3d0806fd7 100644 --- a/tests/test_data_transforms_identity_transfer.py +++ b/tests/test_data_transforms_identity_transfer.py @@ -127,6 +127,7 @@ def test_local_voice_model_artifact_reuses_matching_manifest(self): self.assertFalse(normalised["model"]["push_to_hub"]) self.assertFalse(normalised["model"]["public"]) + self.assertEqual(normalised["model"]["separation_method"], "demucs") fingerprint = transform._voice_model_fingerprint(normalised) cache_dir = Path(normalised["model"]["cache_dir"]) @@ -154,6 +155,17 @@ def test_local_voice_model_artifact_reuses_matching_manifest(self): self.assertEqual(artifact.model_path, cache_dir / "model.safetensors") + def test_identity_stem_debug_dir_does_not_change_voice_model_fingerprint(self): + transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) + base = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) + with_debug = json.loads(json.dumps(base)) + with_debug["model"]["identity_stem_debug_dir"] = "debug-stems" + + self.assertEqual( + transform._voice_model_fingerprint(base), + transform._voice_model_fingerprint(with_debug), + ) + def test_local_voice_model_artifact_loads_legacy_pth_manifest(self): transform = self._transform({"task": "identity_transfer", "id": "voice-transfer"}) normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) @@ -317,6 +329,7 @@ def test_process_data_transforms_trains_and_converts_with_rvc_artifact(self): (source_dir / "source.txt").write_text("rock vocal test", encoding="utf-8") def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): + self.assertEqual(transform_config["model"]["separation_method"], "demucs") cache_dir.mkdir(parents=True) model_path = cache_dir / "model.pth" manifest_path = cache_dir / "manifest.json" diff --git a/tests/test_run_rvc_model_script.py b/tests/test_run_rvc_model_script.py index a30514cc8..fefe6d2f8 100644 --- a/tests/test_run_rvc_model_script.py +++ b/tests/test_run_rvc_model_script.py @@ -59,6 +59,8 @@ def test_dry_run_builds_identity_transfer_backend_config(self): self.assertEqual(transform["task"], "identity_transfer") self.assertEqual(transform["method"], "rvc") self.assertEqual(transform["model"]["identity_audio_mode"], "separate") + self.assertEqual(transform["model"]["separation_method"], "demucs") + self.assertNotIn("identity_stem_debug_dir", transform["model"]) self.assertEqual(transform["model"]["model_name"], "Test Voice") self.assertTrue(transform["model"]["public"]) self.assertEqual(transform["conversion"]["audio_mode"], "separate_convert_remix") @@ -67,6 +69,27 @@ def test_dry_run_builds_identity_transfer_backend_config(self): self.assertEqual(transform["model"]["device"], "cpu") self.assertEqual(transform["target"]["instance_data_dir"], str(generated_dir)) + def test_build_config_accepts_identity_stem_debug_dir(self): + args = self.script.build_parser().parse_args( + [ + "--source-dir", + "source", + "--identity-dir", + "identity", + "--generated-dir", + "generated", + "--output-dir", + "output", + "--identity-stem-debug-dir", + "debug-stems", + ] + ) + + config = self.script.build_data_backend_config(args) + + transform = config[0]["data_transforms"][0] + self.assertEqual(transform["model"]["identity_stem_debug_dir"], "debug-stems") + if __name__ == "__main__": unittest.main() From 7b39aa9e6ddc5ea117bce8219ddedfe7626e7371 Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Thu, 27 Aug 2026 15:40:44 -0600 Subject: [PATCH 11/12] Support remote RVC identity transfer sources --- simpletuner/helpers/data_transforms/base.py | 11 +- .../data_transforms/identity_transfer.py | 190 +++++++++++++++++- .../test_data_transforms_identity_transfer.py | 152 +++++++++++++- 3 files changed, 343 insertions(+), 10 deletions(-) diff --git a/simpletuner/helpers/data_transforms/base.py b/simpletuner/helpers/data_transforms/base.py index 8c05c7b9f..5150e9c3b 100644 --- a/simpletuner/helpers/data_transforms/base.py +++ b/simpletuner/helpers/data_transforms/base.py @@ -12,6 +12,7 @@ class DataTransformTask(ABC): TASK: str = "" SUPPORTED_SOURCE_DATASET_TYPES: tuple[str, ...] = () + SOURCE_ONLY_BY_DEFAULT = False REQUIRES_METADATA_CLONE = False def __init__( @@ -84,11 +85,15 @@ def process_data_transforms( if backend.get("disabled", False) or backend.get("disable", False): continue - for transform_config in _normalise_transform_list(backend.get("id", "<unknown>"), backend.get("data_transforms")): + transform_configs = _normalise_transform_list(backend.get("id", "<unknown>"), backend.get("data_transforms")) + source_only = bool(backend.get("data_transform_source_only", False)) + for transform_config in transform_configs: task_name = transform_config.get("task") if not task_name: raise ValueError(f"data_transforms entry for backend {backend.get('id')!r} requires a 'task' value.") + source_only = source_only or bool(transform_config.get("source_only", False)) task_cls = get_data_transform_task(str(task_name)) + source_only = source_only or bool(getattr(task_cls, "SOURCE_ONLY_BY_DEFAULT", False)) task = task_cls( global_config=global_config, source_backend_config=backend, @@ -105,7 +110,9 @@ def process_data_transforms( raise ValueError(f"Data transform {task_name!r} generated duplicate backend id {backend_id!r}.") existing_backend_ids.add(backend_id) generated_backends.append(new_backend) + if source_only and transform_configs: + backend["_data_transform_source_only"] = True if generated_backends: data_backend_config.extend(generated_backends) - return data_backend_config + return [backend for backend in data_backend_config if not backend.get("_data_transform_source_only", False)] diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py index 363289f20..8fceb9c9b 100644 --- a/simpletuner/helpers/data_transforms/identity_transfer.py +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -26,6 +26,8 @@ VOICE_TRANSFORM_FORMAT = "simpletuner-voice-transform" VOICE_TRANSFORM_FORMAT_VERSION = 1 +AUDIO_EXTENSIONS = {".flac", ".wav", ".mp3", ".ogg", ".m4a", ".aac", ".opus"} +HF_DATASET_URI_PREFIX = "hf://datasets/" def _utc_now() -> str: @@ -247,10 +249,41 @@ def _voice_model_name(manifest: Dict[str, Any], hub_model_id: Optional[str] = No return "RVC Voice Model" +def _flatten_data_files(data_files: Any) -> List[str]: + if data_files is None: + return [] + if isinstance(data_files, str): + return [data_files] + if isinstance(data_files, dict): + values: List[str] = [] + for value in data_files.values(): + values.extend(_flatten_data_files(value)) + return values + if isinstance(data_files, (list, tuple)): + values = [] + for value in data_files: + values.extend(_flatten_data_files(value)) + return values + return [] + + +def _parse_hf_dataset_uri(uri: str) -> tuple[str, str]: + if not uri.startswith(HF_DATASET_URI_PREFIX): + raise ValueError(f"Expected a Hugging Face dataset URI starting with {HF_DATASET_URI_PREFIX!r}: {uri!r}") + path = uri[len(HF_DATASET_URI_PREFIX) :].lstrip("/") + parts = path.split("/", 2) + if len(parts) < 2 or not parts[0] or not parts[1]: + raise ValueError(f"Hugging Face dataset URI must include an org/name repo id: {uri!r}") + repo_id = f"{parts[0]}/{parts[1]}" + pattern = parts[2] if len(parts) > 2 and parts[2] else "**/*" + return repo_id, pattern + + @register_data_transform class IdentityTransferTransform(DataTransformTask): TASK = "identity_transfer" SUPPORTED_SOURCE_DATASET_TYPES = ("audio",) + SOURCE_ONLY_BY_DEFAULT = True REQUIRES_METADATA_CLONE = False def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: @@ -275,6 +308,7 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: ) return [target_backend_config] + transform = self._with_materialized_identity_backend(transform, run_logger) artifact = self._resolve_voice_model(transform, voice_model_fingerprint, model_cache_dir, run_logger) if not self._is_main_process(): self._wait_for_everyone() @@ -283,7 +317,8 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: ) return [target_backend_config] - input_paths = self._rank_shard(self._discover_source_audio_paths()) + runtime_source_backend = self._materialized_source_backend(transform, run_logger) + input_paths = self._rank_shard(self._discover_source_audio_paths(runtime_source_backend)) run_logger.event( transform_id, "conversion_start", @@ -292,7 +327,7 @@ def prepare(self, existing_backend_ids: set[str]) -> List[Dict[str, Any]]: world_size=self._world_size(), ) RVCConverter().convert( - source_backend_config=self.source_backend_config, + source_backend_config=runtime_source_backend, target_backend_config=target_backend_config, transform_config=transform, artifact=artifact, @@ -336,6 +371,8 @@ def _normalise_transform_config(self, existing_backend_ids: set[str]) -> Dict[st model.setdefault("batch_size", 4) model.setdefault("learning_rate", 1e-4) model.setdefault("max_seconds_per_file", 180.0) + if "identity_data_backend" in model and not isinstance(model["identity_data_backend"], dict): + raise ValueError("identity_transfer.model.identity_data_backend must be a dataset backend config dictionary.") transform["model"] = model conversion = deepcopy(transform.get("conversion") or {}) @@ -393,6 +430,9 @@ def _target_backend_config(self, transform: Dict[str, Any], generated_dir: Path) target_audio.update(deepcopy(target.get("audio") or {})) if target_audio: target_cfg["audio"] = target_audio + if target_cfg.get("type") != source_cfg.get("type"): + for storage_key in ("huggingface", "parquet", "csv", "audio_column"): + target_cfg.pop(storage_key, None) return target_cfg def _resolve_voice_model( @@ -561,20 +601,158 @@ def _voice_model_fingerprint(self, transform: Dict[str, Any]) -> str: def _fingerprint(self, transform: Dict[str, Any]) -> str: return self._generated_fingerprint(transform) - def _discover_source_audio_paths(self) -> List[str]: + def _materialized_source_backend( + self, + transform: Dict[str, Any], + run_logger: RVCTransformLogger, + ) -> Dict[str, Any]: source_type = self.source_backend_config.get("type") + if source_type == "local": + return self.source_backend_config + if source_type != "huggingface": + raise NotImplementedError( + "identity_transfer input discovery is currently implemented for local and Hugging Face audio source " + f"backends only. Received type={source_type!r}." + ) + + materialized_dir = Path(self._output_dir()) / "cache" / "data_transforms" / transform["id"] / "source_audio" + self._materialize_huggingface_backend( + self.source_backend_config, + materialized_dir, + run_logger, + transform["id"], + role="source", + ) + runtime_backend = deepcopy(self.source_backend_config) + runtime_backend["type"] = "local" + runtime_backend["metadata_backend"] = "discovery" + runtime_backend["caption_strategy"] = runtime_backend.get("caption_strategy") or "textfile" + runtime_backend["instance_data_dir"] = str(materialized_dir) + runtime_backend.pop("huggingface", None) + runtime_backend.pop("data_files", None) + return runtime_backend + + def _with_materialized_identity_backend( + self, + transform: Dict[str, Any], + run_logger: RVCTransformLogger, + ) -> Dict[str, Any]: + model_cfg = transform.get("model") or {} + identity_backend = model_cfg.get("identity_data_backend") + identity_data_dir = model_cfg.get("identity_data_dir") or model_cfg.get("voice_data_dir") + if not identity_backend and not ( + isinstance(identity_data_dir, str) and identity_data_dir.startswith(HF_DATASET_URI_PREFIX) + ): + return transform + + materialized_dir = Path(self._output_dir()) / "cache" / "data_transforms" / transform["id"] / "identity_audio" + if identity_backend: + backend_config = deepcopy(identity_backend) + else: + backend_config = { + "id": f"{transform['id']}-identity", + "type": "huggingface", + "dataset_type": "audio", + "data_files": {"train": identity_data_dir}, + "huggingface": {"data_files": {"train": identity_data_dir}}, + } + self._materialize_huggingface_backend( + backend_config, + materialized_dir, + run_logger, + transform["id"], + role="identity", + ) + runtime_transform = deepcopy(transform) + runtime_transform["model"]["identity_data_dir"] = str(materialized_dir) + runtime_transform["model"].pop("voice_data_dir", None) + return runtime_transform + + def _materialize_huggingface_backend( + self, + backend_config: Dict[str, Any], + destination: Path, + run_logger: RVCTransformLogger, + transform_id: str, + role: str, + ) -> None: + if backend_config.get("type") != "huggingface": + raise ValueError(f"identity_transfer {role} materialization expected a Hugging Face backend config.") + + data_files = _flatten_data_files(backend_config.get("data_files")) + hf_config = backend_config.get("huggingface") or {} + data_files.extend(_flatten_data_files(hf_config.get("data_files"))) + data_files = [value for value in dict.fromkeys(data_files) if value.startswith(HF_DATASET_URI_PREFIX)] + if not data_files: + raise ValueError( + f"identity_transfer Hugging Face {role} backend requires data_files entries using hf://datasets/ URIs." + ) + + repo_patterns: Dict[str, set[str]] = {} + for uri in data_files: + repo_id, pattern = _parse_hf_dataset_uri(uri) + repo_patterns.setdefault(repo_id, set()).add(pattern) + + if len(repo_patterns) != 1: + repos = ", ".join(sorted(repo_patterns)) + raise ValueError( + f"identity_transfer Hugging Face {role} materialization currently expects one dataset repo, got: {repos}" + ) + + repo_id, patterns = next(iter(repo_patterns.items())) + allow_patterns = sorted(patterns | {"**/*.txt", "**/*.lyrics"}) + run_logger.event( + transform_id, + "huggingface_materialization_start", + role=role, + repo_id=repo_id, + destination=str(destination), + allow_patterns=allow_patterns, + ) + if self._is_main_process(): + destination.mkdir(parents=True, exist_ok=True) + try: + from huggingface_hub import snapshot_download + except ImportError as exc: + raise ImportError("huggingface_hub is required for identity_transfer Hugging Face sources.") from exc + snapshot_path = Path( + snapshot_download( + repo_id=repo_id, + repo_type="dataset", + allow_patterns=allow_patterns, + token=hf_config.get("token") or backend_config.get("token"), + ) + ) + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(snapshot_path, destination) + self._wait_for_everyone() + if not destination.exists(): + raise FileNotFoundError(f"identity_transfer failed to materialize Hugging Face {role} data at {destination}") + discovered = self._discover_source_audio_paths({"type": "local", "instance_data_dir": str(destination)}) + run_logger.event( + transform_id, + "huggingface_materialization_complete", + role=role, + repo_id=repo_id, + file_count=len(discovered), + destination=str(destination), + ) + + def _discover_source_audio_paths(self, backend_config: Optional[Dict[str, Any]] = None) -> List[str]: + backend_config = backend_config or self.source_backend_config + source_type = backend_config.get("type") if source_type != "local": raise NotImplementedError( "identity_transfer input discovery is currently implemented for local audio source backends only." ) - instance_data_dir = self.source_backend_config.get("instance_data_dir") + instance_data_dir = backend_config.get("instance_data_dir") if not instance_data_dir: raise ValueError("identity_transfer local source backend requires instance_data_dir.") - audio_exts = {".flac", ".wav", ".mp3", ".ogg", ".m4a", ".aac", ".opus"} root = Path(instance_data_dir) if not root.exists(): raise FileNotFoundError(f"identity_transfer source instance_data_dir does not exist: {instance_data_dir}") - paths = [str(path) for path in sorted(root.rglob("*")) if path.suffix.lower() in audio_exts] + paths = [str(path) for path in sorted(root.rglob("*")) if path.suffix.lower() in AUDIO_EXTENSIONS] if not paths: raise ValueError(f"identity_transfer found no audio files under {instance_data_dir}.") return paths diff --git a/tests/test_data_transforms_identity_transfer.py b/tests/test_data_transforms_identity_transfer.py index 3d0806fd7..e8e7ae72e 100644 --- a/tests/test_data_transforms_identity_transfer.py +++ b/tests/test_data_transforms_identity_transfer.py @@ -89,7 +89,8 @@ def test_process_data_transforms_reuses_generated_cache_as_primary_audio_dataset ], ) - generated = config[1] + self.assertEqual(len(config), 1) + generated = config[0] self.assertEqual(generated["id"], "artist-source_identity_transfer") self.assertEqual(generated["dataset_type"], "audio") self.assertEqual(generated["generated_by"], "data_transforms") @@ -384,13 +385,160 @@ def fake_convert(_self, source_backend_config, target_backend_config, *_args, ** ], ) - generated = config[1] + self.assertEqual(len(config), 1) + generated = config[0] self.assertEqual(generated["id"], "voice-transfer") self.assertTrue((generated_dir / "source.wav").exists()) self.assertEqual((generated_dir / "source.txt").read_text(encoding="utf-8"), "rock vocal test") model_path = Path(self.args.output_dir) / "cache" / "data_transforms" / "voice-transfer" / "rvc_model" / "model.pth" self.assertTrue(model_path.exists()) + def test_primary_dataset_remains_when_identity_transfer_source_is_auxiliary(self): + generated_dir = Path(self.temp_dir) / "generated" + identity_dir = Path(self.temp_dir) / "identity" + self._write_wav(identity_dir / "voice.wav", frequency=320.0) + + def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): + cache_dir.mkdir(parents=True) + model_path = cache_dir / "model.pth" + manifest_path = cache_dir / "manifest.json" + model_path.write_bytes(b"rvc") + manifest = {**manifest_base, "fingerprint": fingerprint, "voice_model": {"kind": "simpletuner-rvc-v2-f0"}} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return VoiceModelArtifact(cache_dir, manifest_path, model_path, None, manifest) + + def fake_convert(_self, source_backend_config, target_backend_config, *_args, **_kwargs): + output_dir = Path(target_backend_config["instance_data_dir"]) + output_dir.mkdir(parents=True) + shutil.copy2(Path(source_backend_config["instance_data_dir"]) / "sample.flac", output_dir / "sample.wav") + + primary_backend = {**self.source_backend, "id": "primary-singer"} + transform_source = { + **self.source_backend, + "id": "random-expansion", + "data_transforms": [ + { + "task": "identity_transfer", + "id": "voice-transfer", + "model": { + "identity_data_dir": str(identity_dir), + "sample_rate": 48000, + "identity_audio_mode": "vocal_only", + }, + "conversion": {"audio_mode": "vocal_only"}, + "target": {"instance_data_dir": str(generated_dir)}, + } + ], + } + + with ( + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCTrainer.train", new=fake_train), + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCConverter.convert", new=fake_convert), + ): + config = process_data_transforms( + global_config=self.args, + data_backend_config=[primary_backend, transform_source], + ) + + self.assertEqual([backend["id"] for backend in config], ["primary-singer", "voice-transfer"]) + self.assertTrue((generated_dir / "sample.wav").exists()) + + def test_huggingface_source_and_identity_backend_materialize_before_rvc(self): + hf_source = Path(self.temp_dir) / "hf-source-snapshot" + hf_identity = Path(self.temp_dir) / "hf-identity-snapshot" + generated_dir = Path(self.temp_dir) / "generated-hf" + self._write_wav(hf_source / "source.wav", frequency=180.0) + self._write_wav(hf_identity / "voice.wav", frequency=320.0) + (hf_source / "source.txt").write_text("diverse rock source", encoding="utf-8") + (hf_source / "source.lyrics").write_text("[verse]\nwords", encoding="utf-8") + + def fake_snapshot_download(repo_id, **kwargs): + self.assertEqual(kwargs["repo_type"], "dataset") + self.assertIn("**/*.txt", kwargs["allow_patterns"]) + self.assertIn("**/*.lyrics", kwargs["allow_patterns"]) + if repo_id == "RareConcepts/random-test-music-data": + return str(hf_source) + if repo_id == "RareConcepts/music-test-data-serj-vocals": + return str(hf_identity) + raise AssertionError(f"unexpected repo_id={repo_id}") + + def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): + identity_dir = Path(transform_config["model"]["identity_data_dir"]) + self.assertTrue((identity_dir / "voice.wav").exists()) + cache_dir.mkdir(parents=True) + model_path = cache_dir / "model.pth" + manifest_path = cache_dir / "manifest.json" + model_path.write_bytes(b"rvc") + manifest = {**manifest_base, "fingerprint": fingerprint, "voice_model": {"kind": "simpletuner-rvc-v2-f0"}} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + return VoiceModelArtifact(cache_dir, manifest_path, model_path, None, manifest) + + def fake_convert( + _self, source_backend_config, target_backend_config, transform_config, artifact, input_paths, **_kwargs + ): + self.assertEqual(source_backend_config["type"], "local") + self.assertTrue(Path(source_backend_config["instance_data_dir"]).exists()) + self.assertEqual([Path(path).name for path in input_paths], ["source.wav"]) + output_dir = Path(target_backend_config["instance_data_dir"]) + output_dir.mkdir(parents=True) + shutil.copy2(Path(input_paths[0]), output_dir / "source.wav") + shutil.copy2(Path(input_paths[0]).with_suffix(".txt"), output_dir / "source.txt") + shutil.copy2(Path(input_paths[0]).with_suffix(".lyrics"), output_dir / "source.lyrics") + + with ( + patch("huggingface_hub.snapshot_download", new=fake_snapshot_download), + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCTrainer.train", new=fake_train), + patch("simpletuner.helpers.data_transforms.identity_transfer.RVCConverter.convert", new=fake_convert), + ): + config = process_data_transforms( + global_config=self.args, + data_backend_config=[ + { + "id": "random-expansion-hf", + "type": "huggingface", + "dataset_type": "audio", + "metadata_backend": "huggingface", + "caption_strategy": "huggingface", + "data_files": {"train": "hf://datasets/RareConcepts/random-test-music-data/**/*.wav"}, + "huggingface": { + "data_files": { + "train": "hf://datasets/RareConcepts/random-test-music-data/**/*.wav", + }, + }, + "audio": {"sample_rate": 48000, "channels": 2}, + "data_transforms": [ + { + "task": "identity_transfer", + "id": "voice-transfer-hf", + "model": { + "identity_data_backend": { + "id": "serj-identity", + "type": "huggingface", + "dataset_type": "audio", + "data_files": { + "train": "hf://datasets/RareConcepts/music-test-data-serj-vocals/**/*.wav", + }, + }, + "sample_rate": 48000, + "identity_audio_mode": "vocal_only", + }, + "conversion": {"audio_mode": "vocal_only"}, + "target": {"instance_data_dir": str(generated_dir)}, + } + ], + } + ], + ) + + self.assertEqual(len(config), 1) + generated = config[0] + self.assertEqual(generated["id"], "voice-transfer-hf") + self.assertEqual(generated["type"], "local") + self.assertNotIn("huggingface", generated) + self.assertTrue((generated_dir / "source.wav").exists()) + self.assertTrue((generated_dir / "source.txt").exists()) + self.assertTrue((generated_dir / "source.lyrics").exists()) + def test_training_requires_identity_data_dir(self): transform = self._transform({"task": "identity_transfer"}) normalised = transform._normalise_transform_config(existing_backend_ids={"artist-source"}) From c05310080f632ab658a174e60ab268f1593c4741 Mon Sep 17 00:00:00 2001 From: bghira <bghira@users.github.com> Date: Fri, 28 Aug 2026 22:04:27 -0600 Subject: [PATCH 12/12] Fix RVC identity transfer startup issues --- cog.yaml | 5 ++ predict.py | 13 +-- .../data_transforms/identity_transfer.py | 87 ++++++++++++++----- .../helpers/models/minimaxmusic/model.py | 20 ++++- simpletuner/helpers/multiaspect/dataset.py | 6 +- simpletuner/helpers/rvc/runtime.py | 9 ++ simpletuner/helpers/rvc/simple.py | 4 + simpletuner/helpers/training/trainer.py | 3 +- .../server/services/cloud/replicate_client.py | 2 +- .../test_data_transforms_identity_transfer.py | 80 ++++++++++++++++- tests/test_dataset.py | 20 +++++ tests/test_minimaxmusic_model.py | 42 +++++++++ tests/test_replicate_hardware_profiles.py | 36 ++++++++ tests/test_trainer.py | 21 +++++ 14 files changed, 313 insertions(+), 35 deletions(-) create mode 100644 simpletuner/helpers/rvc/runtime.py diff --git a/cog.yaml b/cog.yaml index 81debd708..2ff44f8d6 100644 --- a/cog.yaml +++ b/cog.yaml @@ -9,10 +9,15 @@ build: - libsndfile1 run: - rm -rf SimpleTuner && git clone --depth 1 --branch main https://github.com/bghira/SimpleTuner.git SimpleTuner + - python -c "from pathlib import Path; p=Path('SimpleTuner/simpletuner/helpers/multiaspect/dataset.py'); s=p.read_text(); old='if model_family != \"ace_step\" and \"target_size\" in image_metadata:'; new='if (\\n model_family != \"ace_step\"\\n and image_metadata.get(\"dataset_type\") != \"audio\"\\n and \"target_size\" in image_metadata\\n ):'; assert old in s or new in s; p.write_text(s.replace(old, new))" + - python -c 'from pathlib import Path; import base64; p=Path("SimpleTuner/simpletuner/helpers/models/minimaxmusic/model.py"); s=p.read_text(); old=base64.b64decode("ICAgICAgICAgICAgY2FwdGlvbiA9IGV4YW1wbGUuZ2V0KCJwcm9tcHQiKSBvciBleGFtcGxlLmdldCgidGFncyIpCiAgICAgICAgICAgIGx5cmljcyA9IGV4YW1wbGUuZ2V0KCJseXJpY3MiKQogICAgICAgICAgICBpZiBub3QgaXNpbnN0YW5jZShjYXB0aW9uLCBzdHIpIG9yIG5vdCBjYXB0aW9uLnN0cmlwKCk6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJNaW5pTWF4IE11c2ljIDMgbGFuZ3VhZ2UgbW9kZWwgdHJhaW5pbmcgcmVxdWlyZXMgJ3Byb21wdCcgKG9yICd0YWdzJykgbWV0YWRhdGEuIikKICAgICAgICAgICAgaWYgbm90IGlzaW5zdGFuY2UobHlyaWNzLCBzdHIpOgogICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAiTWluaU1heCBNdXNpYyAzIGxhbmd1YWdlIG1vZGVsIHRyYWluaW5nIHJlcXVpcmVzICdseXJpY3MnIG1ldGFkYXRhIChhbiBlbXB0eSBzdHJpbmcgaXMgIgogICAgICAgICAgICAgICAgICAgICJhbGxvd2VkIGZvciBpbnN0cnVtZW50YWwgb3IgcmVndWxhcmlzYXRpb24gdHJhY2tzKS4iCiAgICAgICAgICAgICAgICApCg==").decode(); new=base64.b64decode("ICAgICAgICAgICAgY2FwdGlvbiA9IE5vbmUKICAgICAgICAgICAgY2FwdGlvbl9wcmVzZW50ID0gRmFsc2UKICAgICAgICAgICAgZm9yIGtleSBpbiAoInByb21wdCIsICJ0YWdzIiwgImluc3RhbmNlX3Byb21wdF90ZXh0Iik6CiAgICAgICAgICAgICAgICBpZiBrZXkgbm90IGluIGV4YW1wbGU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGNhbmRpZGF0ZSA9IGV4YW1wbGVba2V5XQogICAgICAgICAgICAgICAgaWYgY2FuZGlkYXRlIGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGNhcHRpb24gPSBjYW5kaWRhdGUKICAgICAgICAgICAgICAgIGNhcHRpb25fcHJlc2VudCA9IFRydWUKICAgICAgICAgICAgICAgIGJyZWFrCiAgICAgICAgICAgIGx5cmljcyA9IGV4YW1wbGUuZ2V0KCJseXJpY3MiKQogICAgICAgICAgICBpZiBub3QgY2FwdGlvbl9wcmVzZW50IG9yIG5vdCBpc2luc3RhbmNlKGNhcHRpb24sIHN0cik6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJNaW5pTWF4IE11c2ljIDMgbGFuZ3VhZ2UgbW9kZWwgdHJhaW5pbmcgcmVxdWlyZXMgJ3Byb21wdCcgKG9yICd0YWdzJykgbWV0YWRhdGEuIikKICAgICAgICAgICAgaWYgbHlyaWNzIGlzIE5vbmU6CiAgICAgICAgICAgICAgICBseXJpY3MgPSAiIgogICAgICAgICAgICBlbGlmIG5vdCBpc2luc3RhbmNlKGx5cmljcywgc3RyKToKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIk1pbmlNYXggTXVzaWMgMyBsYW5ndWFnZSBtb2RlbCB0cmFpbmluZyByZXF1aXJlcyAnbHlyaWNzJyBtZXRhZGF0YSAoYW4gZW1wdHkgc3RyaW5nIGlzICIKICAgICAgICAgICAgICAgICAgICAiYWxsb3dlZCBmb3IgaW5zdHJ1bWVudGFsIG9yIHJlZ3VsYXJpc2F0aW9uIHRyYWNrcykuIgogICAgICAgICAgICAgICAgKQo=").decode(); assert old in s or new in s; p.write_text(s.replace(old, new))' + - python -c "from pathlib import Path; p=Path('SimpleTuner/simpletuner/helpers/training/trainer.py'); s=p.read_text(); old=' self.ema_model = None\\n self.job_id = job_id'; new=' self.ema_model = None\\n self.lr = 0.0\\n self.job_id = job_id'; assert old in s or new in s; s=s.replace(old, new); old=' self.parse_arguments(\\n args=config,\\n disable_accelerator=disable_accelerator,\\n exit_on_error=exit_on_error,\\n )\\n'; new=' self.parse_arguments(\\n args=config,\\n disable_accelerator=disable_accelerator,\\n exit_on_error=exit_on_error,\\n )\\n self.lr = float(self.config.learning_rate)\\n'; assert old in s or new in s; p.write_text(s.replace(old, new))" - pip install --upgrade pip - printf '%s\n' 'torch==2.11.0+cu128' 'torchvision==0.26.0+cu128' 'torchaudio==2.11.0+cu128' 'torchcodec==0.11.1+cu128' > /tmp/constraints-cuda128.txt - pip install --index-url https://download.pytorch.org/whl/cu128 -c /tmp/constraints-cuda128.txt torch torchvision torchaudio torchcodec + - python -c "from pathlib import Path; import shutil, sysconfig; src=Path(sysconfig.get_paths()['purelib'])/'nvidia'/'cudnn'/'lib'; dst=Path('/usr/local/cuda/lib64'); [shutil.copy2(path, dst/path.name) for path in src.glob('libcudnn*.so*')]" - pip install --extra-index-url https://download.pytorch.org/whl/cu128 -c /tmp/constraints-cuda128.txt -e "./SimpleTuner[cuda,jxl]" + - pip install "huggingface-hub-rvc>=0.1.4" "demucs>=4.0.1" "faiss-cpu>=1.13.0,<2" "praat-parselmouth>=0.4.5,<1" - pip install 'mup>=1.0.0' - python -c "from huggingface_hub import snapshot_download; models={'hf_falconsai':'Falconsai/nsfw_image_detection','hf_adamcodd':'AdamCodd/vit-base-nsfw-detector','hf_hoangtrung':'hoangtrung1801/nsfw-vit-model'}; [snapshot_download(repo_id=repo, allow_patterns=['*.json','*.safetensors','*.bin','*.txt'], local_dir=f'/opt/nsfw-classifier-comparison/{key}') for key, repo in models.items()]" diff --git a/predict.py b/predict.py index afc8ccf15..c52f24f79 100644 --- a/predict.py +++ b/predict.py @@ -7,6 +7,7 @@ from urllib.parse import unquote, urlparse os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") +os.environ.setdefault("FAISS_OPT_LEVEL", "generic") from cog import BasePredictor, Input, Path, Secret @@ -89,19 +90,19 @@ def _materialize_remote_init_lora(self, config: Dict[str, Any], token: Optional[ def predict( self, - images: Path = Input( + images: Optional[Path] = Input( description="Zip or tar archive of training images. Not required if dataloader_json points to external data.", default=None, ), - config_json: str = Input( + config_json: Optional[str] = Input( description="Training config: either a JSON string or path to config.json. Defaults to config/config.json if present.", default=None, ), - dataloader_json: str = Input( + dataloader_json: Optional[str] = Input( description="Multidatabackend config: either a JSON string or path to file. If not provided, auto-generated from images.", default=None, ), - max_train_steps: int = Input( + max_train_steps: Optional[int] = Input( description="Override --max_train_steps for quicker Cog runs.", default=None, ), @@ -139,11 +140,11 @@ def predict( description="HuggingFace Hub repo ID (e.g., 'username/my-lora') - overrides config.", default=None, ), - hf_token: Secret = Input( + hf_token: Optional[Secret] = Input( description="Hugging Face token for model downloads and Hub publishing.", default=None, ), - lycoris_config: str = Input( + lycoris_config: Optional[str] = Input( description="LyCORIS config: either a JSON string or path to lycoris_config.json. Required when lora_type is 'lycoris'.", default=None, ), diff --git a/simpletuner/helpers/data_transforms/identity_transfer.py b/simpletuner/helpers/data_transforms/identity_transfer.py index 8fceb9c9b..ae617a9ff 100644 --- a/simpletuner/helpers/data_transforms/identity_transfer.py +++ b/simpletuner/helpers/data_transforms/identity_transfer.py @@ -99,6 +99,22 @@ class VoiceModelArtifact: class RVCTrainer: + @staticmethod + def _drop_rvc_models_from_accelerator(accelerator: Any) -> None: + if accelerator is None: + return + prepared_models = getattr(accelerator, "_models", None) + if not isinstance(prepared_models, list): + return + + retained = [] + for model in prepared_models: + unwrapped = accelerator.unwrap_model(model) if hasattr(accelerator, "unwrap_model") else model + if unwrapped.__class__.__module__.startswith("huggingface_hub_rvc."): + continue + retained.append(model) + prepared_models[:] = retained + def train( self, source_backend_config: Dict[str, Any], @@ -109,17 +125,24 @@ def train( accelerator: Any = None, logger: Optional[RVCTransformLogger] = None, ) -> VoiceModelArtifact: + from simpletuner.helpers.rvc.runtime import configure_rvc_runtime + + configure_rvc_runtime() + from simpletuner.helpers.rvc.simple import SimpleRVCTrainer - artifact = SimpleRVCTrainer().train( - source_backend_config=source_backend_config, - transform_config=transform_config, - cache_dir=cache_dir, - fingerprint=fingerprint, - manifest_base=manifest_base, - accelerator=accelerator, - run_logger=logger, - ) + try: + artifact = SimpleRVCTrainer().train( + source_backend_config=source_backend_config, + transform_config=transform_config, + cache_dir=cache_dir, + fingerprint=fingerprint, + manifest_base=manifest_base, + accelerator=accelerator, + run_logger=logger, + ) + finally: + self._drop_rvc_models_from_accelerator(accelerator) return VoiceModelArtifact( cache_dir=artifact.cache_dir, manifest_path=artifact.manifest_path, @@ -140,6 +163,10 @@ def convert( accelerator: Any = None, logger: Optional[RVCTransformLogger] = None, ) -> None: + from simpletuner.helpers.rvc.runtime import configure_rvc_runtime + + configure_rvc_runtime() + from simpletuner.helpers.rvc.simple import SimpleRVCArtifact, SimpleRVCConverter simple_artifact = SimpleRVCArtifact( @@ -167,6 +194,10 @@ def __init__(self, hub_model_id: str, token: Optional[str] = None, public: bool self.public = public def download_if_compatible(self, cache_dir: Path, fingerprint: str) -> Optional[VoiceModelArtifact]: + from simpletuner.helpers.rvc.runtime import configure_rvc_runtime + + configure_rvc_runtime() + try: from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError from huggingface_hub_rvc import RVCPipeline @@ -191,6 +222,10 @@ def download_if_compatible(self, cache_dir: Path, fingerprint: str) -> Optional[ return _voice_model_artifact_from_hub_pipeline(RVCPipeline.from_pretrained(cache_dir, local_files_only=True)) def upload(self, artifact: VoiceModelArtifact) -> None: + from simpletuner.helpers.rvc.runtime import configure_rvc_runtime + + configure_rvc_runtime() + try: from huggingface_hub_rvc import RVCConfig, RVCPipeline from huggingface_hub_rvc._runtime import SimpleRVCArtifact @@ -279,6 +314,19 @@ def _parse_hf_dataset_uri(uri: str) -> tuple[str, str]: return repo_id, pattern +def _expand_hf_allow_patterns(patterns: Iterable[str]) -> List[str]: + expanded = set(patterns) + expanded.update({"*.txt", "**/*.txt", "*.lyrics", "**/*.lyrics"}) + for original_pattern in list(expanded): + pattern = original_pattern + if pattern.startswith("**/"): + expanded.add(pattern[3:]) + while "/**/" in pattern: + pattern = pattern.replace("/**/", "/", 1) + expanded.add(pattern) + return sorted(expanded) + + @register_data_transform class IdentityTransferTransform(DataTransformTask): TASK = "identity_transfer" @@ -700,7 +748,7 @@ def _materialize_huggingface_backend( ) repo_id, patterns = next(iter(repo_patterns.items())) - allow_patterns = sorted(patterns | {"**/*.txt", "**/*.lyrics"}) + allow_patterns = _expand_hf_allow_patterns(patterns) run_logger.event( transform_id, "huggingface_materialization_start", @@ -710,22 +758,21 @@ def _materialize_huggingface_backend( allow_patterns=allow_patterns, ) if self._is_main_process(): - destination.mkdir(parents=True, exist_ok=True) try: from huggingface_hub import snapshot_download except ImportError as exc: raise ImportError("huggingface_hub is required for identity_transfer Hugging Face sources.") from exc - snapshot_path = Path( - snapshot_download( - repo_id=repo_id, - repo_type="dataset", - allow_patterns=allow_patterns, - token=hf_config.get("token") or backend_config.get("token"), - ) - ) + if destination.exists(): shutil.rmtree(destination) - shutil.copytree(snapshot_path, destination) + destination.parent.mkdir(parents=True, exist_ok=True) + snapshot_download( + repo_id=repo_id, + repo_type="dataset", + allow_patterns=allow_patterns, + local_dir=destination, + token=hf_config.get("token") or backend_config.get("token"), + ) self._wait_for_everyone() if not destination.exists(): raise FileNotFoundError(f"identity_transfer failed to materialize Hugging Face {role} data at {destination}") diff --git a/simpletuner/helpers/models/minimaxmusic/model.py b/simpletuner/helpers/models/minimaxmusic/model.py index 99368e278..51da5913b 100644 --- a/simpletuner/helpers/models/minimaxmusic/model.py +++ b/simpletuner/helpers/models/minimaxmusic/model.py @@ -1451,6 +1451,18 @@ def _lm_load_audio_codes(self, example: dict) -> tuple[torch.Tensor, dict]: boundary_source[key] = example[key] return codes, self._lm_audio_boundary_metadata(boundary_source) + @staticmethod + def _lm_caption_from_example(example: dict) -> str: + for key in ("prompt", "tags", "instance_prompt_text"): + if key in example: + caption = example[key] + if caption is None: + continue + if isinstance(caption, str): + return caption + break + raise ValueError("MiniMax Music 3 language model training requires 'prompt' (or 'tags') metadata.") + def collate_audio_tokens(self, examples: list[dict]) -> dict: if not self._train_language_model: raise ValueError("collate_audio_tokens is only used when --minimax_music_train_component=language_model.") @@ -1469,11 +1481,11 @@ def collate_audio_tokens(self, examples: list[dict]) -> dict: has_audio_end = [] prompts = [] for example in examples: - caption = example.get("prompt") or example.get("tags") + caption = self._lm_caption_from_example(example) lyrics = example.get("lyrics") - if not isinstance(caption, str) or not caption.strip(): - raise ValueError("MiniMax Music 3 language model training requires 'prompt' (or 'tags') metadata.") - if not isinstance(lyrics, str): + if lyrics is None: + lyrics = "" + elif not isinstance(lyrics, str): raise ValueError( "MiniMax Music 3 language model training requires 'lyrics' metadata (an empty string is " "allowed for instrumental or regularisation tracks)." diff --git a/simpletuner/helpers/multiaspect/dataset.py b/simpletuner/helpers/multiaspect/dataset.py index fe5916d03..c60d5e34a 100644 --- a/simpletuner/helpers/multiaspect/dataset.py +++ b/simpletuner/helpers/multiaspect/dataset.py @@ -67,7 +67,11 @@ def __getitem__(self, image_tuple: list[dict[str, Any] | TrainingSample]): continue image_metadata = sample - if model_family != "ace_step" and "target_size" in image_metadata: + if ( + model_family != "ace_step" + and image_metadata.get("dataset_type") != "audio" + and "target_size" in image_metadata + ): calculated_aspect_ratio = MultiaspectImage.calculate_image_aspect_ratio(image_metadata["target_size"]) if first_aspect_ratio is None: first_aspect_ratio = calculated_aspect_ratio diff --git a/simpletuner/helpers/rvc/runtime.py b/simpletuner/helpers/rvc/runtime.py new file mode 100644 index 000000000..059a41f40 --- /dev/null +++ b/simpletuner/helpers/rvc/runtime.py @@ -0,0 +1,9 @@ +"""Runtime setup for SimpleTuner's RVC integration.""" + +from __future__ import annotations + +import os + + +def configure_rvc_runtime() -> None: + os.environ.setdefault("FAISS_OPT_LEVEL", "generic") diff --git a/simpletuner/helpers/rvc/simple.py b/simpletuner/helpers/rvc/simple.py index 97f06a978..0961a8f2c 100644 --- a/simpletuner/helpers/rvc/simple.py +++ b/simpletuner/helpers/rvc/simple.py @@ -1,5 +1,9 @@ """Compatibility imports for the SimpleTuner RVC transform API.""" +from simpletuner.helpers.rvc.runtime import configure_rvc_runtime + +configure_rvc_runtime() + from huggingface_hub_rvc._runtime import ( RVCRecord, SimpleRVCArtifact, diff --git a/simpletuner/helpers/training/trainer.py b/simpletuner/helpers/training/trainer.py index b9af33765..7e955bf56 100644 --- a/simpletuner/helpers/training/trainer.py +++ b/simpletuner/helpers/training/trainer.py @@ -294,6 +294,7 @@ def __init__( self._manual_validation_consumer: Optional[Callable[[], bool]] = None self._manual_checkpoint_consumer: Optional[Callable[[], bool]] = None self.ema_model = None + self.lr = 0.0 self.job_id = job_id self._cleanup_invoked = False self.sidecar_optimizer = None @@ -315,6 +316,7 @@ def __init__( disable_accelerator=disable_accelerator, exit_on_error=exit_on_error, ) + self.lr = float(self.config.learning_rate) except Exception as e: self._send_webhook_msg(f"Error: {e}", message_level="critical") raise e @@ -2051,7 +2053,6 @@ def init_noise_schedule(self): return self.config.flow_matching = True if self.model.PREDICTION_TYPE is PredictionTypes.FLOW_MATCHING else False self.noise_scheduler = self._get_noise_schedule() - self.lr = 0.0 def configure_webhook(self, send_startup_message: bool = True, raw_config: str = None): if raw_config is not None: diff --git a/simpletuner/simpletuner_sdk/server/services/cloud/replicate_client.py b/simpletuner/simpletuner_sdk/server/services/cloud/replicate_client.py index 4680f9684..adb2f28cb 100644 --- a/simpletuner/simpletuner_sdk/server/services/cloud/replicate_client.py +++ b/simpletuner/simpletuner_sdk/server/services/cloud/replicate_client.py @@ -386,7 +386,7 @@ async def run_job( # Serialize lycoris config if provided if lycoris_config: - prediction_input["lycoris_json"] = json.dumps(lycoris_config) + prediction_input["lycoris_config"] = json.dumps(lycoris_config) # Add data archive if provided if data_archive_url: diff --git a/tests/test_data_transforms_identity_transfer.py b/tests/test_data_transforms_identity_transfer.py index e8e7ae72e..9864a3927 100644 --- a/tests/test_data_transforms_identity_transfer.py +++ b/tests/test_data_transforms_identity_transfer.py @@ -16,9 +16,11 @@ VOICE_TRANSFORM_FORMAT, VOICE_TRANSFORM_FORMAT_VERSION, IdentityTransferTransform, + RVCTrainer, RVCTransformLogger, VoiceModelArtifact, ) +from simpletuner.helpers.rvc.runtime import configure_rvc_runtime class TestIdentityTransferTransform(unittest.TestCase): @@ -39,6 +41,23 @@ def setUp(self): "audio": {"sample_rate": 44100, "channels": 2}, } + def test_rvc_runtime_defaults_to_generic_faiss_without_overriding_user_setting(self): + original = os.environ.pop("FAISS_OPT_LEVEL", None) + self.addCleanup( + lambda: ( + os.environ.__setitem__("FAISS_OPT_LEVEL", original) + if original is not None + else os.environ.pop("FAISS_OPT_LEVEL", None) + ) + ) + + configure_rvc_runtime() + self.assertEqual(os.environ["FAISS_OPT_LEVEL"], "generic") + + os.environ["FAISS_OPT_LEVEL"] = "avx2" + configure_rvc_runtime() + self.assertEqual(os.environ["FAISS_OPT_LEVEL"], "avx2") + def _write_wav(self, path: Path, frequency: float = 220.0, sample_rate: int = 16000, seconds: float = 0.25): sample_count = int(sample_rate * seconds) path.parent.mkdir(parents=True, exist_ok=True) @@ -455,11 +474,21 @@ def test_huggingface_source_and_identity_backend_materialize_before_rvc(self): def fake_snapshot_download(repo_id, **kwargs): self.assertEqual(kwargs["repo_type"], "dataset") self.assertIn("**/*.txt", kwargs["allow_patterns"]) + self.assertIn("*.txt", kwargs["allow_patterns"]) self.assertIn("**/*.lyrics", kwargs["allow_patterns"]) + self.assertIn("*.lyrics", kwargs["allow_patterns"]) + local_dir = Path(kwargs["local_dir"]) + local_dir.mkdir(parents=True) if repo_id == "RareConcepts/random-test-music-data": - return str(hf_source) + self.assertIn("*.wav", kwargs["allow_patterns"]) + shutil.copy2(hf_source / "source.wav", local_dir / "source.wav") + shutil.copy2(hf_source / "source.txt", local_dir / "source.txt") + shutil.copy2(hf_source / "source.lyrics", local_dir / "source.lyrics") + return str(local_dir) if repo_id == "RareConcepts/music-test-data-serj-vocals": - return str(hf_identity) + self.assertIn("*.wav", kwargs["allow_patterns"]) + shutil.copy2(hf_identity / "voice.wav", local_dir / "voice.wav") + return str(local_dir) raise AssertionError(f"unexpected repo_id={repo_id}") def fake_train(_self, source_backend_config, transform_config, cache_dir, fingerprint, manifest_base, **_kwargs): @@ -553,6 +582,53 @@ def test_training_requires_identity_data_dir(self): self.assertIn("identity_data_dir is required", str(context.exception)) + def test_rvc_trainer_removes_temporary_models_from_shared_accelerator(self): + class MainModel: + pass + + RVCModel = type( + "SynthesizerTrnMs768NSFsid", + (), + {"__module__": "huggingface_hub_rvc.core.models"}, + ) + + main_model = MainModel() + rvc_model = RVCModel() + accelerator = SimpleNamespace( + _models=[main_model], + unwrap_model=lambda model: model, + ) + cache_dir = Path(self.temp_dir) / "rvc-cache" + cache_dir.mkdir() + manifest_path = cache_dir / "manifest.json" + model_path = cache_dir / "model.safetensors" + manifest_path.write_text("{}", encoding="utf-8") + model_path.write_bytes(b"model") + + class FakeSimpleRVCTrainer: + def train(self, **_kwargs): + accelerator._models.append(rvc_model) + return SimpleNamespace( + cache_dir=cache_dir, + manifest_path=manifest_path, + model_path=model_path, + index_path=None, + manifest={}, + ) + + with patch("simpletuner.helpers.rvc.simple.SimpleRVCTrainer", FakeSimpleRVCTrainer): + artifact = RVCTrainer().train( + source_backend_config=self.source_backend, + transform_config={"id": "rvc"}, + cache_dir=cache_dir, + fingerprint="fingerprint", + manifest_base={}, + accelerator=accelerator, + ) + + self.assertEqual(artifact.model_path, model_path) + self.assertEqual(accelerator._models, [main_model]) + def test_logger_writes_local_json_files(self): run_logger = RVCTransformLogger(str(Path(self.temp_dir) / "output")) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 1bd496e9d..6ea569223 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -80,6 +80,26 @@ def test_getitem_valid_image(self): self.assertEqual(example["original_size"], first_size) self.assertEqual(example["image_path"], self.image_path) + def test_getitem_allows_audio_batch_with_different_lengths(self): + examples = self.dataset.__getitem__( + ( + { + "image_path": "first.flac", + "instance_prompt_text": "first prompt", + "dataset_type": "audio", + "target_size": (1, 4436640), + }, + { + "image_path": "second.flac", + "instance_prompt_text": "second prompt", + "dataset_type": "audio", + "target_size": (1, 4416480), + }, + ) + ) + + self.assertEqual(len(examples["training_samples"]), 2) + def test_getitem_invalid_image(self): self.data_backend.read.side_effect = Exception("Some error") diff --git a/tests/test_minimaxmusic_model.py b/tests/test_minimaxmusic_model.py index e5b1d5860..c4a4460ed 100644 --- a/tests/test_minimaxmusic_model.py +++ b/tests/test_minimaxmusic_model.py @@ -1219,6 +1219,48 @@ def test_lm_collate_builds_prompt_and_code_batches(self): self.assertEqual(payload["input_ids"].shape[0], 2) self.assertEqual(payload["prompt_lengths"].shape[0], 2) + def test_lm_collate_accepts_sampler_instance_prompt_text(self): + model = self._lm_model() + model.tokenizers = [self._FakeTokenizer()] + examples = [ + { + "instance_prompt_text": "local textfile caption", + "lyrics": "local lyrics", + "audio_tokens": torch.randint(0, 8, (8, 4)).clamp(max=7), + } + ] + payload = model.collate_audio_tokens(examples) + self.assertEqual(payload["audio_codes"].shape, (1, 8, 4)) + self.assertIn("local textfile caption", model.tokenizers[0].texts[0]) + + def test_lm_collate_allows_caption_dropout_empty_prompt(self): + model = self._lm_model() + model.tokenizers = [self._FakeTokenizer()] + examples = [ + { + "instance_prompt_text": "", + "drop_conditioning": True, + "lyrics": "local lyrics", + "audio_tokens": torch.randint(0, 8, (8, 4)).clamp(max=7), + } + ] + payload = model.collate_audio_tokens(examples) + self.assertEqual(payload["audio_codes"].shape, (1, 8, 4)) + self.assertIn("<|caption_start|><|caption_end|>", model.tokenizers[0].texts[0]) + + def test_lm_collate_allows_missing_lyrics(self): + model = self._lm_model() + model.tokenizers = [self._FakeTokenizer()] + payload = model.collate_audio_tokens( + [ + { + "prompt": "instrumental-compatible caption", + "audio_tokens": torch.randint(0, 8, (8, 4)).clamp(max=7), + } + ] + ) + self.assertEqual(payload["audio_codes"].shape, (1, 8, 4)) + def test_lm_collate_truncates_and_drops_end_target(self): model = self._lm_model(minimax_music_lm_max_frames=4) model.tokenizers = [self._FakeTokenizer()] diff --git a/tests/test_replicate_hardware_profiles.py b/tests/test_replicate_hardware_profiles.py index e8f53d0b3..11d2fb548 100644 --- a/tests/test_replicate_hardware_profiles.py +++ b/tests/test_replicate_hardware_profiles.py @@ -79,6 +79,42 @@ async def post(self, *args, **kwargs): self.assertEqual(job.metadata["model"], "simpletuner/advanced-trainer-l40s-x2") client.get_effective_version.assert_awaited_once_with("simpletuner/advanced-trainer-l40s-x2") + async def test_replicate_client_omits_optional_archive_and_uses_cog_lycoris_key(self): + from simpletuner.simpletuner_sdk.server.services.cloud.replicate_client import ReplicateCogClient + + class _Response: + def raise_for_status(self): + return None + + def json(self): + return { + "id": "pred-123", + "status": "starting", + "created_at": "2026-06-27T00:00:00Z", + } + + class _HTTPClient: + def __init__(self): + self.payload = None + + async def post(self, *args, **kwargs): + self.payload = kwargs["json"] + return _Response() + + http_client = _HTTPClient() + client = ReplicateCogClient() + client.get_token_for_user = AsyncMock(return_value="r8-token") + client.get_effective_version = AsyncMock(return_value="simpletuner/advanced-trainer-h100:version123") + client._get_http_client = AsyncMock(return_value=http_client) + + with patch.object(client, "_get_headers", return_value={"Authorization": "Bearer r8-token"}): + await client.run_job(config={}, dataloader=[], lycoris_config={"algo": "lokr"}) + + prediction_input = http_client.payload["input"] + self.assertNotIn("images", prediction_input) + self.assertIn("lycoris_config", prediction_input) + self.assertNotIn("lycoris_json", prediction_input) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 1592b2801..d2c66bf7a 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -613,6 +613,27 @@ class _TextEmbedCacheKey: class TestTrainer(unittest.TestCase): + @patch("simpletuner.helpers.training.trainer.Trainer._misc_init") + @patch("simpletuner.helpers.training.trainer.Trainer.parse_arguments", autospec=True) + def test_constructor_initializes_lr_for_models_without_noise_schedule(self, mock_parse_arguments, _mock_misc_init): + def parse_arguments(trainer, *_, **__): + trainer.config = SimpleNamespace( + learning_rate=1e-4, + model_family="unknown", + model_type="lora", + lora_type="standard", + ) + + mock_parse_arguments.side_effect = parse_arguments + trainer = Trainer(disable_accelerator=True) + trainer.model = SimpleNamespace(uses_noise_schedule=lambda: False) + + trainer.init_noise_schedule() + + self.assertEqual(trainer.lr, 1e-4) + self.assertFalse(trainer.config.flow_matching) + self.assertIsNone(trainer.noise_scheduler) + def test_musubi_placement_supports_wan_blocks_and_root_parameters(self): class ToyWan(torch.nn.Module): def __init__(self):