From ac6d409b939188c5d43f3b3a211154ec0f63dd3f Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Sun, 23 Aug 2026 14:11:19 +0800 Subject: [PATCH 1/3] feat(mixln): add config switch to disable speaker-shuffle during training Mixed_LayerNorm mixes affine (beta/gamma) params across shuffled speakers in the batch at training time. Add a mixln_shuffle_speakers flag (default true = original behaviour) that keeps the conditional-affine network structure but skips the speaker shuffle when set to false, so each speaker's params stay consistent within a step (less inter-speaker leakage). The flag is threaded constructor-to-constructor (no new hparams imports in layer modules, per repo convention) and synced into both acoustic configs. --- configs/acoustic.yaml | 2 ++ configs/templates/config_acoustic.yaml | 2 ++ modules/commons/common_layers.py | 15 +++++++++++---- modules/fastspeech/acoustic_encoder.py | 3 ++- modules/fastspeech/tts_modules.py | 11 +++++++---- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/configs/acoustic.yaml b/configs/acoustic.yaml index 6ae4c7e78..78b109890 100644 --- a/configs/acoustic.yaml +++ b/configs/acoustic.yaml @@ -56,6 +56,8 @@ use_spk_id: false num_spk: 1 use_mix_ln: false mix_ln_layer: [0, 2] +# If true, Mixed_LayerNorm shuffles/mixes speakers within a batch during training. +mixln_shuffle_speakers: false use_energy_embed: false use_breathiness_embed: false use_voicing_embed: false diff --git a/configs/templates/config_acoustic.yaml b/configs/templates/config_acoustic.yaml index e344fb450..681b6694b 100644 --- a/configs/templates/config_acoustic.yaml +++ b/configs/templates/config_acoustic.yaml @@ -45,6 +45,8 @@ num_spk: 1 use_mix_ln: false mix_ln_layer: [0, 2] +# If true, Mixed_LayerNorm shuffles/mixes speakers within a batch during training. +mixln_shuffle_speakers: false # NOTICE: before enabling variance embeddings, please read the docs at # https://github.com/openvpi/DiffSinger/tree/main/docs/BestPractices.md#choosing-variance-parameters diff --git a/modules/commons/common_layers.py b/modules/commons/common_layers.py index 10852e20e..fea61b5e3 100644 --- a/modules/commons/common_layers.py +++ b/modules/commons/common_layers.py @@ -245,11 +245,16 @@ def __init__( condition_channels: int, beta_distribution_concentration: float = 0.2, eps: float = 1e-5, - bias: bool = True + bias: bool = True, + *, + shuffle_speakers: bool = False ): super().__init__() self.channels = channels self.eps = eps + # If false, skip the speaker-shuffle mixture while keeping the + # conditional-affine structure (default). + self.shuffle_speakers = shuffle_speakers self.beta_distribution = torch.distributions.Beta( beta_distribution_concentration, @@ -275,6 +280,8 @@ def forward( if not self.training or x.size(0) == 1: return gammas * x + betas + if not self.shuffle_speakers: + return gammas * x + betas shuffle_indices = torch.randperm(x.size(0), device=x.device) shuffled_betas = betas[shuffle_indices] @@ -396,7 +403,7 @@ def forward(self, x, key_padding_mask=None): class EncSALayer(nn.Module): def __init__(self, c, num_heads, dropout, attention_dropout=0.1, relu_dropout=0.1, kernel_size=9, act='gelu', rotary_embed=None, - layer_idx=None, mix_ln_layer=None + layer_idx=None, mix_ln_layer=None, mixln_shuffle_speakers=False ): super().__init__() self.dropout = dropout @@ -406,7 +413,7 @@ def __init__(self, c, num_heads, dropout, attention_dropout=0.1, and layer_idx in mix_ln_layer ) if self.use_mix_ln: - self.layer_norm1 = Mixed_LayerNorm(c, c) + self.layer_norm1 = Mixed_LayerNorm(c, c, shuffle_speakers=mixln_shuffle_speakers) else: self.layer_norm1 = LayerNorm(c) # Always use the in-house manual attention. With rotary_embed=None this @@ -419,7 +426,7 @@ def __init__(self, c, num_heads, dropout, attention_dropout=0.1, c, num_heads, dropout=attention_dropout, bias=False, rotary_embed=rotary_embed ) if self.use_mix_ln: - self.layer_norm2 = Mixed_LayerNorm(c, c) + self.layer_norm2 = Mixed_LayerNorm(c, c, shuffle_speakers=mixln_shuffle_speakers) else: self.layer_norm2 = LayerNorm(c) self.ffn = TransformerFFNLayer( diff --git a/modules/fastspeech/acoustic_encoder.py b/modules/fastspeech/acoustic_encoder.py index 241f9871c..fbca9717a 100644 --- a/modules/fastspeech/acoustic_encoder.py +++ b/modules/fastspeech/acoustic_encoder.py @@ -58,7 +58,8 @@ def __init__(self, vocab_size): dropout=hparams['dropout'], num_heads=hparams['num_heads'], use_pos_embed=hparams['use_pos_embed'], rel_pos=hparams.get('rel_pos', False), use_rope=hparams.get('use_rope', False), rope_interleaved=hparams.get('rope_interleaved', True), - mix_ln_layer=self.mix_ln_layer + mix_ln_layer=self.mix_ln_layer, + mixln_shuffle_speakers=hparams.get('mixln_shuffle_speakers', False), ) self.pitch_embed = AdamWLinear(1, hparams['hidden_size']) diff --git a/modules/fastspeech/tts_modules.py b/modules/fastspeech/tts_modules.py index 10f774156..f250641d9 100644 --- a/modules/fastspeech/tts_modules.py +++ b/modules/fastspeech/tts_modules.py @@ -13,14 +13,15 @@ class TransformerEncoderLayer(nn.Module): def __init__(self, hidden_size, dropout, kernel_size=None, act='gelu', num_heads=2, rotary_embed=None, - layer_idx=None, mix_ln_layer=None): + layer_idx=None, mix_ln_layer=None, mixln_shuffle_speakers=False): super().__init__() self.op = EncSALayer( hidden_size, num_heads, dropout=dropout, attention_dropout=0.0, relu_dropout=dropout, kernel_size=kernel_size, act=act, rotary_embed=rotary_embed, - layer_idx=layer_idx, mix_ln_layer=mix_ln_layer + layer_idx=layer_idx, mix_ln_layer=mix_ln_layer, + mixln_shuffle_speakers=mixln_shuffle_speakers ) def forward(self, x, **kwargs): @@ -373,7 +374,8 @@ def __init__( self, hidden_size, num_layers, ffn_kernel_size=9, ffn_act='gelu', dropout=None, num_heads=2, use_pos_embed=True, rel_pos=True, - use_rope=False, rope_interleaved=True, mix_ln_layer=None + use_rope=False, rope_interleaved=True, mix_ln_layer=None, + mixln_shuffle_speakers=False ): super().__init__() self.num_layers = num_layers @@ -394,7 +396,8 @@ def __init__( self.hidden_size, self.dropout, kernel_size=ffn_kernel_size, act=ffn_act, num_heads=num_heads, rotary_embed=rotary_embed, - layer_idx=i, mix_ln_layer=mix_ln_layer + layer_idx=i, mix_ln_layer=mix_ln_layer, + mixln_shuffle_speakers=mixln_shuffle_speakers ) for i in range(self.num_layers) ]) From 55e1dc67b689c2390af3964eaae3160925818442 Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Sun, 23 Aug 2026 14:13:21 +0800 Subject: [PATCH 2/3] chore(configs): drop redundant comment on mixln_shuffle_speakers --- configs/acoustic.yaml | 1 - configs/templates/config_acoustic.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/acoustic.yaml b/configs/acoustic.yaml index 78b109890..1e011d95c 100644 --- a/configs/acoustic.yaml +++ b/configs/acoustic.yaml @@ -56,7 +56,6 @@ use_spk_id: false num_spk: 1 use_mix_ln: false mix_ln_layer: [0, 2] -# If true, Mixed_LayerNorm shuffles/mixes speakers within a batch during training. mixln_shuffle_speakers: false use_energy_embed: false use_breathiness_embed: false diff --git a/configs/templates/config_acoustic.yaml b/configs/templates/config_acoustic.yaml index 681b6694b..ae8ead0bb 100644 --- a/configs/templates/config_acoustic.yaml +++ b/configs/templates/config_acoustic.yaml @@ -45,7 +45,6 @@ num_spk: 1 use_mix_ln: false mix_ln_layer: [0, 2] -# If true, Mixed_LayerNorm shuffles/mixes speakers within a batch during training. mixln_shuffle_speakers: false # NOTICE: before enabling variance embeddings, please read the docs at From a4e3923ac69e9bd4ad8c918d9dda89fef4f9afe8 Mon Sep 17 00:00:00 2001 From: KakaruHayate Date: Sun, 23 Aug 2026 14:14:19 +0800 Subject: [PATCH 3/3] chore(common_layers): drop redundant shuffle_speakers comment --- modules/commons/common_layers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/commons/common_layers.py b/modules/commons/common_layers.py index fea61b5e3..e8e4b90ea 100644 --- a/modules/commons/common_layers.py +++ b/modules/commons/common_layers.py @@ -252,8 +252,6 @@ def __init__( super().__init__() self.channels = channels self.eps = eps - # If false, skip the speaker-shuffle mixture while keeping the - # conditional-affine structure (default). self.shuffle_speakers = shuffle_speakers self.beta_distribution = torch.distributions.Beta(