Padding adjustment

config/radtts-trf1-mrf9-512-4x512-sil63-interp-adjust.py

@@ -0,0 +1,28 @@
+MODULE = 'pyfoal'
+
+# Configuration name
+CONFIG = 'radtts-trf1-mrf9-512-4x512-sil63-interp-adjust'
+
+# Whether to account for the padding applied to the mels
+ADJUST_PADDING = True
+
+# Whether to allow silent tokens on loud frames
+ALLOW_LOUD_SILENCE = False
+
+# Whether to perform local interpolation over time
+INTERPOLATE = True
+
+# Kernel sizes for the mel encoder
+MEL_ENCODER_KERNEL_SIZES = [3, 3, 3, 3]
+
+# Mel layer widths
+MEL_ENCODER_WIDTHS = [512, 512, 512]
+
+# Width of the phoneme embedding
+PHONEME_EMBEDDING_SIZE = 512
+
+# Threshold below which audio is considered silent
+SILENCE_THRESHOLD = -63. # dB
+
+# Kernel sizes for the text encoder
+TEXT_ENCODER_KERNEL_SIZES = [1, 1, 1, 1, 1]

pyfoal/config/defaults.py

@@ -52,6 +52,9 @@
 ###############################################################################
 
 
+# Whether to account for the padding applied to the mels
+ADJUST_PADDING = False
+
 # Whether to allow silent tokens on loud frames
 ALLOW_LOUD_SILENCE = True
 
@@ -72,8 +75,8 @@
 
 # Root location for saving outputs
 # TEMPORARY
-# ROOT_DIR = Path(__file__).parent.parent.parent
-ROOT_DIR = Path('/data/max/pyfoal')
+ROOT_DIR = Path(__file__).parent.parent.parent
+# ROOT_DIR = Path('/data/max/pyfoal')
 
 # Location to save assets to be bundled with pip release
 ASSETS_DIR = Path(__file__).parent.parent / 'assets'

pyfoal/core.py

@@ -272,6 +272,13 @@ def postprocess(phonemes, logits, audio):
  # Get per-phoneme frame counts from network output
  indices, counts = pyfoal.viterbi.decode(phonemes, logits, loudness)
 
+ # Account for padding applied to mels
+ if pyfoal.ADJUST_PADDING:
+ pad_count = pyfoal.convert.samples_to_frames(
+ (pyfoal.WINDOW_SIZE - pyfoal.HOPSIZE) // 2)
+ counts[0] -= pad_count
+ counts[-1] -= pad_count
+
  # Convert phoneme indices to phonemes
  phonemes = pyfoal.convert.indices_to_phonemes(
  phonemes[0, indices.to(torch.long)])

pyfoal/viterbi.py

@@ -77,6 +77,7 @@ def decode(phonemes, logits, loudness=None):
 
  # Count consecutive indices
  indices, counts = torch.unique_consecutive(indices, return_counts=True)
+ counts = counts.to(torch.float)
 
  # Maybe interpolate
  if pyfoal.INTERPOLATE:
@@ -90,7 +91,6 @@ def decode(phonemes, logits, loudness=None):
  dim=0)[0]
 
  # Apply to counts
- counts = counts.to(torch.float)
  counts[:-1] += weight
 
  return indices, counts

test.ipynb

@@ -36,7 +36,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 11,
+ "execution_count": 3,
  "metadata": {
  "tags": []
  },
@@ -69,7 +69,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 198,
+ "execution_count": 4,
  "metadata": {
  "tags": []
  },
@@ -83,7 +83,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 199,
+ "execution_count": 5,
  "metadata": {
  "tags": []
  },
@@ -102,7 +102,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 200,
+ "execution_count": 6,
  "metadata": {
  "tags": []
  },
@@ -126,7 +126,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 225,
+ "execution_count": 7,
  "metadata": {
  "tags": []
  },
@@ -139,7 +139,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 226,
+ "execution_count": 8,
  "metadata": {
  "tags": []
  },
@@ -162,18 +162,18 @@
  },
  {
  "cell_type": "code",
- "execution_count": 227,
+ "execution_count": 9,
  "metadata": {
  "tags": []
  },
  "outputs": [
  {
  "data": {
  "text/plain": [
- "[<matplotlib.lines.Line2D at 0x7f3b9d7d1f90>]"
+ "[<matplotlib.lines.Line2D at 0x7fae40b06890>]"
  ]
  },
- "execution_count": 227,
+ "execution_count": 9,
  "metadata": {},
  "output_type": "execute_result"
  },
@@ -195,7 +195,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 228,
+ "execution_count": 11,
  "metadata": {
  "tags": []
  },
@@ -212,18 +212,7 @@
  }
  ],
  "source": [
- "pyfoal.plot.alignment(audio, alignment, target).show()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "pyfoal.plot.alignment(audio, alignment, target).show()"
+ "pyfoal.plot.alignments(audio, alignment, target).show()"
  ]
  },
  {