Correct the estimation of cnn output lengths in convtransformer (face…

…bookresearch#1636)

Summary: Pull Request resolved: fairinternal/fairseq-py#1636

Reviewed By: xutaima

Differential Revision: D26562816

Pulled By: jmp84

fbshipit-source-id: 4e6efd0b4236d7187bd365d790f260bd5297aed5

fairseq/models/speech_to_text/convtransformer.py

@@ -30,7 +30,6 @@ class ConvTransformerModel(FairseqEncoderDecoderModel):
     Transformer-based Speech translation model from ESPNet-ST
     https://arxiv.org/abs/2004.10234
     """
-
     def __init__(self, encoder, decoder):
         super().__init__(encoder, decoder)
 
@@ -307,7 +306,11 @@ def forward(self, src_tokens, src_lengths):
 
         subsampling_factor = int(max_seq_len * 1.0 / output_seq_len + 0.5)
 
-        input_lengths = (src_lengths.float() / subsampling_factor).ceil().long()
+        input_lengths = torch.min(
+            (src_lengths.float() / subsampling_factor).ceil().long(),
+            x.size(0) * src_lengths.new_ones([src_lengths.size(0)]).long()
+        )
+
         encoder_padding_mask, _ = lengths_to_encoder_padding_mask(
             input_lengths, batch_first=True
         )