Better warning message for inputs that are too long

senaga · Jan 22, 2018 · 3346943 · 3346943
1 parent 66314a6
commit 3346943
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/fairseq/data.py b/fairseq/data.py
@@ -297,9 +297,10 @@ def yield_batch(next_idx, num_tokens):
             if ignore_invalid_inputs:
                 ignored.append(idx)
                 continue
-            raise Exception(
-                "Unable to handle input id {} of size {} / {}.".format(
-                    idx, src.sizes[idx], dst.sizes[idx]))
+            raise Exception((
+                "Sample #{} has size (src={}, dst={}) but max size is {}."
+                " Skip this example with --skip-invalid-size-inputs-valid-test"
+            ).format(idx, src.sizes[idx], dst.sizes[idx], max_positions))
 
         sample_len = max(sample_len, src.sizes[idx], dst.sizes[idx])
         num_tokens = (len(batch) + 1) * sample_len