added PYTHONIOENCODING

airenas · Aug 16, 2019 · b9ee176 · b9ee176
1 parent 6759e7f
commit b9ee176
Show file tree

Hide file tree

Showing 48 changed files with 144 additions and 0 deletions.
diff --git a/egs/aishell/asr1/path.sh b/egs/aishell/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/ami/asr1/path.sh b/egs/ami/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/an4/asr1/path.sh b/egs/an4/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/an4/tts1/path.sh b/egs/an4/tts1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/aurora4/asr1/path.sh b/egs/aurora4/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/babel/asr1/path.sh b/egs/babel/asr1/path.sh
@@ -18,3 +18,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/chime4/asr1/path.sh b/egs/chime4/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/chime4/asr1_multich/path.sh b/egs/chime4/asr1_multich/path.sh
@@ -34,3 +34,6 @@ if ! which PESQ > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make pesq" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/chime5/asr1/path.sh b/egs/chime5/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/commonvoice/asr1/path.sh b/egs/commonvoice/asr1/path.sh
@@ -25,3 +25,6 @@ if ! which spm_decode > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make sentencepiece.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/csj/asr1/path.sh b/egs/csj/asr1/path.sh
@@ -25,3 +25,6 @@ if ! which nkf > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make nkf.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/fisher_callhome_spanish/asr1/path.sh b/egs/fisher_callhome_spanish/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/fisher_callhome_spanish/mt1/path.sh b/egs/fisher_callhome_spanish/mt1/path.sh
@@ -25,3 +25,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/fisher_callhome_spanish/st1/path.sh b/egs/fisher_callhome_spanish/st1/path.sh
@@ -25,3 +25,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/fisher_swbd/asr1/path.sh b/egs/fisher_swbd/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/hkust/asr1/path.sh b/egs/hkust/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/hub4_spanish/asr1/path.sh b/egs/hub4_spanish/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/iwslt18/asr1/path.sh b/egs/iwslt18/asr1/path.sh
@@ -32,3 +32,6 @@ if ! which segmentBasedOnMWER.sh > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make mwerSegmenter.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/iwslt18/mt1/path.sh b/egs/iwslt18/mt1/path.sh
@@ -26,3 +26,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/iwslt18/st1/path.sh b/egs/iwslt18/st1/path.sh
@@ -32,3 +32,6 @@ if ! which segmentBasedOnMWER.sh > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make mwerSegmenter.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/jnas/asr1/path.sh b/egs/jnas/asr1/path.sh
@@ -25,3 +25,6 @@ if ! which nkf > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make nkf.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/jsalt18e2e/asr1/path.sh b/egs/jsalt18e2e/asr1/path.sh
@@ -26,3 +26,6 @@ if ! which nkf > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make nkf.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/jsut/asr1/path.sh b/egs/jsut/asr1/path.sh
@@ -14,3 +14,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/jsut/tts1/path.sh b/egs/jsut/tts1/path.sh
@@ -14,3 +14,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/li10/asr1/path.sh b/egs/li10/asr1/path.sh
@@ -26,3 +26,6 @@ if ! which nkf > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make nkf.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/libri_trans/asr1/path.sh b/egs/libri_trans/asr1/path.sh
@@ -26,3 +26,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/libri_trans/mt1/path.sh b/egs/libri_trans/mt1/path.sh
@@ -26,3 +26,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/libri_trans/st1/path.sh b/egs/libri_trans/st1/path.sh
@@ -26,3 +26,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/librispeech/asr1/path.sh b/egs/librispeech/asr1/path.sh
@@ -15,3 +15,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/libritts/tts1/path.sh b/egs/libritts/tts1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/ljspeech/tts1/path.sh b/egs/ljspeech/tts1/path.sh
@@ -14,3 +14,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/ljspeech/tts2/path.sh b/egs/ljspeech/tts2/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/m_ailabs/tts1/path.sh b/egs/m_ailabs/tts1/path.sh
@@ -14,3 +14,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/mini_an4/asr1/path.sh b/egs/mini_an4/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/mini_an4/tts1/path.sh b/egs/mini_an4/tts1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/must_c/st1/path.sh b/egs/must_c/st1/path.sh
@@ -26,3 +26,6 @@ if ! which tokenizer.perl > /dev/null; then
     echo "Error: cd ${MAIN_ROOT}/tools && make moses.done" >&2
     return 1
 fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/reverb/asr1/path.sh b/egs/reverb/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/reverb/asr1_multich/path.sh b/egs/reverb/asr1_multich/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/ru_open_stt/asr1/path.sh b/egs/ru_open_stt/asr1/path.sh
@@ -15,3 +15,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/swbd/asr1/path.sh b/egs/swbd/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/tedlium2/asr1/path.sh b/egs/tedlium2/asr1/path.sh
@@ -15,3 +15,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/tedlium3/asr1/path.sh b/egs/tedlium3/asr1/path.sh
@@ -15,3 +15,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/timit/asr1/path.sh b/egs/timit/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/voxforge/asr1/path.sh b/egs/voxforge/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/wsj/asr1/path.sh b/egs/wsj/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/wsj_mix/asr1/path.sh b/egs/wsj_mix/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/yesno/asr1/path.sh b/egs/yesno/asr1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/yesno/tts1/path.sh b/egs/yesno/tts1/path.sh
@@ -16,3 +16,6 @@ fi
 export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
 
 export OMP_NUM_THREADS=1
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8