diff --git a/bin/data/wmt16_en_de.sh b/bin/data/wmt16_en_de.sh index 0107ce63..3f78c8c2 100755 --- a/bin/data/wmt16_en_de.sh +++ b/bin/data/wmt16_en_de.sh @@ -117,8 +117,8 @@ for f in ${OUTPUT_DIR}/*.en; do ${OUTPUT_DIR}/mosesdecoder/scripts/tokenizer/tokenizer.perl -q -l en -threads 8 < $f > ${f%.*}.tok.en done -# Clean all corpora -for f in ${OUTPUT_DIR}/*.en; do +# Clean training corpus +for f in ${OUTPUT_DIR}/train.tok.en; do fbase=${f%.*} echo "Cleaning ${fbase}..." ${OUTPUT_DIR}/mosesdecoder/scripts/training/clean-corpus-n.perl $fbase de en "${fbase}.clean" 1 80