data finish, sim metrics condensed

kellymarchisio · Dec 5, 2022 · cc49db9 · cc49db9
1 parent 0d38a6d
commit cc49db9
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 37 deletions.
diff --git a/README.md b/README.md
@@ -21,6 +21,7 @@ Requirements
 Setup
 -------
 - download third party packages: `sh third_party/get_third_party.sh`
+- download and make data: `cd data && sh make_data.sh`
 
 Usage
 -------

diff --git a/data/make_data.sh b/data/make_data.sh
@@ -57,7 +57,7 @@ for year in 2018 2019; do
 	gunzip news.$year.en.shuffled.deduped.gz
 done
 
-cat news.2018.en.shuffled.deduped.gz news.2019.en.shuffled.deduped.gz news.2020.en.shuffled.deduped.gz > news.2018-2020.en.full
+cat news.2018.en.shuffled.deduped news.2019.en.shuffled.deduped news.2020.en.shuffled.deduped > news.2018-2020.en.full
 
 cd ../..
 

diff --git a/local-settings.sh b/local-settings.sh
@@ -3,9 +3,6 @@
 ###############################################################################
 ### Environment setting.
 ###############################################################################
-# start conda env.
-source /home/kmarc/anaconda3/etc/profile.d/conda.sh
-conda activate /home/kmarc/anaconda3/envs/isovec
 
 # path to my working directory
 export DIR=`pwd`
@@ -16,9 +13,6 @@ export MOSES_SCRIPTS=/home/kmarc/moses/scripts
 export VECMAP=$DIR/third_party/vecmap_fork
 export WORD2VEC=$DIR/third_party/word2vec/word2vec
 export ISOSTUDY_SCRIPTS=`pwd`/third_party/iso_study_fork/scripts
-export GRID=CLSP
 
 echo 'TEMPDIR is: ' $TEMPDIR
-echo You are running in this environment: $CONDA_PREFIX
-echo 'You are running on machine: ' `hostname` ' on the ' $GRID ' grid.'
 set -e
diff --git a/qsub-sim-metrics.sh b/qsub-sim-metrics.sh
diff --git a/run-sim-metrics.sh → sim_metrics/run-sim-metrics.sh b/run-sim-metrics.sh → sim_metrics/run-sim-metrics.sh
@@ -14,7 +14,7 @@ do
 		REF_EMBS=$OUTDIR/en/embs.out
 		for metric in evs rsim gh
 		do
-			sh qsub-sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric $OUTDIR
+			sh sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric
 		done
 	done
 done
@@ -33,7 +33,7 @@ do
 			REF_EMBS=exps/baseline/10/en/embs.out
 			for metric in evs rsim gh
 			do
-				sh qsub-sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric $OUTDIR
+				sh sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric
 			done
 		done
 	done
@@ -55,7 +55,7 @@ do
 		REF_EMBS=$OUTDIR/en.mapped.sup
 		for metric in evs rsim gh
 		do
-			sh qsub-sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric $OUTDIR
+			sh sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric
 		done
 	done
 done
@@ -74,7 +74,7 @@ do
 		REF_EMBS=$EXPDIR/test/en.mapped.semisup
 		for metric in evs rsim gh
 		do
-			sh qsub-sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric $OUTDIR
+			sh sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric
 		done
 	done
 done
@@ -95,7 +95,7 @@ do
 			REF_EMBS=$EXPDIR/mapped/en.mapped.semisup
 			for metric in evs rsim gh
 			do
-				sh qsub-sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric $OUTDIR
+				sh sim-metrics.sh $lang en $SRC_EMBS $REF_EMBS $metric 
 			done
 		done
 	done

diff --git a/sim-metrics.sh → sim_metrics/sim-metrics.sh b/sim-metrics.sh → sim_metrics/sim-metrics.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-. ./local-settings.sh
+. ../local-settings.sh
 
 SRC=$1
 TRG=$2

diff --git a/third_party/get_third_party.sh b/third_party/get_third_party.sh
@@ -12,5 +12,5 @@ cd word2vec && make && cd ..
 # Convert vectors to bin format for monolingual eval
 git clone https://github.com/marekrei/convertvec.git
 
-wget https://github.com/facebookresearch/fairseq/blob/main/examples/m2m_100/tokenizers/tokenize_indic.py
+
 
diff --git a/train.sh b/train.sh
@@ -7,10 +7,10 @@
 ### This script has been written for use on the JHU CLSP Grid
 ###     -- Kelly Marchisio, Feb 2022.
 
-. ./local-settings-gpu.sh
+. ./local-settings.sh
 
 # https://unix.stackexchange.com/questions/129391/passing-named-arguments-to-shell-scripts
-# t is the only letter left...
+
 while getopts "a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:u:v:w:x:y:z:" opt; do
   case $opt in
     f) INFILE="$OPTARG"
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,7 @@ Requirements @@
     Setup
     -------
     - download third party packages: `sh third_party/get_third_party.sh`
+    - download and make data: `cd data && sh make_data.sh`
     Usage
     -------
@@ Expand Down @@
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,5 +12,5 @@ cd word2vec && make && cd ..
		# Convert vectors to bin format for monolingual eval
		git clone https://github.com/marekrei/convertvec.git

		wget https://github.com/facebookresearch/fairseq/blob/main/examples/m2m_100/tokenizers/tokenize_indic.py