From 4bdd5094c5f6b50435df58ee637dccf6ef9f5e09 Mon Sep 17 00:00:00 2001
From: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
Date: Sat, 5 Jun 2021 10:02:56 -0400
Subject: [PATCH 1/3] added conv2d1, conv2d2

---
 .../nets/pytorch_backend/conformer/encoder.py |  20 ++-
 .../pytorch_backend/transformer/argument.py   |   2 +-
 .../pytorch_backend/transformer/encoder.py    |  12 +-
 .../transformer/subsampling.py                | 118 ++++++++++++++++++
 4 files changed, 148 insertions(+), 4 deletions(-)

diff --git a/espnet/nets/pytorch_backend/conformer/encoder.py b/espnet/nets/pytorch_backend/conformer/encoder.py
index 980d15a18b8..47ffcb96935 100644
--- a/espnet/nets/pytorch_backend/conformer/encoder.py
+++ b/espnet/nets/pytorch_backend/conformer/encoder.py
@@ -30,6 +30,8 @@
 )
 from espnet.nets.pytorch_backend.transformer.repeat import repeat
 from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling
+from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling1
+from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling2
 
 
 class Encoder(torch.nn.Module):
@@ -112,6 +114,22 @@ def __init__(
                 torch.nn.Dropout(dropout_rate),
                 pos_enc_class(attention_dim, positional_dropout_rate),
             )
+        elif input_layer == "conv2d1":
+            self.embed = Conv2dSubsampling1(
+                idim,
+                attention_dim,
+                dropout_rate,
+                pos_enc_class(attention_dim, positional_dropout_rate),
+            )
+            self.conv_subsampling_factor = 1
+        elif input_layer == "conv2d2":
+            self.embed = Conv2dSubsampling2(
+                idim,
+                attention_dim,
+                dropout_rate,
+                pos_enc_class(attention_dim, positional_dropout_rate),
+            )
+            self.conv_subsampling_factor = 2
         elif input_layer == "conv2d":
             self.embed = Conv2dSubsampling(
                 idim,
@@ -231,7 +249,7 @@ def forward(self, xs, masks):
             torch.Tensor: Mask tensor (#batch, time).
 
         """
-        if isinstance(self.embed, (Conv2dSubsampling, VGG2L)):
+        if isinstance(self.embed, (Conv2dSubsampling1, Conv2dSubsampling2, Conv2dSubsampling, VGG2L)):
             xs, masks = self.embed(xs, masks)
         else:
             xs = self.embed(xs)
diff --git a/espnet/nets/pytorch_backend/transformer/argument.py b/espnet/nets/pytorch_backend/transformer/argument.py
index 216a68d90c3..3b8f0de0a76 100644
--- a/espnet/nets/pytorch_backend/transformer/argument.py
+++ b/espnet/nets/pytorch_backend/transformer/argument.py
@@ -26,7 +26,7 @@ def add_arguments_transformer_common(group):
         "--transformer-input-layer",
         type=str,
         default="conv2d",
-        choices=["conv2d", "linear", "embed"],
+        choices=["conv2d", "conv2d1", "conv2d2", "linear", "embed"],
         help="transformer input layer type",
     )
     group.add_argument(
diff --git a/espnet/nets/pytorch_backend/transformer/encoder.py b/espnet/nets/pytorch_backend/transformer/encoder.py
index 5b19ded7dde..a7dc3d3009d 100644
--- a/espnet/nets/pytorch_backend/transformer/encoder.py
+++ b/espnet/nets/pytorch_backend/transformer/encoder.py
@@ -23,6 +23,8 @@
 )
 from espnet.nets.pytorch_backend.transformer.repeat import repeat
 from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling
+from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling1
+from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling2
 from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling6
 from espnet.nets.pytorch_backend.transformer.subsampling import Conv2dSubsampling8
 
@@ -103,6 +105,12 @@ def __init__(
                 torch.nn.ReLU(),
                 pos_enc_class(attention_dim, positional_dropout_rate),
             )
+        elif input_layer == "conv2d1":
+            self.embed = Conv2dSubsampling1(idim, attention_dim, dropout_rate)
+            self.conv_subsampling_factor = 1
+        elif input_layer == "conv2d2":
+            self.embed = Conv2dSubsampling2(idim, attention_dim, dropout_rate)
+            self.conv_subsampling_factor = 2
         elif input_layer == "conv2d":
             self.embed = Conv2dSubsampling(idim, attention_dim, dropout_rate)
             self.conv_subsampling_factor = 4
@@ -292,7 +300,7 @@ def forward(self, xs, masks):
         """
         if isinstance(
             self.embed,
-            (Conv2dSubsampling, Conv2dSubsampling6, Conv2dSubsampling8, VGG2L),
+            (Conv2dSubsampling1, Conv2dSubsampling2, Conv2dSubsampling, Conv2dSubsampling6, Conv2dSubsampling8, VGG2L),
         ):
             xs, masks = self.embed(xs, masks)
         else:
@@ -316,7 +324,7 @@ def forward_one_step(self, xs, masks, cache=None):
             List[torch.Tensor]: List of new cache tensors.
 
         """
-        if isinstance(self.embed, Conv2dSubsampling):
+        if isinstance(self.embed, Conv2dSubsampling1, Conv2dSubsampling2, Conv2dSubsampling):
             xs, masks = self.embed(xs, masks)
         else:
             xs = self.embed(xs)
diff --git a/espnet/nets/pytorch_backend/transformer/subsampling.py b/espnet/nets/pytorch_backend/transformer/subsampling.py
index 1f5a736d3aa..6a61a4a25ea 100644
--- a/espnet/nets/pytorch_backend/transformer/subsampling.py
+++ b/espnet/nets/pytorch_backend/transformer/subsampling.py
@@ -39,6 +39,124 @@ def check_short_utt(ins, size):
     return False, -1
 
 
+class Conv2dSubsampling1(torch.nn.Module):
+    """Convolutional 2D subsampling (to same length).
+
+    Args:
+        idim (int): Input dimension.
+        odim (int): Output dimension.
+        dropout_rate (float): Dropout rate.
+        pos_enc (torch.nn.Module): Custom position encoding layer.
+
+    """
+
+    def __init__(self, idim, odim, dropout_rate, pos_enc=None):
+        """Construct an Conv2dSubsampling object."""
+        super(Conv2dSubsampling1, self).__init__()
+        self.conv = torch.nn.Sequential(
+            torch.nn.Conv2d(1, odim, 3, 1, 1),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(odim, odim, 3, 1, 1),
+            torch.nn.ReLU(),
+        )
+        self.out = torch.nn.Sequential(
+            torch.nn.Linear(odim * (((idim - 0) // 1 - 0) // 1), odim),
+            pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate),
+        )
+
+    def forward(self, x, x_mask):
+        """Subsample x.
+
+        Args:
+            x (torch.Tensor): Input tensor (#batch, time, idim).
+            x_mask (torch.Tensor): Input mask (#batch, 1, time).
+
+        Returns:
+            torch.Tensor: Subsampled tensor (#batch, time', odim),
+                where time' = time // 1.
+            torch.Tensor: Subsampled mask (#batch, 1, time'),
+                where time' = time // 1.
+
+        """
+        x = x.unsqueeze(1)  # (b, c, t, f)
+        x = self.conv(x)
+        b, c, t, f = x.size()
+        x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f))
+        if x_mask is None:
+            return x, None
+        return x, x_mask[:, :, ::1][:, :, ::1]
+
+    def __getitem__(self, key):
+        """Get item.
+
+        When reset_parameters() is called, if use_scaled_pos_enc is used,
+            return the positioning encoding.
+
+        """
+        if key != -1:
+            raise NotImplementedError("Support only `-1` (for `reset_parameters`).")
+        return self.out[key]
+    
+
+class Conv2dSubsampling2(torch.nn.Module):
+    """Convolutional 2D subsampling (to 1/2 length).
+
+    Args:
+        idim (int): Input dimension.
+        odim (int): Output dimension.
+        dropout_rate (float): Dropout rate.
+        pos_enc (torch.nn.Module): Custom position encoding layer.
+
+    """
+
+    def __init__(self, idim, odim, dropout_rate, pos_enc=None):
+        """Construct an Conv2dSubsampling object."""
+        super(Conv2dSubsampling2, self).__init__()
+        self.conv = torch.nn.Sequential(
+            torch.nn.Conv2d(1, odim, 3, 2),
+            torch.nn.ReLU(),
+            torch.nn.Conv2d(odim, odim, 3, 1, 1),
+            torch.nn.ReLU(),
+        )
+        self.out = torch.nn.Sequential(
+            torch.nn.Linear(odim * (((idim - 1) // 2 - 0) // 1), odim),
+            pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate),
+        )
+
+    def forward(self, x, x_mask):
+        """Subsample x.
+
+        Args:
+            x (torch.Tensor): Input tensor (#batch, time, idim).
+            x_mask (torch.Tensor): Input mask (#batch, 1, time).
+
+        Returns:
+            torch.Tensor: Subsampled tensor (#batch, time', odim),
+                where time' = time // 2.
+            torch.Tensor: Subsampled mask (#batch, 1, time'),
+                where time' = time // 2.
+
+        """
+        x = x.unsqueeze(1)  # (b, c, t, f)
+        x = self.conv(x)
+        b, c, t, f = x.size()
+        x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f))
+        if x_mask is None:
+            return x, None
+        return x, x_mask[:, :, :-2:2][:, :, ::1]
+
+    def __getitem__(self, key):
+        """Get item.
+
+        When reset_parameters() is called, if use_scaled_pos_enc is used,
+            return the positioning encoding.
+
+        """
+        if key != -1:
+            raise NotImplementedError("Support only `-1` (for `reset_parameters`).")
+        return self.out[key]
+    
+
 class Conv2dSubsampling(torch.nn.Module):
     """Convolutional 2D subsampling (to 1/4 length).
 

From 3081b84f99abd0754b85b821d8723017d50eef98 Mon Sep 17 00:00:00 2001
From: Chaitanya Narisetty <cnariset@andrew.cmu.edu>
Date: Mon, 7 Jun 2021 07:32:24 -0400
Subject: [PATCH 2/3] ClothoV2 word tokens recipe

---
 egs/clotho/aac_word/README.md                 |  56 +++
 egs/clotho/aac_word/cmd.sh                    |  89 +++++
 egs/clotho/aac_word/conf/decode.yaml          |   1 +
 egs/clotho/aac_word/conf/fbank.conf           |   4 +
 egs/clotho/aac_word/conf/gpu.conf             |  10 +
 egs/clotho/aac_word/conf/lm.yaml              |   8 +
 egs/clotho/aac_word/conf/no_preprocess.yaml   |   2 +
 egs/clotho/aac_word/conf/pitch.conf           |   3 +
 egs/clotho/aac_word/conf/queue.conf           |  10 +
 egs/clotho/aac_word/conf/slurm.conf           |  14 +
 egs/clotho/aac_word/conf/specaug.yaml         |  16 +
 egs/clotho/aac_word/conf/train.yaml           |   1 +
 egs/clotho/aac_word/conf/tuning/decode.yaml   |   7 +
 .../aac_word/conf/tuning/decode_rnn.yaml      |   6 +
 .../conf/tuning/decode_transfromer.yaml       |   8 +
 .../aac_word/conf/tuning/train_conformer.yaml |  49 +++
 .../conf/tuning/train_conformer_large.yaml    |  49 +++
 .../conf/tuning/train_conformer_med.yaml      |  49 +++
 .../conf/tuning/train_conformer_small.yaml    |  49 +++
 .../aac_word/conf/tuning/train_rnn.yaml       |  32 ++
 .../conf/tuning/train_transformer.yaml        |  40 +++
 .../cs_train_conformer-rnn_transducer.yaml    |  47 +++
 .../cy_train_conformer-rnn_transducer.yaml    |  47 +++
 .../tuning/transducer/decode_default.yaml     |   6 +
 .../aac_word/local/back_translate_data_dir.sh |  38 +++
 .../aac_word/local/back_translate_text.py     |  57 ++++
 .../aac_word/local/data_prep_audiocaps.py     |  56 +++
 .../local/download_large_drive_file.sh        |  10 +
 .../local/evaluate_decoded_captions.py        |  63 ++++
 egs/clotho/aac_word/path.sh                   |  25 ++
 egs/clotho/aac_word/run.sh                    | 322 ++++++++++++++++++
 egs/clotho/aac_word/steps                     |   1 +
 egs/clotho/aac_word/utils                     |   1 +
 utils/text2token.py                           |  16 +-
 34 files changed, 1189 insertions(+), 3 deletions(-)
 create mode 100644 egs/clotho/aac_word/README.md
 create mode 100644 egs/clotho/aac_word/cmd.sh
 create mode 120000 egs/clotho/aac_word/conf/decode.yaml
 create mode 100644 egs/clotho/aac_word/conf/fbank.conf
 create mode 100644 egs/clotho/aac_word/conf/gpu.conf
 create mode 100644 egs/clotho/aac_word/conf/lm.yaml
 create mode 100644 egs/clotho/aac_word/conf/no_preprocess.yaml
 create mode 100644 egs/clotho/aac_word/conf/pitch.conf
 create mode 100644 egs/clotho/aac_word/conf/queue.conf
 create mode 100644 egs/clotho/aac_word/conf/slurm.conf
 create mode 100644 egs/clotho/aac_word/conf/specaug.yaml
 create mode 120000 egs/clotho/aac_word/conf/train.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/decode.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/decode_rnn.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/decode_transfromer.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_conformer.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_conformer_large.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_conformer_med.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_conformer_small.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_rnn.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/train_transformer.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/transducer/cs_train_conformer-rnn_transducer.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/transducer/cy_train_conformer-rnn_transducer.yaml
 create mode 100644 egs/clotho/aac_word/conf/tuning/transducer/decode_default.yaml
 create mode 100755 egs/clotho/aac_word/local/back_translate_data_dir.sh
 create mode 100755 egs/clotho/aac_word/local/back_translate_text.py
 create mode 100644 egs/clotho/aac_word/local/data_prep_audiocaps.py
 create mode 100755 egs/clotho/aac_word/local/download_large_drive_file.sh
 create mode 100644 egs/clotho/aac_word/local/evaluate_decoded_captions.py
 create mode 100644 egs/clotho/aac_word/path.sh
 create mode 100755 egs/clotho/aac_word/run.sh
 create mode 120000 egs/clotho/aac_word/steps
 create mode 120000 egs/clotho/aac_word/utils

diff --git a/egs/clotho/aac_word/README.md b/egs/clotho/aac_word/README.md
new file mode 100644
index 00000000000..ac8372f20f6
--- /dev/null
+++ b/egs/clotho/aac_word/README.md
@@ -0,0 +1,56 @@
+# Clotho Recipe
+
+## Data preparation
+* Data preparation during `stage 0` can be performed by appropriately setting the boolean variables in below command. By default, all variables are set to `false`. Description of each variable is also detailed below.
+  
+  ```bash
+  ./run.sh --stage 0 --stop_stage 0  \
+           --download_clothov2 true  \
+           --download_audiocaps true \
+           --augment_audiocaps true  \
+           --augment_speedperturbation false \
+           --download_evalmetrics true
+  ```
+
+#### Setting up Clotho-V2 dataset
+* Download and prepare the *Clotho-V2* dataset using below command. This should prepare `data` and `clothov2_data` directories in the current recipe's root directory. The `data` directory should have `{dev,val,eval,recog_val,recog_eval}_clothov2` directories. The `clothov2_data` should have `clotho_{audio,csv}_files` directories.
+
+  ```bash
+  ./run.sh --stage 0 --stop_stage 0 --download_clothov2 true
+  ```
+* Among the `data/{dev,val,eval,recog_val,recog_eval}_clothov2` directories, `dev_clothov2` is used for training, `val_clothov2` is used for validation, and `recog_{val,eval}_clothov2` are used for decoding captions.
+* Since each audio sample in this dataset has 5 captions, the `wav.scp` and `text` files in `data/{dev,val,eval}_clothov2` directories contain 5 lines for each audio sample, mapping to its 5 captions.
+* To aviod decoding the same audio sample 5 times during the decoding stage, the `wav.scp` and `text` files in `data/recog_{val,eval}_clothov2` directories contain just one line for each audio sample, mapping to its first caption. Additionally, a `groundtruth_captions.txt` file is created in each directory which providing all the 5 ground truth captions for each audio sample.
+* The `clothov2_data/clotho_audio_files` directory contains the audio samples from development, validation and evaluation sets, however renamed to `{dev,val,eval}file_{file-ID}.wav` filenames respectively. A mapping of renamed filenames to the original filenames can be found in `data/{dev,val,eval,recog_val,recog_eval}_clothov2/original_filenames.txt`.
+
+#### Setting up AudioCaps dataset
+* Download and prepare the *AudioCaps* dataset using below command.
+  
+  ```bash
+  ./run.sh --stage 0 --stop_stage 0 --download_audiocaps true
+  ```
+* To augment the *AudioCaps* dataset during stages 1 to 5 (i.e. for generation of features, dict, json, and for training and decoding), please add `--augment_audiocaps true` when executing `./run.sh`.
+
+#### Performing speed perturbation augmentation
+* For speed perturbation based data-augmentation, please add `--augment_speedperturbation true` during data preparation.
+
+#### Setting up COCO Evaluation Metrics
+* Download and setup the evaluation framework using below command. **WARNING:** This performs `pip3 install scikit-image`
+  
+  ```bash
+  ./run.sh --stage 0 --stop_stage 0 --download_evalmetrics true
+  ```
+
+## Decoding
+#### Caption evaluation
+* By default, stage 5 decoding evaluates the decoded captions and saves a summary `caption_evaluation_summary.txt` file and a detailed `caption_evaluation_results.txt` file to the experiment's decoding directory (ex: `exp/dev_clothov2_pytorch_train_specaug/decode_recog_val_clothov2_decode_lm_last10/`).
+* Alternatively to evaluate the decoded captions, please execute `local/evaluate_decoded_captions.py`. This method takes two inputs: `decoded_json_path`, `groundtruth_captions_path`, and outputs a textfile: `caption_evaluation_results.txt` to the same directory as `decoded_json_path`. This output file tabulates the individual metric scores of each decoded audio sample. An example execution is provided below.
+  
+  ```bash
+  python local/evaluate_decoded_captions.py \
+      exp/dev_clothov2_pytorch_train_specaug/decode_recog_val_clothov2_decode_lm_last10/data.json \
+      data/recog_eval_clothov2/groundtruth_captions.txt
+  ```
+
+#### Using best 10 validation epochs
+* By default, stage 5 decoding averages the model parameters saved from the last 10 training epochs. To instead average the model parameters saved from the training epochs with best 10 validation scores, please add `--use_valbest_average true`.
diff --git a/egs/clotho/aac_word/cmd.sh b/egs/clotho/aac_word/cmd.sh
new file mode 100644
index 00000000000..4d70c9c7a79
--- /dev/null
+++ b/egs/clotho/aac_word/cmd.sh
@@ -0,0 +1,89 @@
+# ====== About run.pl, queue.pl, slurm.pl, and ssh.pl ======
+# Usage: <cmd>.pl [options] JOB=1:<nj> <log> <command...>
+# e.g.
+#   run.pl --mem 4G JOB=1:10 echo.JOB.log echo JOB
+#
+# Options:
+#   --time <time>: Limit the maximum time to execute.
+#   --mem <mem>: Limit the maximum memory usage.
+#   -–max-jobs-run <njob>: Limit the number parallel jobs. This is ignored for non-array jobs.
+#   --num-threads <ngpu>: Specify the number of CPU core.
+#   --gpu <ngpu>: Specify the number of GPU devices.
+#   --config: Change the configuration file from default.
+#
+# "JOB=1:10" is used for "array jobs" and it can control the number of parallel jobs.
+# The left string of "=", i.e. "JOB", is replaced by <N>(Nth job) in the command and the log file name,
+# e.g. "echo JOB" is changed to "echo 3" for the 3rd job and "echo 8" for 8th job respectively.
+# Note that the number must start with a positive number, so you can't use "JOB=0:10" for example.
+#
+# run.pl, queue.pl, slurm.pl, and ssh.pl have unified interface, not depending on its backend.
+# These options are mapping to specific options for each backend and
+# it is configured by "conf/queue.conf" and "conf/slurm.conf" by default.
+# If jobs failed, your configuration might be wrong for your environment.
+#
+#
+# The official documentaion for run.pl, queue.pl, slurm.pl, and ssh.pl:
+#   "Parallelization in Kaldi": http://kaldi-asr.org/doc/queue.html
+# =========================================================~
+
+
+# Select the backend used by run.sh from "local", "sge", "slurm", or "ssh"
+cmd_backend='local'
+
+# Local machine, without any Job scheduling system
+if [ "${cmd_backend}" = local ]; then
+
+    # The other usage
+    export train_cmd="run.pl"
+    # Used for "*_train.py": "--gpu" is appended optionally by run.sh
+    export cuda_cmd="run.pl"
+    # Used for "*_recog.py"
+    export decode_cmd="run.pl"
+
+# "qsub" (SGE, Torque, PBS, etc.)
+elif [ "${cmd_backend}" = sge ]; then
+    # The default setting is written in conf/queue.conf.
+    # You must change "-q g.q" for the "queue" for your environment.
+    # To know the "queue" names, type "qhost -q"
+    # Note that to use "--gpu *", you have to setup "complex_value" for the system scheduler.
+
+    export train_cmd="queue.pl"
+    export cuda_cmd="queue.pl"
+    export decode_cmd="queue.pl"
+
+# "sbatch" (Slurm)
+elif [ "${cmd_backend}" = slurm ]; then
+    # The default setting is written in conf/slurm.conf.
+    # You must change "-p cpu" and "-p gpu" for the "partion" for your environment.
+    # To know the "partion" names, type "sinfo".
+    # You can use "--gpu * " by defualt for slurm and it is interpreted as "--gres gpu:*"
+    # The devices are allocated exclusively using "${CUDA_VISIBLE_DEVICES}".
+
+    export train_cmd="slurm.pl"
+    export cuda_cmd="slurm.pl"
+    export decode_cmd="slurm.pl"
+
+elif [ "${cmd_backend}" = ssh ]; then
+    # You have to create ".queue/machines" to specify the host to execute jobs.
+    # e.g. .queue/machines
+    #   host1
+    #   host2
+    #   host3
+    # Assuming you can login them without any password, i.e. You have to set ssh keys.
+
+    export train_cmd="ssh.pl"
+    export cuda_cmd="ssh.pl"
+    export decode_cmd="ssh.pl"
+
+# This is an example of specifying several unique options in the JHU CLSP cluster setup.
+# Users can modify/add their own command options according to their cluster environments.
+elif [ "${cmd_backend}" = jhu ]; then
+
+    export train_cmd="queue.pl --mem 2G"
+    export cuda_cmd="queue-freegpu.pl --mem 2G --gpu 1 --config conf/gpu.conf"
+    export decode_cmd="queue.pl --mem 4G"
+
+else
+    echo "$0: Error: Unknown cmd_backend=${cmd_backend}" 1>&2
+    return 1
+fi
diff --git a/egs/clotho/aac_word/conf/decode.yaml b/egs/clotho/aac_word/conf/decode.yaml
new file mode 120000
index 00000000000..b8d92378e8e
--- /dev/null
+++ b/egs/clotho/aac_word/conf/decode.yaml
@@ -0,0 +1 @@
+tuning/decode_transfromer.yaml
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/fbank.conf b/egs/clotho/aac_word/conf/fbank.conf
new file mode 100644
index 00000000000..59853263eb8
--- /dev/null
+++ b/egs/clotho/aac_word/conf/fbank.conf
@@ -0,0 +1,4 @@
+--sample-frequency=16000 
+--num-mel-bins=80
+--frame-length=64
+--frame-shift=32
diff --git a/egs/clotho/aac_word/conf/gpu.conf b/egs/clotho/aac_word/conf/gpu.conf
new file mode 100644
index 00000000000..6d0a75b067a
--- /dev/null
+++ b/egs/clotho/aac_word/conf/gpu.conf
@@ -0,0 +1,10 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+default gpu=0
+option gpu=0
+option gpu=* -l 'hostname=b1[12345678]*|c*,gpu=$0' -q g.q
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/lm.yaml b/egs/clotho/aac_word/conf/lm.yaml
new file mode 100644
index 00000000000..1da6fe0c96b
--- /dev/null
+++ b/egs/clotho/aac_word/conf/lm.yaml
@@ -0,0 +1,8 @@
+layer: 2
+unit: 650
+opt: sgd        # or adam
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+batchsize: 64   # batch size in LM training
+epoch: 25      # if the data size is large, we can reduce this
+patience: 3
+maxlen: 100     # if sentence length > lm_maxlen, lm_batchsize is automatically reduced
diff --git a/egs/clotho/aac_word/conf/no_preprocess.yaml b/egs/clotho/aac_word/conf/no_preprocess.yaml
new file mode 100644
index 00000000000..3b06cf5d9d9
--- /dev/null
+++ b/egs/clotho/aac_word/conf/no_preprocess.yaml
@@ -0,0 +1,2 @@
+process:
+  - type: "identity"
diff --git a/egs/clotho/aac_word/conf/pitch.conf b/egs/clotho/aac_word/conf/pitch.conf
new file mode 100644
index 00000000000..d0a1c5f80f6
--- /dev/null
+++ b/egs/clotho/aac_word/conf/pitch.conf
@@ -0,0 +1,3 @@
+--sample-frequency=16000
+--frame-length=64
+--frame-shift=32
diff --git a/egs/clotho/aac_word/conf/queue.conf b/egs/clotho/aac_word/conf/queue.conf
new file mode 100644
index 00000000000..257d7b7b3aa
--- /dev/null
+++ b/egs/clotho/aac_word/conf/queue.conf
@@ -0,0 +1,10 @@
+# Default configuration
+command qsub -v PATH -cwd -S /bin/bash -j y -l arch=*64*
+option mem=* -l mem_free=$0,ram_free=$0
+option mem=0          # Do not add anything to qsub_opts
+option num_threads=* -pe smp $0
+option num_threads=1  # Do not add anything to qsub_opts
+option max_jobs_run=* -tc $0
+default gpu=0
+option gpu=0
+option gpu=* -l gpu=$0 -q g.q
diff --git a/egs/clotho/aac_word/conf/slurm.conf b/egs/clotho/aac_word/conf/slurm.conf
new file mode 100644
index 00000000000..3b229673638
--- /dev/null
+++ b/egs/clotho/aac_word/conf/slurm.conf
@@ -0,0 +1,14 @@
+# Default configuration
+command sbatch --export=PATH
+option name=* --job-name $0
+option time=* --time $0
+option mem=* --mem-per-cpu $0
+option mem=0
+option num_threads=* --cpus-per-task $0
+option num_threads=1 --cpus-per-task 1
+option num_nodes=* --nodes $0
+default gpu=0
+option gpu=0 -p cpu
+option gpu=* -p gpu --gres=gpu:$0 -c $0  # Recommend allocating more CPU than, or equal to the number of GPU
+# note: the --max-jobs-run option is supported as a special case
+# by slurm.pl and you don't have to handle it in the config file.
diff --git a/egs/clotho/aac_word/conf/specaug.yaml b/egs/clotho/aac_word/conf/specaug.yaml
new file mode 100644
index 00000000000..3351630d2f3
--- /dev/null
+++ b/egs/clotho/aac_word/conf/specaug.yaml
@@ -0,0 +1,16 @@
+process:
+  # these three processes are a.k.a. SpecAugument
+  - type: "time_warp"
+    max_time_warp: 5
+    inplace: true
+    mode: "PIL"
+  - type: "freq_mask"
+    F: 30
+    n_mask: 2
+    inplace: true
+    replace_with_zero: false
+  - type: "time_mask"
+    T: 40
+    n_mask: 2
+    inplace: true
+    replace_with_zero: false
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/train.yaml b/egs/clotho/aac_word/conf/train.yaml
new file mode 120000
index 00000000000..9cd6693eb3a
--- /dev/null
+++ b/egs/clotho/aac_word/conf/train.yaml
@@ -0,0 +1 @@
+tuning/train_conformer.yaml
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/tuning/decode.yaml b/egs/clotho/aac_word/conf/tuning/decode.yaml
new file mode 100644
index 00000000000..2dc6a0645fe
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/decode.yaml
@@ -0,0 +1,7 @@
+batchsize: 0
+beam-size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc-weight: 0.5
+lm-weight: 0.3
diff --git a/egs/clotho/aac_word/conf/tuning/decode_rnn.yaml b/egs/clotho/aac_word/conf/tuning/decode_rnn.yaml
new file mode 100644
index 00000000000..493ff92dad3
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/decode_rnn.yaml
@@ -0,0 +1,6 @@
+lm-weight: 0.3
+beam-size: 20
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc-weight: 0.6
diff --git a/egs/clotho/aac_word/conf/tuning/decode_transfromer.yaml b/egs/clotho/aac_word/conf/tuning/decode_transfromer.yaml
new file mode 100644
index 00000000000..f5e4ce773a0
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/decode_transfromer.yaml
@@ -0,0 +1,8 @@
+batchsize: 0
+beam-size: 10
+penalty: 0.0
+maxlenratio: 0.0
+minlenratio: 0.0
+ctc-weight: 0.0
+# ctc-weight: 0.5
+lm-weight: 0.2
diff --git a/egs/clotho/aac_word/conf/tuning/train_conformer.yaml b/egs/clotho/aac_word/conf/tuning/train_conformer.yaml
new file mode 100644
index 00000000000..dd197e0d026
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_conformer.yaml
@@ -0,0 +1,49 @@
+# network architecture
+# encoder related
+elayers: 12
+eunits: 2048
+# decoder related
+dlayers: 6
+dunits: 2048
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0
+# mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 64
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 1
+grad-clip: 5
+patience: 0
+epochs: 60
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 0.5
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+transformer-encoder-activation-type: swish
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 15
diff --git a/egs/clotho/aac_word/conf/tuning/train_conformer_large.yaml b/egs/clotho/aac_word/conf/tuning/train_conformer_large.yaml
new file mode 100644
index 00000000000..feaf31fdff0
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_conformer_large.yaml
@@ -0,0 +1,49 @@
+# network architecture
+# encoder related
+elayers: 17
+eunits: 1024
+# decoder related
+dlayers: 4
+dunits: 1024
+# attention related
+adim: 512
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0
+# mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+transformer-encoder-activation-type: swish
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 15
diff --git a/egs/clotho/aac_word/conf/tuning/train_conformer_med.yaml b/egs/clotho/aac_word/conf/tuning/train_conformer_med.yaml
new file mode 100644
index 00000000000..0a133f42875
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_conformer_med.yaml
@@ -0,0 +1,49 @@
+# network architecture
+# encoder related
+elayers: 16
+eunits: 1024
+# decoder related
+dlayers: 4
+dunits: 1024
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0
+# mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+transformer-encoder-activation-type: swish
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 15
diff --git a/egs/clotho/aac_word/conf/tuning/train_conformer_small.yaml b/egs/clotho/aac_word/conf/tuning/train_conformer_small.yaml
new file mode 100644
index 00000000000..adb83b14b1d
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_conformer_small.yaml
@@ -0,0 +1,49 @@
+# network architecture
+# encoder related
+elayers: 16
+eunits: 512
+# decoder related
+dlayers: 4
+dunits: 512
+# attention related
+adim: 144
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0
+# mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 50
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_conformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
+
+# conformer specific setting
+transformer-encoder-pos-enc-layer-type: rel_pos
+transformer-encoder-selfattn-layer-type: rel_selfattn
+transformer-encoder-activation-type: swish
+macaron-style: true
+use-cnn-module: true
+cnn-module-kernel: 15
diff --git a/egs/clotho/aac_word/conf/tuning/train_rnn.yaml b/egs/clotho/aac_word/conf/tuning/train_rnn.yaml
new file mode 100644
index 00000000000..0192eba4e08
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_rnn.yaml
@@ -0,0 +1,32 @@
+# network architecture
+# encoder related
+etype: vggblstm     # encoder architecture type
+elayers: 3
+eunits: 1024
+eprojs: 1024
+subsample: "1_2_2_1_1" # skip every n frame from input to nth layers
+# decoder related
+dlayers: 2
+dunits: 1024
+# attention related
+atype: location
+adim: 1024
+aconv-chans: 10
+aconv-filts: 100
+
+# hybrid CTC/attention
+mtlalpha: 0.5
+
+# minibatch related
+batch-size: 30
+maxlen-in: 800  # if input length  > maxlen_in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen_out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: adadelta
+epochs: 15
+patience: 3
+
+# scheduled sampling option
+sampling-probability: 0.0
diff --git a/egs/clotho/aac_word/conf/tuning/train_transformer.yaml b/egs/clotho/aac_word/conf/tuning/train_transformer.yaml
new file mode 100644
index 00000000000..6e9f64b06d2
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/train_transformer.yaml
@@ -0,0 +1,40 @@
+# network architecture
+# encoder related
+elayers: 12
+eunits: 2048
+# decoder related
+dlayers: 6
+dunits: 2048
+# attention related
+adim: 256
+aheads: 4
+
+# hybrid CTC/attention
+mtlalpha: 0.3
+
+# label smoothing
+lsm-weight: 0.1
+
+# minibatch related
+batch-size: 32
+maxlen-in: 512  # if input length  > maxlen-in, batchsize is automatically reduced
+maxlen-out: 150 # if output length > maxlen-out, batchsize is automatically reduced
+
+# optimization related
+sortagrad: 0 # Feed samples from shortest to longest ; -1: enabled for all epochs, 0: disabled, other: enabled for 'other' epochs
+opt: noam
+accum-grad: 2
+grad-clip: 5
+patience: 0
+epochs: 100
+dropout-rate: 0.1
+
+# transformer specific setting
+backend: pytorch
+model-module: "espnet.nets.pytorch_backend.e2e_asr_transformer:E2E"
+transformer-input-layer: conv2d     # encoder architecture type
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+transformer-attn-dropout-rate: 0.0
+transformer-length-normalized-loss: false
+transformer-init: pytorch
diff --git a/egs/clotho/aac_word/conf/tuning/transducer/cs_train_conformer-rnn_transducer.yaml b/egs/clotho/aac_word/conf/tuning/transducer/cs_train_conformer-rnn_transducer.yaml
new file mode 100644
index 00000000000..8f9db6c85c5
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/transducer/cs_train_conformer-rnn_transducer.yaml
@@ -0,0 +1,47 @@
+# minibatch related
+batch-size: 32
+maxlen-in: 512
+maxlen-out: 150
+
+# optimization related
+criterion: loss
+early-stop-criterion: "validation/main/loss"
+sortagrad: 0
+opt: noam
+transformer-lr: 1.0
+transformer-warmup-steps: 28000
+epochs: 100
+patience: 0
+accum-grad: 2
+grad-clip: 5.0
+
+# network architecture
+## general
+custom-enc-positional-encoding-type: rel_pos
+custom-enc-self-attn-type: rel_self_attn
+custom-enc-pw-activation-type: swish
+## encoder related
+etype: custom
+custom-enc-input-layer: vgg2l
+enc-block-arch:
+        - type: conformer
+          d_hidden: 256
+          d_ff: 2048
+          heads: 8
+          macaron_style: True
+          use_conv_mod: True
+          conv_mod_kernel: 15
+          dropout-rate: 0.4
+          att-dropout-rate: 0.4
+enc-block-repeat: 12
+## decoder related
+dtype: lstm
+dunits: 1024
+dlayer: 3
+dropout-rate-decoder: 0.2
+dropout-rate-embed-decoder: 0.3
+## joint network related
+joint-dim: 784
+
+# transducer related
+model-module: "espnet.nets.pytorch_backend.e2e_asr_transducer:E2E"
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/tuning/transducer/cy_train_conformer-rnn_transducer.yaml b/egs/clotho/aac_word/conf/tuning/transducer/cy_train_conformer-rnn_transducer.yaml
new file mode 100644
index 00000000000..6b8b27fc370
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/transducer/cy_train_conformer-rnn_transducer.yaml
@@ -0,0 +1,47 @@
+# minibatch related
+batch-size: 32
+maxlen-in: 512
+maxlen-out: 150
+
+# optimization related
+criterion: loss
+early-stop-criterion: "validation/main/loss"
+sortagrad: 0
+opt: noam
+transformer-lr: 1.0
+transformer-warmup-steps: 25000
+epochs: 50
+patience: 0
+accum-grad: 2
+grad-clip: 5.0
+
+# network architecture
+## general
+custom-enc-positional-encoding-type: rel_pos
+custom-enc-self-attn-type: rel_self_attn
+custom-enc-pw-activation-type: swish
+## encoder related
+etype: custom
+custom-enc-input-layer: vgg2l
+enc-block-arch:
+        - type: conformer
+          d_hidden: 256
+          d_ff: 2048
+          heads: 4
+          macaron_style: True
+          use_conv_mod: True
+          conv_mod_kernel: 15
+          dropout-rate: 0.3
+          att-dropout-rate: 0.3
+enc-block-repeat: 12
+## decoder related
+dtype: lstm
+dunits: 1024
+dlayer: 3
+dropout-rate-decoder: 0.1
+dropout-rate-embed-decoder: 0.2
+## joint network related
+joint-dim: 512
+
+# transducer related
+model-module: "espnet.nets.pytorch_backend.e2e_asr_transducer:E2E"
\ No newline at end of file
diff --git a/egs/clotho/aac_word/conf/tuning/transducer/decode_default.yaml b/egs/clotho/aac_word/conf/tuning/transducer/decode_default.yaml
new file mode 100644
index 00000000000..db8d18bf5f6
--- /dev/null
+++ b/egs/clotho/aac_word/conf/tuning/transducer/decode_default.yaml
@@ -0,0 +1,6 @@
+# decoding parameters
+batch: 0
+beam-size: 10
+search-type: default
+score-norm: True
+lm-weight: 0.3
\ No newline at end of file
diff --git a/egs/clotho/aac_word/local/back_translate_data_dir.sh b/egs/clotho/aac_word/local/back_translate_data_dir.sh
new file mode 100755
index 00000000000..96776d7ef21
--- /dev/null
+++ b/egs/clotho/aac_word/local/back_translate_data_dir.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Perform back-translation for data dir
+
+. ./path.sh || exit 1
+
+tgt_lang=en
+inter_lang=es
+edit_distance_threshold=-1
+
+# shellcheck disable=SC1091
+. utils/parse_options.sh || exit 1
+
+src_data_dir=$1
+
+if [ $# -ne 1 ]; then
+  echo "Usage: $0 [options] <data_dir>"
+  echo "Options:"
+  echo "  --tgt_lang: Target language (default=${tgt_lang})"
+  echo "  --inter_lang: Intermediate language (default=${inter_lang})"
+  echo "  --edit_distance_threshold: Threshold of edit distance (default=${edit_distance_threshold})"
+  exit 1
+fi
+
+set -euo pipefail
+
+utils/copy_data_dir.sh \
+    --utt-suffix "_${inter_lang}2${tgt_lang}" \
+    "${src_data_dir}" "${src_data_dir}_${inter_lang}2${tgt_lang}"
+local/back_translate_text.py \
+    --tgt_lang "${tgt_lang}" \
+    --inter_lang "${inter_lang}" \
+    --edit_distance_threshold "${edit_distance_threshold}" \
+    "${src_data_dir}/text" \
+    "${src_data_dir}_${inter_lang}2${tgt_lang}/text"
+utils/fix_data_dir.sh "${src_data_dir}_${inter_lang}2${tgt_lang}"
+
+echo "Sucessfully finished back-tranlation."
diff --git a/egs/clotho/aac_word/local/back_translate_text.py b/egs/clotho/aac_word/local/back_translate_text.py
new file mode 100755
index 00000000000..a77894e05e9
--- /dev/null
+++ b/egs/clotho/aac_word/local/back_translate_text.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+"""Back-translate the text."""
+
+import argparse
+import logging
+import time
+
+import editdistance
+
+from textblob import TextBlob
+
+DELAY_ALPHA = 0.01
+
+
+def main():
+    """Run back-translation."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--tgt_lang", type=str, default="en")
+    parser.add_argument("--inter_lang", type=str, default="es")
+    parser.add_argument("--edit_distance_threshold", type=int, default=-1)
+    parser.add_argument("--verbose", type=int, default=1)
+    parser.add_argument("in_text")
+    parser.add_argument("out_text")
+    args = parser.parse_args()
+
+    if args.verbose > 0:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+        )
+
+    with open(args.in_text) as f:
+        lines = [line.strip() for line in f.readlines()]
+    text = {line.split()[0]: " ".join(line.split()[1:]) for line in lines}
+
+    with open(args.out_text, "w") as f:
+        for utt_id, txt in text.items():
+            blob = TextBlob(txt.lower())
+            # NOTE(kan-bayashi): Sleep to avoid too many request
+            time.sleep(len(txt) * DELAY_ALPHA)
+            blob = blob.translate(to=args.inter_lang, from_lang=args.tgt_lang)
+            # NOTE(kan-bayashi): Sleep to avoid too many request
+            time.sleep(len(txt) * DELAY_ALPHA)
+            blob = blob.translate(to=args.tgt_lang, from_lang=args.inter_lang)
+            new_txt = str(blob).upper()
+            logging.info(f"original  : {txt}")
+            logging.info(f"translated: {new_txt}")
+            if args.edit_distance_threshold > 0:
+                distance = editdistance.eval(txt, new_txt)
+                if distance > args.edit_distance_threshold:
+                    continue
+            f.write(f"{utt_id}_{args.inter_lang}2{args.tgt_lang} {new_txt}\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/egs/clotho/aac_word/local/data_prep_audiocaps.py b/egs/clotho/aac_word/local/data_prep_audiocaps.py
new file mode 100644
index 00000000000..e6109ae7896
--- /dev/null
+++ b/egs/clotho/aac_word/local/data_prep_audiocaps.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+import os
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("data_dir", type=str, help="path to audiocaps wav data directory")
+    parser.add_argument("annotations_dir", type=str, help="path to audiocaps annotation data directory")
+    parser.add_argument("out_dirname", type=str, help="name of output data directory")
+
+    args = parser.parse_args()
+    
+    data_dir = args.data_dir
+    file_names = os.path.join(args.annotations_dir, 'filenames.txt')
+    file_captions = os.path.join(args.annotations_dir, 'captions.txt')
+    out_dir = os.path.join('data',args.out_dirname)
+    
+    filepaths_dict = {}
+    with open(file_names,'r') as f:
+        for line in f.readlines():
+            file_id = 'file_audiocaps_'+line[:-1].split(' ')[0].split('_')[1].zfill(6)
+            filename = line[:-1].split(' ')[1]
+            filepath = os.path.join(data_dir, filename)
+            if os.path.isfile(filepath):
+                filepaths_dict[file_id] = filepath
+
+    filecaptions_dict = {}
+    with open(file_captions,'r') as f:
+        for line in f.readlines():
+            file_id = 'file_audiocaps_'+line[:-1].split(' ')[0].split('_')[1].zfill(6) # line[:-1].split(' ')[0]
+            caption = ' '.join(line[:-1].split(' ')[1:])
+            if file_id in filepaths_dict:
+                filecaptions_dict[file_id] = caption
+
+    filespeakers_dict = {key:'spk_audiocaps_'+f"{key.split('_')[-1]}".zfill(6) for key in filecaptions_dict}
+    # filespeakers_dict = {key:f"spk_{key}" for key in filecaptions_dict}
+
+    if not os.path.isdir(out_dir): os.makedirs(out_dir)
+
+    with open(os.path.join(out_dir,'text'), 'w') as f:
+        for key,val in filecaptions_dict.items():
+            f.write(f'{key} {val}\n')
+
+    with open(os.path.join(out_dir,'utt2spk'), 'w') as f:
+        for key,val in filespeakers_dict.items():
+            f.write(f'{key} {val}\n')
+            
+    with open(os.path.join(out_dir,'spk2utt'), 'w') as f:
+        for key,val in filespeakers_dict.items():
+            f.write(f'{val} {key}\n')
+            
+    with open(os.path.join(out_dir,'wav.scp'), 'w') as f:
+        for key,val in filepaths_dict.items():
+            f.write(f'{key} {val}\n')
\ No newline at end of file
diff --git a/egs/clotho/aac_word/local/download_large_drive_file.sh b/egs/clotho/aac_word/local/download_large_drive_file.sh
new file mode 100755
index 00000000000..0158e57f8fa
--- /dev/null
+++ b/egs/clotho/aac_word/local/download_large_drive_file.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+fileid=$1         # "1-QO73Elfdhtx8Jo918eJlksZKvTAoidx"
+filename=$2       # "tmp/audiocaps_data_train.tar.gz"
+cookie_path=./tmp/cookie
+
+curl -c ${cookie_path} -s -L "https://drive.google.com/uc?export=download&id=${fileid}" > /dev/null
+curl -Lb ${cookie_path} "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ${cookie_path}`&id=${fileid}" -o ${filename}
+
+rm ${cookie_path}
\ No newline at end of file
diff --git a/egs/clotho/aac_word/local/evaluate_decoded_captions.py b/egs/clotho/aac_word/local/evaluate_decoded_captions.py
new file mode 100644
index 00000000000..8d90e562575
--- /dev/null
+++ b/egs/clotho/aac_word/local/evaluate_decoded_captions.py
@@ -0,0 +1,63 @@
+import argparse
+import json
+import os
+from eval_metrics import evaluate_metrics
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("decoded_json_path", type=str)
+    parser.add_argument("groundtruth_captions_path", type=str)
+    args = parser.parse_args()
+
+    gts = {}
+    with open(args.groundtruth_captions_path) as f:
+        lines = [line[:-1] for line in f.readlines()]
+        for line in lines:
+            key = line.split(' ')[0]
+            caption = ' '.join(line.split(' ')[1:])
+            fileid = key.split('_')[0]
+            capid = int(key[-1])
+            if fileid not in gts:
+                gts[fileid] = {'file_name': fileid}
+            gts[fileid][f'caption_{capid+1}'] = caption.lower()
+
+    preds = {}
+    with open(args.decoded_json_path) as f:
+        json_data = json.load(f)
+        for key, val in json_data['utts'].items():
+            fileid = key.split('_')[0]
+            if fileid not in preds:
+                preds[fileid] = {'file_name': fileid, 'caption_predicted': ''}
+            pred_caption = val['output'][0]['rec_text'].replace(' <eos>','').replace('<eos>','').lower().replace('▁',' ')
+            preds[fileid][f'caption_predicted'] = pred_caption
+
+    captions_gts = [val for _,val in gts.items()]
+    captions_preds = [val for _,val in preds.items()]
+    
+    metrics = evaluate_metrics(captions_preds, captions_gts)
+    metrics_individual = {}
+    for metric in metrics:
+        for fileid,val in metrics[metric]['scores'].items():
+            if fileid not in metrics_individual: metrics_individual[fileid] = []
+            metrics_individual[fileid].append(round(val,3))
+
+    dashes = '|'.join(['{:-^10}'.format('')]*10)
+    headers = ['fileID','BLEU_1','BLEU_2','BLEU_3','BLEU_4','METEOR','ROUGE_L','CIDEr','SPICE','SPIDEr']
+    def tabled_row(arr): return '|'.join(['{:^10}'.format(x) for x in arr])
+
+    decode_dirpath = os.path.dirname(args.decoded_json_path)
+    caption_evalpath = os.path.join(decode_dirpath,'caption_evaluation_results.txt')
+    with open(caption_evalpath, 'w') as f:
+        f.write(f'|{dashes}|\n')
+        f.write(f'|{tabled_row(headers)}|\n')
+        f.write(f'|{dashes}|\n')
+        metrics_summary = ['overall']+[round(metrics[metric]['score'],3) for metric in metrics]
+        f.write(f'|{tabled_row(metrics_summary)}|\n')
+        f.write(f'|{dashes}|\n')
+        for fileid,score_list in metrics_individual.items():
+            metrics_fileid = [fileid]+score_list
+            f.write(f'|{tabled_row(metrics_fileid)}|\n')
+        f.write(f'|{dashes}|\n')
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/egs/clotho/aac_word/path.sh b/egs/clotho/aac_word/path.sh
new file mode 100644
index 00000000000..9048e150308
--- /dev/null
+++ b/egs/clotho/aac_word/path.sh
@@ -0,0 +1,25 @@
+MAIN_ROOT=$PWD/../../..
+KALDI_ROOT=$MAIN_ROOT/tools/kaldi
+
+export PATH=$PWD/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH
+[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1
+. $KALDI_ROOT/tools/config/common_path.sh
+export LC_ALL=C
+
+export PATH=$MAIN_ROOT/tools/sentencepiece/build/src:$PATH
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$MAIN_ROOT/tools/chainer_ctc/ext/warp-ctc/build
+. "${MAIN_ROOT}"/tools/activate_python.sh && . "${MAIN_ROOT}"/tools/extra_path.sh
+export PATH=$MAIN_ROOT/utils:$MAIN_ROOT/espnet/bin:$PATH
+
+export OMP_NUM_THREADS=1
+
+# check extra module installation
+if ! which spm_decode > /dev/null; then
+    echo "Error: it seems that sentencepiece is not installed." >&2
+    echo "Error: please install sentencepiece as follows." >&2
+    echo "Error: cd ${MAIN_ROOT}/tools && make sentencepiece.done" >&2
+    return 1
+fi
+
+# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C
+export PYTHONIOENCODING=UTF-8
diff --git a/egs/clotho/aac_word/run.sh b/egs/clotho/aac_word/run.sh
new file mode 100755
index 00000000000..a30292c7581
--- /dev/null
+++ b/egs/clotho/aac_word/run.sh
@@ -0,0 +1,322 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 Johns Hopkins University (Shinji Watanabe)
+#  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+
+. ./path.sh || exit 1;
+. ./cmd.sh || exit 1;
+
+# general configuration
+backend=pytorch
+stage=-1       # start from 0 if you need to start from data preparation
+stop_stage=100
+ngpu=1         # number of gpus ("0" uses cpu, otherwise use gpu)
+debugmode=1
+dumpdir=dump   # directory to dump full features
+N=0            # number of minibatches to be used (mainly for debugging). "0" uses all minibatches.
+verbose=0      # verbose option
+resume=        # Resume the training from snapshot
+
+# feature configuration
+do_delta=false
+
+# preprocess_config=conf/no_preprocess.yaml
+preprocess_config=conf/specaug.yaml
+train_config=conf/train.yaml                   # conf/tuning/train_conformer_small.yaml
+lm_config=conf/lm.yaml
+decode_config=conf/decode.yaml          # conf/decode.yaml
+
+# rnnlm related
+lm_vocabsize=65000
+lm_resume=        # specify a snapshot file to resume LM training
+lmtag=            # tag for managing LMs
+
+# decoding parameter
+recog_model=model.acc.best # set a model to be used for decoding: 'model.acc.best' or 'model.loss.best'
+n_average=10
+use_valbest_average=false
+use_custom_model=""
+
+lang=en # en de fr cy tt kab ca zh-TW it fa eu es ru tr nl eo zh-CN rw pt zh-HK cs pl uk 
+
+# exp tag
+tag="" # tag for managing experiments.
+
+# flags for clotho-v2 dataset
+download_clothov2=false
+
+# flags for audiocaps dataset
+download_audiocaps=false
+augment_audiocaps=true
+
+# flag for performing speed perturbation augmentation
+augment_speedperturbation=false
+
+# flag for setting caption evaluation tools
+download_evalmetrics=false
+
+. utils/parse_options.sh || exit 1;
+
+# Set bash to 'debug' mode, it will exit on :
+# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands',
+set -e
+set -u
+set -o pipefail
+
+train_set=dev_clothov2
+train_dev=val_clothov2
+test_set=                                             # test_clothov2
+recog_set="recog_val_clothov2 recog_eval_clothov2"
+if ${augment_audiocaps} || ${download_audiocaps}; then
+    train_set_without_audiocaps=${train_set}
+    train_set=${train_set}_audiocaps
+fi
+
+clothov2_data_urlid="1-0s5SVN-SagFHDql95JAcfdn1GW4V7o9"
+clothov2_kaldidata_urlid="1XE9ZKqW2sSMfA3xpdKl6ZyG2CFsIbbBQ"
+
+audiocaps_train_urlid="1-QO73Elfdhtx8Jo918eJlksZKvTAoidx"
+audiocaps_annotations_urlid="1Fn-dBl6SCKVukq1gMId4LTGBfrE3DJ-r"
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    ### Task dependent. You have to make data the following preparation part by yourself.
+    ### But you can utilize Kaldi recipes in most cases 
+
+    ### ---- below lines are for preparing clotho-v2 dataset ----
+    if ${download_clothov2}; then
+        echo "Downloading and Preparing Clotho-V2 Dataset"
+        mkdir -p tmp
+        wget "https://drive.google.com/uc?export=download&id=${clothov2_kaldidata_urlid}" -O "tmp/clothov2_kaldidata.tar.gz"
+        tar -xzf "tmp/clothov2_kaldidata.tar.gz" -C ./
+        ./local/download_large_drive_file.sh ${clothov2_data_urlid} "tmp/clothov2_data.tar.gz"
+        tar -xzf "tmp/clothov2_data.tar.gz" -C ./
+    fi
+
+    ### ---- below lines are for preparing audiocaps dataset ----
+    if ${download_audiocaps}; then
+        echo "Downloading and Preparing AudioCaps Dataset"
+        mkdir -p tmp
+        wget "https://drive.google.com/uc?export=download&id=${audiocaps_annotations_urlid}" -O "tmp/annotations.tar.gz"
+        tar -xzf "tmp/annotations.tar.gz" -C ./
+        ./local/download_large_drive_file.sh ${audiocaps_train_urlid} "tmp/audiocaps_data_train.tar.gz"
+        tar -xzf "tmp/audiocaps_data_train.tar.gz" -C ./
+        python local/data_prep_audiocaps.py audiocaps_data/train audiocaps_data/annotations/train dev_audiocaps
+        utils/combine_data.sh data/${train_set} data/${train_set_without_audiocaps} data/dev_audiocaps
+    fi
+
+    ### ---- below lines are for speed perturbation based augmentation ----
+    if ${augment_speedperturbation}; then
+        utils/perturb_data_dir_speed.sh 0.9 data/${train_set} data/temp1
+        utils/perturb_data_dir_speed.sh 1.0 data/${train_set} data/temp2
+        utils/perturb_data_dir_speed.sh 1.1 data/${train_set} data/temp3
+        utils/combine_data.sh --extra-files utt2uniq data/${train_set} data/temp1 data/temp2 data/temp3
+        rm -r data/temp1 data/temp2 data/temp3
+    fi
+    
+    ### ---- below lines are for preparing caption evaluation tools ----
+    if ${download_evalmetrics}; then
+        echo "Download and Prepare Evaluation Metrics"
+        mkdir -p tmp
+        pip3 install scikit-image
+        git clone https://github.com/audio-captioning/dcase-2020-baseline.git tmp/dcase_baseline
+        cp -r tmp/dcase_baseline/coco_caption ./local/
+        cp tmp/dcase_baseline/eval_metrics.py ./local/
+        chmod -R 755 tmp/dcase_baseline
+        rm -r tmp/dcase_baseline
+        ./local/coco_caption/get_stanford_models.sh
+    fi
+    echo "Data Preparation Successful"
+fi
+
+feat_tr_dir=${dumpdir}/${train_set}/delta${do_delta}; mkdir -p ${feat_tr_dir}
+feat_dt_dir=${dumpdir}/${train_dev}/delta${do_delta}; mkdir -p ${feat_dt_dir}
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+    ### Task dependent. You have to design training and dev sets by yourself.
+    ### But you can utilize Kaldi recipes in most cases
+    echo "stage 1: Feature Generation"
+    fbankdir=fbank/${lang}
+    # Generate the fbank features; by default 80-dimensional fbanks with pitch on each frame
+    for x in ${train_set} ${train_dev} ${recog_set}; do
+        steps/make_fbank_pitch.sh --cmd "$train_cmd" --nj 16 --write_utt2num_frames true \
+                                  data/${x} exp/make_fbank/${x} ${fbankdir}
+        utils/fix_data_dir.sh data/${x}
+    done
+
+    # for x in ${train_set} ${train_dev} ${recog_set}; do
+    #     # Remove features with too long frames in training data
+    #     max_len=3000
+    #     if [ -d "data/${x}/${x}_temp" ]; then
+    #         rm -r data/${x}/${x}_temp
+    #     fi
+    #     remove_longshortdata.sh --maxframes $max_len data/${x} data/${x}_temp
+    #     mv data/${x}_temp data/${x}
+    # done
+
+    # compute global CMVN
+    compute-cmvn-stats scp:data/${train_set}/feats.scp data/${train_set}/cmvn.ark
+
+    dump.sh --cmd "$train_cmd" --nj 16 --do_delta ${do_delta} \
+            data/${train_set}/feats.scp data/${train_set}/cmvn.ark exp/dump_feats/train ${feat_tr_dir}
+    dump.sh --cmd "$train_cmd" --nj 16 --do_delta ${do_delta} \
+            data/${train_dev}/feats.scp data/${train_set}/cmvn.ark exp/dump_feats/dev ${feat_dt_dir} 
+    for rtask in ${recog_set}; do
+        feat_recog_dir=${dumpdir}/${rtask}/delta${do_delta}; mkdir -p ${feat_recog_dir}
+        dump.sh --cmd "$train_cmd" --nj 16 --do_delta ${do_delta} \
+                data/${rtask}/feats.scp data/${train_set}/cmvn.ark exp/dump_feats/recog/${rtask} \
+                ${feat_recog_dir}
+    done
+    echo "Feature Generation Successful"
+fi
+
+dict=data/${lang}_lang_char/${train_set}_word_units.txt
+echo "dictionary: ${dict}"
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+    ### Task dependent. You have to check non-linguistic symbols used in the corpus.
+    echo "stage 2: Dictionary and Json Data Preparation"
+    mkdir -p data/${lang}_lang_char/
+    echo "make a dictionary"
+    echo "<unk> 1" > ${dict} # <unk> must be 1, 0 will be used for "blank" in CTC
+    text2token.py -s 1 -n 1 -t word data/${train_set}/text | cut -f 2- -d" " | tr " " "\n" \
+    | sort | uniq | grep -v -e '^\s*$' | awk '{print $0 " " NR+1}' >> ${dict}
+    wc -l ${dict}
+
+    echo "make json files"
+    data2json.sh --feat ${feat_tr_dir}/feats.scp --trans_type word \
+                 data/${train_set} ${dict} > ${feat_tr_dir}/data.json
+    data2json.sh --feat ${feat_dt_dir}/feats.scp --trans_type word \
+                 data/${train_dev} ${dict} > ${feat_dt_dir}/data.json
+    for rtask in ${recog_set}; do
+        feat_recog_dir=${dumpdir}/${rtask}/delta${do_delta}
+        data2json.sh --feat ${feat_recog_dir}/feats.scp --trans_type word \
+                     data/${rtask} ${dict} > ${feat_recog_dir}/data.json
+    done
+    echo "Dictionary and Json Data Preparation Successful"
+fi
+
+# you can skip this and remove --rnnlm option in the recognition (stage 5)
+if [ -z ${lmtag} ]; then
+    lmtag=$(basename ${lm_config%.*})
+fi
+lmexpname=train_${lang}_rnnlm_${backend}_${lmtag}
+lmexpdir=exp/${lmexpname}
+mkdir -p ${lmexpdir}
+
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+    echo "stage 3: LM Preparation"
+    lmdatadir=data/local/wordlm_train
+    mkdir -p ${lmdatadir}
+    text2token.py -s 1 -n 1 -t word data/${train_set}/text | cut -f 2- -d" " > ${lmdatadir}/train.txt
+    text2token.py -s 1 -n 1 -t word data/${train_dev}/text | cut -f 2- -d" " > ${lmdatadir}/valid.txt
+
+    ${cuda_cmd} --gpu ${ngpu} ${lmexpdir}/train.log \
+        lm_train.py \
+        --config ${lm_config} \
+        --ngpu ${ngpu} \
+        --backend ${backend} \
+        --verbose 1 \
+        --outdir ${lmexpdir} \
+        --tensorboard-dir tensorboard/${lmexpname} \
+        --train-label ${lmdatadir}/train.txt \
+        --valid-label ${lmdatadir}/valid.txt \
+        --resume ${lm_resume} \
+        --dict ${dict}
+fi
+
+if [ -z ${tag} ]; then
+    expname=${train_set}_${backend}_$(basename ${train_config%.*})_$(basename ${preprocess_config%.*})
+    if ${do_delta}; then
+        expname=${expname}_delta
+    fi
+else
+    expname=${train_set}_${backend}_${tag}
+fi
+expdir=exp/${expname}
+mkdir -p ${expdir}
+
+if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+    echo "stage 4: Network Training"
+    ${cuda_cmd} --gpu ${ngpu} ${expdir}/train.log \
+        asr_train.py \
+        --config ${train_config} \
+        --preprocess-conf ${preprocess_config} \
+        --ngpu ${ngpu} \
+        --backend ${backend} \
+        --outdir ${expdir}/results \
+        --tensorboard-dir tensorboard/${expname} \
+        --debugmode ${debugmode} \
+        --dict ${dict} \
+        --debugdir ${expdir} \
+        --minibatches ${N} \
+        --verbose ${verbose} \
+        --resume ${resume} \
+        --train-json ${feat_tr_dir}/data.json \
+        --valid-json ${feat_dt_dir}/data.json
+fi
+
+if [ ${stage} -le 5 ] && [ ${stop_stage} -ge 5 ]; then
+    echo "stage 5: Decoding"
+    nj=12
+    if [[ $(get_yaml.py ${train_config} model-module) = *transformer* ]] || \
+           [[ $(get_yaml.py ${train_config} model-module) = *conformer* ]] || \
+           [[ $(get_yaml.py ${train_config} etype) = custom ]] || \
+           [[ $(get_yaml.py ${train_config} dtype) = custom ]]; then
+        if ${use_valbest_average}; then
+            recog_model=model.val${n_average}.avg.best
+            opt="--log ${expdir}/results/log"
+        else
+            recog_model=model.last${n_average}.avg.best
+            opt="--log"
+        fi
+        if [ -z ${use_custom_model} ]; then
+            average_checkpoints.py \
+                        ${opt} \
+                        --backend ${backend} \
+                        --snapshots ${expdir}/results/snapshot.ep.* \
+                        --out ${expdir}/results/${recog_model} \
+                        --num ${n_average}
+        else
+            recog_model=${use_custom_model}
+        fi
+    fi
+    pids=() # initialize pids
+    for rtask in ${recog_set}; do
+    (
+        if ${use_valbest_average}; then
+            decode_dir=decode_${rtask}_$(basename ${decode_config%.*})_${lmtag}_val${n_average}
+        else
+            decode_dir=decode_${rtask}_$(basename ${decode_config%.*})_${lmtag}_last${n_average}
+        fi
+        feat_recog_dir=${dumpdir}/${rtask}/delta${do_delta}
+
+        # split data
+        splitjson.py --parts ${nj} ${feat_recog_dir}/data.json
+
+        #### use CPU for decoding
+        ngpu=0
+
+        ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
+            asr_recog.py \
+            --config ${decode_config} \
+            --ngpu ${ngpu} \
+            --backend ${backend} \
+            --batchsize 0 \
+            --recog-json ${feat_recog_dir}/split${nj}utt/data.JOB.json \
+            --result-label ${expdir}/${decode_dir}/data.JOB.json \
+            --model ${expdir}/results/${recog_model}  \
+            --rnnlm ${lmexpdir}/rnnlm.model.best
+
+        # score_sclite.sh --bpe ${nbpe} --bpemodel ${bpemodel}.model --wer true ${expdir}/${decode_dir} ${dict}
+        score_sclite.sh --wer true ${expdir}/${decode_dir} ${dict}
+        python local/evaluate_decoded_captions.py ${expdir}/${decode_dir}/data.json \
+            data/${rtask}/groundtruth_captions.txt > "${expdir}/${decode_dir}/caption_evaluation_summary.txt"
+        echo "Evaluation metrics summary is saved to: ${expdir}/${decode_dir}/caption_evaluation_summary.txt"
+        echo "Evaluation metrics individual results are saved to: ${expdir}/${decode_dir}/caption_evaluation_results.txt"
+
+    ) &
+    pids+=($!) # store background pids
+    done
+    i=0; for pid in "${pids[@]}"; do wait ${pid} || ((++i)); done
+    [ ${i} -gt 0 ] && echo "$0: ${i} background jobs are failed." && false
+    echo "Finished"
+fi
diff --git a/egs/clotho/aac_word/steps b/egs/clotho/aac_word/steps
new file mode 120000
index 00000000000..91f2d234e20
--- /dev/null
+++ b/egs/clotho/aac_word/steps
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/steps
\ No newline at end of file
diff --git a/egs/clotho/aac_word/utils b/egs/clotho/aac_word/utils
new file mode 120000
index 00000000000..f49247da827
--- /dev/null
+++ b/egs/clotho/aac_word/utils
@@ -0,0 +1 @@
+../../../tools/kaldi/egs/wsj/s5/utils
\ No newline at end of file
diff --git a/utils/text2token.py b/utils/text2token.py
index 56c39138ffc..7059b6ae45e 100755
--- a/utils/text2token.py
+++ b/utils/text2token.py
@@ -54,13 +54,15 @@ def get_parser():
         "-t",
         type=str,
         default="char",
-        choices=["char", "phn"],
-        help="""Transcript type. char/phn. e.g., for TIMIT FADG0_SI1279 -
+        choices=["char", "phn", "word"],
+        help="""Transcript type. char/phn/word. e.g., for TIMIT FADG0_SI1279 -
                         If trans_type is char,
                         read from SI1279.WRD file -> "bricks are an alternative"
                         Else if trans_type is phn,
                         read from SI1279.PHN file -> "sil b r ih sil k s aa r er n aa l
-                        sil t er n ih sil t ih v sil" """,
+                        sil t er n ih sil t ih v sil"
+                        If trans_type is word,
+                        read from SI1279.WRD file -> "bricks are an alternative" """,
     )
     return parser
 
@@ -104,6 +106,8 @@ def main():
 
         if args.trans_type == "phn":
             a = a.split(" ")
+        elif args.trans_type == "word":
+            a = a.split()
         else:
             if len(match_pos) > 0:
                 chars = []
@@ -127,6 +131,12 @@ def main():
         a_chars = [z.replace(" ", args.space) for z in a_flat]
         if args.trans_type == "phn":
             a_chars = [z.replace("sil", args.space) for z in a_chars]
+        elif args.trans_type == "word":
+            a_chars = f" ".join(a_chars)
+            a_chars = a_chars.replace(",", " , ")
+            a_chars = f" ▁".join([""]+a_chars.split())
+            a_chars = a_chars.replace("▁,", ",")
+            a_chars = a_chars.split()
         print(" ".join(a_chars))
         line = f.readline()
 

From 4ffc185e37d0e8cc668e88e1018631b38f6148bc Mon Sep 17 00:00:00 2001
From: Chaitanya Prasad Narisetty <cnariset@andrew.cmu.edu>
Date: Mon, 7 Jun 2021 17:10:06 +0530
Subject: [PATCH 3/3] Update README.md

---
 egs/clotho/aac_word/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/egs/clotho/aac_word/README.md b/egs/clotho/aac_word/README.md
index ac8372f20f6..58bd8d0c590 100644
--- a/egs/clotho/aac_word/README.md
+++ b/egs/clotho/aac_word/README.md
@@ -1,7 +1,7 @@
 # Clotho Recipe
 
 ## Data preparation
-* Data preparation during `stage 0` can be performed by appropriately setting the boolean variables in below command. By default, all variables are set to `false`. Description of each variable is also detailed below.
+* Data preparation during `stage 0` can be performed by appropriately setting the boolean variables in below command. By default, all variables are set to `false` except for `--augment_audiocaps` which is set to `true`. Description of each variable is also detailed below.
   
   ```bash
   ./run.sh --stage 0 --stop_stage 0  \