Merge branch 'main' of https://github.com/may-/joeynmt into main

Author: may-

.github/workflows/main.yml

@@ -35,8 +35,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install --upgrade torch==1.11.0+cu115 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu115
-          pip install -e .
+          python -m pip install --upgrade torch torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
+          python -m pip install -e .
 
       # Check code format
       - name: Lint
@@ -48,4 +48,4 @@ jobs:
       # Run unittest
       - name: Test
         run: |
-          python -m pytest
+          python -m unittest

.pylintrc

@@ -34,7 +34,7 @@ unsafe-load-any-extension=no
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=
+extension-pkg-whitelist=fastBPE
 
 [MESSAGES CONTROL]
 

README.md

@@ -14,7 +14,7 @@ Joey S2T implements the following features:
 - CMVN, SpecAugment
 - WER evaluation
 
-Furthermore, all the functionalities in JoeyNMT v2.0 are also available from JoeyS2T:
+Furthermore, all the functionalities in JoeyNMT v2 are also available from JoeyS2T:
 - BLEU and ChrF evaluation
 - BPE tokenization (with BPE dropout option)
 - Beam search and greedy decoding (with repetition penalty, ngram blocker)
@@ -26,31 +26,30 @@ Furthermore, all the functionalities in JoeyNMT v2.0 are also available from Joe
 
 
 ## Installation
+
 JoeyS2T is built on [PyTorch](https://pytorch.org/). Please make sure you have a compatible environment.
 We tested JoeyS2T with
 - python 3.10
-- torch 1.11.0
-- cuda 11.5
+- torch 1.12.1
+- cuda 11.6
 
 Clone this repository and install via pip:
 ```bash
 $ git clone https://github.com/may-/joeys2t.git
 $ cd joeynmt
-$ pip install . -e
-```
-Run the unit tests:
-```bash
-$ python -m unittest
+$ pip install -e .
 ```
 
 
+
 ## Documentation & Tutorials
 
 Please check the JoeyNMT's [documentation](https://joeynmt.readthedocs.io) first, if you are not familiar with JoeyNMT yet.
 
 For details, follow the tutorials in [notebooks](notebooks) dir.
+
 - [quick-start-with-joeynmt2](notebooks/quick-start-with-joeynmt2.ipynb)
-- [speech-to-text-with-joeynmt2](notebooks/joeyS2T_ASR_tutorial.ipynb) 
+- [speech-to-text-with-joeynmt2](notebooks/joeyS2T_ASR_tutorial.ipynb)
 
 
 
@@ -67,5 +66,3 @@ Please leave an issue if you have found a bug in the code.
 
 For general questions, email me at `ohta <at> cl.uni-heidelberg.de`.
 
-
-

configs/iwslt14_deen_sp.yaml

@@ -17,10 +17,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "test/data/iwslt14_sp.vocab"
+        voc_file: "test/data/iwslt14/sp.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "test/data/iwslt14_sp.model"
+            model_file: "test/data/iwslt14/sp.model"
             model_type: "unigram"
             character_coverage: 1.0
             alpha: 0.1
@@ -33,10 +33,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "test/data/iwslt14_sp.vocab"
+        voc_file: "test/data/iwslt14/sp.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "test/data/iwslt14_sp.model"
+            model_file: "test/data/iwslt14/sp.model"
             model_type: "unigram"
             character_coverage: 1.0
             alpha: 0.1
@@ -82,6 +82,7 @@ training:
     overwrite: False
     shuffle: True
     use_cuda: True
+    fp16: True
     print_valid_sents: [0, 1, 2, 3]
     keep_best_ckpts: 5
 

configs/jparacrawl_enja_sp.yaml

@@ -8,7 +8,7 @@ data:
     test: "../datasets/datasets/kftt"
     dataset_type: "huggingface"
     dataset_cfg:
-        name: "en-ja"
+        name: "ja-en"
     sample_train_subset: -1
     sample_dev_subset: 200
     src:
@@ -19,10 +19,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "/scratch5t/ohta/jparacrawl_v3/spm_en.vocab"
+        voc_file: "subwords/jparacrawl_en.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "/scratch5t/ohta/jparacrawl_v3/spm_en.model"
+            model_file: "subwords/jparacrawl_en.model"
             model_type: "unigram"
             character_coverage: 1.0
             nbest_size: 10
@@ -35,10 +35,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "/scratch5t/ohta/jparacrawl_v3/spm_ja.vocab"
+        voc_file: "subwords/jparacrawl_ja.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "/scratch5t/ohta/jparacrawl_v3/spm_ja.model"
+            model_file: "subwords/jparacrawl_ja.model"
             model_type: "unigram"
             character_coverage: 0.995
             nbest_size: 10
@@ -61,8 +61,8 @@ testing:
         tokenize: "ja-mecab"
 
 training:
-    #load_model: "/workspace/mitarb/ohta/models/jparacrawl_enja_seed456/best.ckpt"
-    random_seed: 456
+    #load_model: "models/jparacrawl_enja/best.ckpt"
+    random_seed: 42
     optimizer: "adam"
     normalization: "tokens"
     adam_betas: [0.9, 0.98]

configs/jparacrawl_jaen_sp.yaml

@@ -2,9 +2,9 @@ name: "jparacrawl_jaen_sp"
 joeynmt_version: "2.0.0"
 
 data:
-    train: "../datasets/datasets/jparacrawl"
-    dev: "../datasets/datasets/wmt21"
-    test: "../datasets/datasets/kftt"
+    train: "jparacrawl"
+    dev: "wmt21"
+    test: "kftt"
     dataset_type: "huggingface"
     dataset_cfg:
         name: "ja-en"
@@ -18,10 +18,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "data/jparacrawl_v3/spm_ja.vocab"
+        voc_file: "subwords/jparacrawl_ja.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "data/jparacrawl_v3/spm_ja.model"
+            model_file: "subwords/jparacrawl_ja.model"
             model_type: "unigram"
             character_coverage: 1.0
             nbest_size: 10
@@ -34,10 +34,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "data/jparacrawl_v3/spm_en.vocab"
+        voc_file: "subwords/jparacrawl_en.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "data/jparacrawl_v3/spm_en.model"
+            model_file: "subwords/jparacrawl_en.model"
             model_type: "unigram"
             character_coverage: 1.0
             nbest_size: 10
@@ -60,7 +60,7 @@ testing:
         tokenize: "intl"
 
 training:
-    #load_model: "models/jparacrawl_enja/best.ckpt"
+    #load_model: "models/jparacrawl_jaen/best.ckpt"
     random_seed: 42
     optimizer: "adam"
     normalization: "tokens"

configs/rnn_small.yaml

@@ -113,8 +113,8 @@ model:                              # specify your model architecture here
     initializer: "xavier_uniform"   # initializer for all trainable weights (xavier_uniform, xavier_normal, zeros, normal, uniform)
     init_weight: 0.01               # weight to initialize; for uniform, will use [-weight, weight]
     init_gain: 1.0                  # gain for Xavier initializer (default: 1.0)
-    bias_initializer: "zeros"       # initializer for bias terms (xavier, zeros, normal, uniform)
-    embed_initializer: "normal"     # initializer for embeddings (xavier, zeros, normal, uniform)
+    bias_initializer: "zeros"       # initializer for bias terms (xavier_uniform, xavier_normal, zeros, normal, uniform)
+    embed_initializer: "normal"     # initializer for embeddings (xavier_uniform, xavier_normal, zeros, normal, uniform)
     embed_init_weight: 0.1          # weight to initialize; for uniform, will use [-weight, weight]
     embed_init_gain: 1.0            # gain for Xavier initializer for embeddings (default: 1.0)
     init_rnn_orthogonal: False      # use orthogonal initialization for recurrent weights (default: False)

configs/transformer_reverse.yaml

@@ -59,9 +59,9 @@ model:
     initializer: "xavier_uniform"   # initializer for all trainable weights (xavier_uniform, xavier_normal, zeros, normal, uniform)
     init_gain: 1.0                  # gain for Xavier initializer (default: 1.0)
     bias_initializer: "zeros"       # initializer for bias terms (xavier_uniform, xavier_normal, zeros, normal, uniform)
-    embed_initializer: "xavier_uniform"     # initializer for embeddings (xavier_uniform, xavier_normal, zeros, normal, uniform)
+    embed_initializer: "xavier_uniform"  # initializer for embeddings (xavier_uniform, xavier_normal, zeros, normal, uniform)
     embed_init_gain: 1.0            # gain for Xavier initializer for embeddings (default: 1.0)
-    tied_embeddings: True          # tie src and trg embeddings, only applicable if vocabularies are the same, default: False
+    tied_embeddings: True           # tie src and trg embeddings, only applicable if vocabularies are the same, default: False
     tied_softmax: True
     encoder:
         type: "transformer"

configs/transformer_small.yaml

@@ -114,7 +114,11 @@ model:                              # specify your model architecture here
     initializer: "xavier_uniform"   # initializer for all trainable weights (xavier_uniform, xavier_normal, zeros, normal, uniform)
     init_gain: 1.0                  # gain for Xavier initializer (default: 1.0)
     bias_initializer: "zeros"       # initializer for bias terms (xavier_uniform, xavier_normal, zeros, normal, uniform)
+<<<<<<< HEAD
     embed_initializer: "xavier_uniform"     # initializer for embeddings (xavier_uniform, xavier_normal, zeros, normal, uniform)
+=======
+    embed_initializer: "xavier_uniform"  # initializer for embeddings (xavier_uniform, xavier_normal, zeros, normal, uniform)
+>>>>>>> 4a132900d3ae55d5df9bae11196ae32a5014efd1
     embed_init_gain: 1.0            # gain for Xavier initializer for embeddings (default: 1.0)
     tied_embeddings: False          # tie src and trg embeddings, only applicable if vocabularies are the same, default: False
     tied_softmax: True

configs/wmt17_ende_bpe.yaml

@@ -17,11 +17,11 @@ data:
         normalize: True
         level: "bpe"
         voc_min_freq: 1
-        voc_file: "data/subwords/wmt17_bpe.vocab"
+        voc_file: "subwords/wmt17_bpe.vocab"
         tokenizer_type: "subword-nmt"
         tokenizer_cfg:
             num_merges: 32000
-            codes: "data/subwords/wmt17_bpe.codes"
+            codes: "subwords/wmt17_bpe.codes"
             dropout: 0.1
             pretokenizer: "moses"
     trg:
@@ -32,11 +32,11 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "data/subwords/wmt17_bpe.vocab"
+        voc_file: "subwords/wmt17_bpe.vocab"
         tokenizer_type: "subword-nmt"
         tokenizer_cfg:
             num_merges: 32000
-            codes: "data/subwords/wmt17_bpe.codes"
+            codes: "subwords/wmt17_bpe.codes"
             dropout: 0.1
             pretokenizer: "moses"
 

configs/wmt17_ende_sp.yaml

@@ -18,10 +18,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "data/spm/wmt17_sp.vocab"
+        voc_file: "subwords/wmt17_sp.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "data/spm/wmt17_sp.model"
+            model_file: "subwords/wmt17_sp.model"
             model_type: "unigram"
             character_coverage: 1.0
             nbest_size: 10
@@ -35,10 +35,10 @@ data:
         level: "bpe"
         voc_limit: 32000
         voc_min_freq: 1
-        voc_file: "data/spm/wmt17_sp.vocab"
+        voc_file: "subwords/wmt17_sp.vocab"
         tokenizer_type: "sentencepiece"
         tokenizer_cfg:
-            model_file: "data/spm/wmt17_sp.model"
+            model_file: "subwords/wmt17_sp.model"
             model_type: "unigram"
             character_coverage: 1.0
             nbest_size: 10