chaning learning rate back to default of 1e-3 (#156)

* chaning learning rate back to default of 1e-3, however, we recommend 1e-4 for OPT fine-tuning experiments

* delete duplicate config, fix vocab size

* removed some obsolte config files from OPT experiments

---------

Co-authored-by: sfluegel <simon.fluegel@uni-osnabrueck.de>

Author: schnamo

configs/model/OPT_experiments/electra_LR.yml

@@ -4,16 +4,16 @@ init_args:
     class_path: chebai.loss.pretraining.ElectraPreLoss
   out_dim: null
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3
   config:
     generator:
-      vocab_size: 1400
+      vocab_size: 4400
       max_position_embeddings: 1800
       num_attention_heads: 8
       num_hidden_layers: 6
       type_vocab_size: 1
     discriminator:
-      vocab_size: 1400
+      vocab_size: 4400
       max_position_embeddings: 1800
       num_attention_heads: 8
       num_hidden_layers: 6

configs/model/OPT_experiments/electra_tox_expl.yml

@@ -2,7 +2,7 @@ class_path: chebai.models.Electra
 init_args:
   model_type: classification
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3
   config:
     vocab_size: 4400
     max_position_embeddings: 1800

configs/model/electra-for-pretraining.yml

@@ -1,9 +1,10 @@
 class_path: chebai.models.Electra
 init_args:
+  model_type: classification
   optimizer_kwargs:
     lr: 1e-3
   config:
-    vocab_size: 1400
+    vocab_size: 4400
     max_position_embeddings: 301
     num_attention_heads: 8
     num_hidden_layers: 6

configs/model/electra.yml

@@ -2,7 +2,7 @@ class_path: chebai.models.Electra
 init_args:
   model_type: classification
   optimizer_kwargs:
-    lr: 1e-4
+    lr: 1e-3 # we recommend  1e-4 for OPT finetuning, however, 1e-3 is the default
   config:
     vocab_size: 1400
     max_position_embeddings: 1800