Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,54 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Bible/10k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.sin_Sinh.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/10k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.tam_Taml.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.sin_Sinh.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/15k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.tam_Taml.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.sin_Sinh.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/20k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.tam_Taml.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/en-si/preprocess_en-si/train.en-si.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/en-si/preprocess_en-si/train.en-si.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/en-ta/encoded/train.bpe.ta filter=lfs diff=lfs merge=lfs -text
+Bible/25k/en-ta/preprocess_en-ta/train.en-ta.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/en-ta/preprocess_en-ta/train.en-ta.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/si-en/preprocess_si-en/train.si-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/si-en/preprocess_si-en/train.si-en.si.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/si-en/preprocess_si-en/train.sin_Sinh-eng_Latn.sin_Sinh.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/encoded/train.bpe.ta filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/encoded/train.bpe.tam_Taml filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/preprocess_ta-en/train.ta-en.en.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/preprocess_ta-en/train.ta-en.ta.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.eng_Latn.idx filter=lfs diff=lfs merge=lfs -text
+Bible/25k/ta-en/preprocess_ta-en/train.tam_Taml-eng_Latn.tam_Taml.idx filter=lfs diff=lfs merge=lfs -text

Bible/10k/en-si/cross_domain_outputs/Bible/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,26.039677587403336,23.03587719077928,2.4242883121600367,11.5576279670596,10.263352826655826,0.5392135714292526

Bible/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 22768 types
+[en] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.en: 1000 sents, 32449 tokens, 0.182% replaced (by <unk>)
+[si] Dictionary: 22768 types
+[si] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Bible/encoded/test.bpe.si: 1000 sents, 24684 tokens, 0.207% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/

Bible/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbd5355f399d11a285b5d413700786e6c8ceb6de89afad067f3686af0066dd05
+size 64898

Bible/10k/en-si/cross_domain_outputs/Bible/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1777b56c01d6bfff204421e80bbaebd68a0774e9cecab5a286dc93c701d9865
+size 49368

Bible/10k/en-si/cross_domain_outputs/FloresPlus/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,5.893442952573359,4.996178025293817,0.02774017339055153,0.03137188558404504,0.0365316339298661,0.246042742397592

Bible/10k/en-si/cross_domain_outputs/Government/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,4.266150357311315,3.517489253530752,0.01773648621868038,0.009720462308453153,0.011835331209402347,0.23862525007128715

Bible/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 22768 types
+[en] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe.en: 1000 sents, 44228 tokens, 3.79% replaced (by <unk>)
+[si] Dictionary: 22768 types
+[si] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Government/encoded/test.bpe.si: 1000 sents, 43297 tokens, 3.41% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/

Bible/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88838b86589f0e7a5e5cfad0d62f594e76470c8dadfdea4a6ef8a51c6235e690
+size 88456

Bible/10k/en-si/cross_domain_outputs/Government/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d47b101b20163b98455abdf5a62b1695b203d33f09fa273fc57ef623b0210630
+size 86594

Bible/10k/en-si/cross_domain_outputs/Health/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,4.612283752807804,3.7452257662701047,0.014853625123678398,0.016601194678912502,0.024506845844422774,0.2212059756678142

Bible/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(aim_repo=None, aim_run_hash=None, align_suffix=None, alignfile=None, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/', dict_only=False, empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=False, log_file=None, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, on_cpu_convert_precision=False, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, source_lang='en', srcdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.en.txt', suppress_crashes=False, target_lang='si', task='translation', tensorboard_logdir=None, testpref='../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe', tgtdict='../trained_models/Fairseq/Bible/10k/en-si/preprocess_en-si/dict.si.txt', threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref=None, use_plasma_view=False, user_dir=None, validpref=None, wandb_project=None, workers=16)
+[en] Dictionary: 22768 types
+[en] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe.en: 890 sents, 48078 tokens, 5.15% replaced (by <unk>)
+[si] Dictionary: 22768 types
+[si] ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Health/encoded/test.bpe.si: 890 sents, 55613 tokens, 3.66% replaced (by <unk>)
+Wrote preprocessed data to ../trained_models/Fairseq/Bible/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/

Bible/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d094dadccf63ad549ba12e708bb23eaf0f29c48cd8a942f6a6fe4c1ea3b41354
+size 96156

Bible/10k/en-si/cross_domain_outputs/Health/preprocess_en-si/test.en-si.si.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56048083e2c4a2a47c58c44ec17d2489274608d7b0925adf9bf14b14615951ce
+size 111226

Bible/10k/en-si/cross_domain_outputs/Jailbreak/en-si.csv

@@ -0,0 +1,2 @@
+model,chrf_sacrebleuapi,chrfpp_sacrebleuapi,bleu_sacrebleuapi,spBLEU-flores101,spBLEU-flores200,comet
+_,7.527470058247617,6.380585805438356,0.0340140157854221,0.07595479204554181,0.052340717422767066,0.3239675502181053