update model

README.md

@@ -54,19 +54,19 @@ Training Hyperparemeters
 >>> unmasker = pipeline('fill-mask', model='BSC-LT/MrBERT-science')
 
 >>> pprint(unmasker("Hubble's<mask>describes the expansion of the universe and the relationship between a galaxy's distance and its recessional velocity.", top_k=3))
-[{'score': 0.8203125,
+[{'score': 0.8211672902107239,
   'sequence': "Hubble's law describes the expansion of the universe and the "
               "relationship between a galaxy's distance and its recessional "
               'velocity.',
   'token': 21673,
   'token_str': 'law'},
- {'score': 0.1259765625,
+ {'score': 0.16654537618160248,
   'sequence': "Hubble's Law describes the expansion of the universe and the "
               "relationship between a galaxy's distance and its recessional "
               'velocity.',
   'token': 18573,
   'token_str': 'Law'},
- {'score': 0.0247802734375,
+ {'score': 0.0063100955449044704,
   'sequence': "Hubble's equation describes the expansion of the universe and "
               "the relationship between a galaxy's distance and its "
               'recessional velocity.',

config.json

@@ -12,6 +12,7 @@
   "cls_token_id": 1,
   "decoder_bias": true,
   "deterministic_flash_attn": false,
+  "dtype": "float32",
   "embedding_dropout": 0.0,
   "eos_token_id": 2,
   "global_attn_every_n_layers": 3,
@@ -35,9 +36,10 @@
   "num_hidden_layers": 22,
   "pad_token_id": 3,
   "position_embedding_type": "absolute",
+  "repad_logits_with_grad": false,
   "sep_token_id": 2,
-  "tie_word_embeddings": true,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.48.0",
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "transformers_version": "4.56.2",
   "vocab_size": 256128
-}
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:481c17a5a9437d6079367583b7ad88683b6d85156bdeb4d19593ae833b8f6751
-size 1231552912
+oid sha256:c75df29298505934097bae2cee8afee8b66d68437928f2d2247ec4e0eb532e6c
+size 1231552936

special_tokens_map.json

@@ -1,4 +1,7 @@
 {
+  "additional_special_tokens": [
+    "<mask>"
+  ],
   "bos_token": {
     "content": "<s>",
     "lstrip": false,
@@ -15,7 +18,7 @@
   },
   "mask_token": {
     "content": "<mask>",
-    "lstrip": true,
+    "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
@@ -34,4 +37,4 @@
     "rstrip": false,
     "single_word": false
   }
-}
+}

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:031e825b2072f555233d0d01abc2cde072183173a53967e7e842813b9673748c
-size 19092952
+oid sha256:641ca1224e2bfab63b3df7d6c3002384c358e1f28497741c6b943304d15ca9c2
+size 37007965

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5072e3209a04aa01dbf4db72b8fec52cf8cd06a042c9ba819678e084f7b665d5
-size 4813283
+oid sha256:8ddbda5816a0138ffd754cbbfafceba9628342cdd91df4bea6ee86f0fb44eae9
+size 4813260

tokenizer_config.json

@@ -4,7 +4,7 @@
   "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
-      "content": "<s>",
+      "content": "<unk>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -12,7 +12,7 @@
       "special": true
     },
     "1": {
-      "content": "<pad>",
+      "content": "<s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -28,7 +28,7 @@
       "special": true
     },
     "3": {
-      "content": "<unk>",
+      "content": "<pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -52,8 +52,8 @@
       "special": true
     },
     "6": {
-      "content": "<|reserved_token_1|>",
-      "lstrip": true,
+      "content": "<mask>",
+      "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
@@ -828,7 +828,7 @@
       "special": true
     },
     "103": {
-      "content": "<mask>",
+      "content": "<|reserved_token_98|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -844,7 +844,7 @@
       "special": false
     },
     "105": {
-      "content": "\u2581\u2581",
+      "content": "▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -852,7 +852,7 @@
       "special": false
     },
     "106": {
-      "content": "\u2581\u2581\u2581",
+      "content": "▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -860,7 +860,7 @@
       "special": false
     },
     "107": {
-      "content": "\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -868,7 +868,7 @@
       "special": false
     },
     "108": {
-      "content": "\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -876,7 +876,7 @@
       "special": false
     },
     "109": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -884,7 +884,7 @@
       "special": false
     },
     "110": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -892,7 +892,7 @@
       "special": false
     },
     "111": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -900,7 +900,7 @@
       "special": false
     },
     "112": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -908,7 +908,7 @@
       "special": false
     },
     "113": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -916,7 +916,7 @@
       "special": false
     },
     "114": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -924,7 +924,7 @@
       "special": false
     },
     "115": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -932,7 +932,7 @@
       "special": false
     },
     "116": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -940,7 +940,7 @@
       "special": false
     },
     "117": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -948,7 +948,7 @@
       "special": false
     },
     "118": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -956,7 +956,7 @@
       "special": false
     },
     "119": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -964,7 +964,7 @@
       "special": false
     },
     "120": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -972,7 +972,7 @@
       "special": false
     },
     "121": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -980,7 +980,7 @@
       "special": false
     },
     "122": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -988,7 +988,7 @@
       "special": false
     },
     "123": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -996,7 +996,7 @@
       "special": false
     },
     "124": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1004,7 +1004,7 @@
       "special": false
     },
     "125": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1012,7 +1012,7 @@
       "special": false
     },
     "126": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1020,7 +1020,7 @@
       "special": false
     },
     "127": {
-      "content": "\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581\u2581",
+      "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -1084,9 +1084,13 @@
       "special": false
     }
   },
+  "additional_special_tokens": [
+    "<mask>"
+  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
+  "extra_special_tokens": {},
   "legacy": true,
   "mask_token": "<mask>",
   "model_max_length": 8192,
@@ -1095,9 +1099,5 @@
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false,
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ]
-}
+  "use_default_system_prompt": false
+}