Upload 15 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+eval_nbest_predictions.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,41 @@
 ---
-license: mit
+tags:
+- generated_from_trainer
+datasets:
+- squad_v2
+model-index:
+- name: photocatalysisbert-squad2
+  results: []
 ---
+
+# photocatalysisbert-squad2
+
+This model is a fine-tuned version of [CambridgeMolecularEngineering/photocatalysisbert](https://huggingface.co/CambridgeMolecularEngineering/photocatalysisbert) on the squad_v2 dataset.
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 0
+- distributed_type: multi-GPU
+- num_devices: 20
+- total_train_batch_size: 20
+- total_eval_batch_size: 160
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 2.0
+
+### Framework versions
+
+- Transformers 4.25.1
+- Pytorch 1.12.0a0+git664058f
+- Datasets 2.7.1
+- Tokenizers 0.12.1
+
+## Acknowledgements
+
+This model was trained for the paper "How beneficial is pre-training on a narrow domain-specific corpus for information extraction about photocatalytic water splitting?" by Taketomo Isazawa and Jacqueline M. Cole. J.M.C. is grateful for the BASF/Royal Academy of Engineering Research Chair in Data-Driven Molecular Engineering of Functional Materials, which includes PhD studentship support (for T.I.). This Chair is also partly supported by the Science and Technology Facilities Council. They are also indebted to the Argonne Leadership Computing Facility, which is a DOE Office of Science Facility, for use of its research resources, under contract No. DE-AC02-06CH11357.

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 2.0,
+    "eval_HasAns_exact": 63.107287449392715,
+    "eval_HasAns_f1": 69.28034551777634,
+    "eval_HasAns_total": 5928,
+    "eval_NoAns_exact": 77.54415475189235,
+    "eval_NoAns_f1": 77.54415475189235,
+    "eval_NoAns_total": 5945,
+    "eval_best_exact": 70.34447907015918,
+    "eval_best_exact_thresh": 0.0,
+    "eval_best_f1": 73.42658875005311,
+    "eval_best_f1_thresh": 0.0,
+    "eval_exact": 70.33605659900614,
+    "eval_f1": 73.41816627890007,
+    "eval_samples": 12000,
+    "eval_total": 11873,
+    "train_loss": 1.2332238556411854,
+    "train_runtime": 1237.3872,
+    "train_samples": 130868,
+    "train_samples_per_second": 211.523,
+    "train_steps_per_second": 10.577
+}

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "/grand/projects/SolarWindowsADSP/taketomo/polaris_outputs/final_models/MLM-final-scraped-g",
+  "architectures": [
+    "BertForQuestionAnswering"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.25.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

eval_nbest_predictions.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3689ad3f0e9ef33ecbba7f32b8017c1e16debecbcfb097a630f994359cee42b0
+size 54996272

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 2.0,
+    "eval_HasAns_exact": 63.107287449392715,
+    "eval_HasAns_f1": 69.28034551777634,
+    "eval_HasAns_total": 5928,
+    "eval_NoAns_exact": 77.54415475189235,
+    "eval_NoAns_f1": 77.54415475189235,
+    "eval_NoAns_total": 5945,
+    "eval_best_exact": 70.34447907015918,
+    "eval_best_exact_thresh": 0.0,
+    "eval_best_f1": 73.42658875005311,
+    "eval_best_f1_thresh": 0.0,
+    "eval_exact": 70.33605659900614,
+    "eval_f1": 73.41816627890007,
+    "eval_samples": 12000,
+    "eval_total": 11873
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9260a77caeda080b99a007b1b06db7b52418770a462f6f658694be8743c8404
+size 435643185

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "/grand/projects/SolarWindowsADSP/taketomo/polaris_outputs/final_models/MLM-final-scraped-g",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "train_loss": 1.2332238556411854,
+    "train_runtime": 1237.3872,
+    "train_samples": 130868,
+    "train_samples_per_second": 211.523,
+    "train_steps_per_second": 10.577
+}

trainer_state.json

@@ -0,0 +1,181 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.0,
+  "global_step": 13088,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08,
+      "learning_rate": 2.88539119804401e-05,
+      "loss": 2.3229,
+      "step": 500
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 2.7707823960880195e-05,
+      "loss": 1.7926,
+      "step": 1000
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 2.6561735941320294e-05,
+      "loss": 1.6682,
+      "step": 1500
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 2.5415647921760392e-05,
+      "loss": 1.6135,
+      "step": 2000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 2.4269559902200488e-05,
+      "loss": 1.5343,
+      "step": 2500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.3123471882640587e-05,
+      "loss": 1.4642,
+      "step": 3000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.1977383863080685e-05,
+      "loss": 1.423,
+      "step": 3500
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.083129584352078e-05,
+      "loss": 1.41,
+      "step": 4000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.9685207823960883e-05,
+      "loss": 1.3659,
+      "step": 4500
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.8539119804400978e-05,
+      "loss": 1.3153,
+      "step": 5000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.7393031784841077e-05,
+      "loss": 1.2976,
+      "step": 5500
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.6246943765281176e-05,
+      "loss": 1.2462,
+      "step": 6000
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.5100855745721271e-05,
+      "loss": 1.2594,
+      "step": 6500
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 1.395476772616137e-05,
+      "loss": 1.0087,
+      "step": 7000
+    },
+    {
+      "epoch": 1.15,
+      "learning_rate": 1.2808679706601467e-05,
+      "loss": 1.0272,
+      "step": 7500
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 1.1662591687041566e-05,
+      "loss": 1.0037,
+      "step": 8000
+    },
+    {
+      "epoch": 1.3,
+      "learning_rate": 1.0516503667481663e-05,
+      "loss": 0.968,
+      "step": 8500
+    },
+    {
+      "epoch": 1.38,
+      "learning_rate": 9.37041564792176e-06,
+      "loss": 0.9763,
+      "step": 9000
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 8.224327628361858e-06,
+      "loss": 0.9705,
+      "step": 9500
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 7.078239608801956e-06,
+      "loss": 0.9548,
+      "step": 10000
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 5.932151589242053e-06,
+      "loss": 0.955,
+      "step": 10500
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 4.786063569682151e-06,
+      "loss": 0.932,
+      "step": 11000
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 3.6399755501222492e-06,
+      "loss": 0.9331,
+      "step": 11500
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 2.493887530562347e-06,
+      "loss": 0.9126,
+      "step": 12000
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 1.347799511002445e-06,
+      "loss": 0.909,
+      "step": 12500
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 2.0171149144254277e-07,
+      "loss": 0.89,
+      "step": 13000
+    },
+    {
+      "epoch": 2.0,
+      "step": 13088,
+      "total_flos": 2.5921698247213056e+16,
+      "train_loss": 1.2332238556411854,
+      "train_runtime": 1237.3872,
+      "train_samples_per_second": 211.523,
+      "train_steps_per_second": 10.577
+    }
+  ],
+  "max_steps": 13088,
+  "num_train_epochs": 2,
+  "total_flos": 2.5921698247213056e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3bd0dad9a536e43222030bd8964bf6a32329b194796221bddafad3f310376e4
+size 4271