Upload folder using huggingface_hub

README.md

@@ -0,0 +1,89 @@
+---
+license: apache-2.0
+base_model: jackaduma/SecBERT
+tags:
+- generated_from_trainer
+metrics:
+- precision
+- recall
+- f1
+- accuracy
+model-index:
+- name: our_data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# our_data
+
+This model is a fine-tuned version of [jackaduma/SecBERT](https://huggingface.co/jackaduma/SecBERT) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.6932
+- Precision: 0.3931
+- Recall: 0.4987
+- F1: 0.4397
+- Accuracy: 0.7295
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 10.0
+
+### Training results
+
+| Training Loss | Epoch | Step  | Validation Loss | Precision | Recall | F1     | Accuracy |
+|:-------------:|:-----:|:-----:|:---------------:|:---------:|:------:|:------:|:--------:|
+| 1.7927        | 0.4   | 500   | 1.5607          | 0.0956    | 0.0780 | 0.0859 | 0.6139   |
+| 1.3551        | 0.81  | 1000  | 1.3530          | 0.2064    | 0.2381 | 0.2211 | 0.6495   |
+| 1.0432        | 1.21  | 1500  | 1.3107          | 0.2269    | 0.3082 | 0.2614 | 0.6740   |
+| 0.8468        | 1.61  | 2000  | 1.2497          | 0.2447    | 0.3373 | 0.2836 | 0.6767   |
+| 0.7775        | 2.01  | 2500  | 1.2710          | 0.2895    | 0.3730 | 0.3260 | 0.6939   |
+| 0.5374        | 2.42  | 3000  | 1.3020          | 0.3006    | 0.4048 | 0.3450 | 0.7044   |
+| 0.5071        | 2.82  | 3500  | 1.2614          | 0.2959    | 0.4048 | 0.3419 | 0.7081   |
+| 0.4237        | 3.22  | 4000  | 1.3251          | 0.3367    | 0.4405 | 0.3817 | 0.7166   |
+| 0.3597        | 3.63  | 4500  | 1.3853          | 0.3423    | 0.4524 | 0.3897 | 0.7125   |
+| 0.3632        | 4.03  | 5000  | 1.4156          | 0.3559    | 0.4524 | 0.3984 | 0.7127   |
+| 0.2589        | 4.43  | 5500  | 1.4472          | 0.3504    | 0.4709 | 0.4018 | 0.7173   |
+| 0.323         | 4.83  | 6000  | 1.3997          | 0.3452    | 0.4603 | 0.3946 | 0.7222   |
+| 0.2167        | 5.24  | 6500  | 1.5194          | 0.3467    | 0.4590 | 0.3950 | 0.7233   |
+| 0.2363        | 5.64  | 7000  | 1.5585          | 0.3507    | 0.4722 | 0.4025 | 0.7222   |
+| 0.2721        | 6.04  | 7500  | 1.5420          | 0.3715    | 0.4854 | 0.4209 | 0.7210   |
+| 0.2073        | 6.45  | 8000  | 1.5878          | 0.3536    | 0.4854 | 0.4091 | 0.7147   |
+| 0.2021        | 6.85  | 8500  | 1.6637          | 0.3722    | 0.4854 | 0.4214 | 0.7197   |
+| 0.1648        | 7.25  | 9000  | 1.6724          | 0.3795    | 0.4788 | 0.4234 | 0.7255   |
+| 0.1927        | 7.66  | 9500  | 1.6891          | 0.3801    | 0.4947 | 0.4299 | 0.7245   |
+| 0.1958        | 8.06  | 10000 | 1.6774          | 0.3937    | 0.4974 | 0.4395 | 0.7281   |
+| 0.1508        | 8.46  | 10500 | 1.7379          | 0.3815    | 0.4854 | 0.4272 | 0.7259   |
+| 0.184         | 8.86  | 11000 | 1.7001          | 0.3863    | 0.5013 | 0.4364 | 0.7277   |
+| 0.1696        | 9.27  | 11500 | 1.6932          | 0.3931    | 0.4987 | 0.4397 | 0.7295   |
+| 0.1425        | 9.67  | 12000 | 1.7137          | 0.3824    | 0.5013 | 0.4339 | 0.7276   |
+
+
+### Framework versions
+
+- Transformers 4.36.0.dev0
+- Pytorch 2.1.0+cu118
+- Datasets 2.15.0
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,25 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7295042897998093,
+    "eval_f1": 0.4396501457725948,
+    "eval_loss": 1.6932308673858643,
+    "eval_precision": 0.3931178310740355,
+    "eval_recall": 0.49867724867724866,
+    "eval_runtime": 1.5381,
+    "eval_samples": 304,
+    "eval_samples_per_second": 197.646,
+    "eval_steps_per_second": 98.823,
+    "predict_accuracy": 0.7786952931461602,
+    "predict_f1": 0.5012853470437018,
+    "predict_loss": 1.3568669557571411,
+    "predict_precision": 0.4452054794520548,
+    "predict_recall": 0.5735294117647058,
+    "predict_runtime": 0.6383,
+    "predict_samples_per_second": 205.235,
+    "predict_steps_per_second": 103.401,
+    "train_loss": 0.4448028106443542,
+    "train_runtime": 816.9339,
+    "train_samples": 2481,
+    "train_samples_per_second": 30.37,
+    "train_steps_per_second": 15.191
+}

config.json

@@ -0,0 +1,113 @@
+{
+  "_name_or_path": "jackaduma/SecBERT",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "finetuning_task": "ner",
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "B-ATTACK_MOTIVATION",
+    "1": "B-ATTACK_PATTERN",
+    "2": "B-ATTACK_RESOURCE_LEVEL",
+    "3": "B-ATTACK_SOPHISTICATION_LEVEL",
+    "4": "B-ATTACK_TOOL",
+    "5": "B-CAMPAIGN",
+    "6": "B-COURSE_OF_ACTION",
+    "7": "B-GENERAL_IDENTITY",
+    "8": "B-GENERAL_TOOL",
+    "9": "B-IMPACT",
+    "10": "B-INDICATOR",
+    "11": "B-INFRASTRUCTURE",
+    "12": "B-INTRUSION_SET",
+    "13": "B-LOCATION",
+    "14": "B-MALWARE",
+    "15": "B-MALWARE_ANALYSIS",
+    "16": "B-OBSERVED_DATA",
+    "17": "B-THREAT_ACTOR",
+    "18": "B-VICTIM_IDENTITY",
+    "19": "B-VULNERABILITY",
+    "20": "I-ATTACK_MOTIVATION",
+    "21": "I-ATTACK_PATTERN",
+    "22": "I-ATTACK_RESOURCE_LEVEL",
+    "23": "I-ATTACK_SOPHISTICATION_LEVEL",
+    "24": "I-ATTACK_TOOL",
+    "25": "I-CAMPAIGN",
+    "26": "I-COURSE_OF_ACTION",
+    "27": "I-GENERAL_IDENTITY",
+    "28": "I-GENERAL_TOOL",
+    "29": "I-IMPACT",
+    "30": "I-INDICATOR",
+    "31": "I-INFRASTRUCTURE",
+    "32": "I-INTRUSION_SET",
+    "33": "I-LOCATION",
+    "34": "I-MALWARE",
+    "35": "I-MALWARE_ANALYSIS",
+    "36": "I-OBSERVED_DATA",
+    "37": "I-THREAT_ACTOR",
+    "38": "I-VICTIM_IDENTITY",
+    "39": "I-VULNERABILITY",
+    "40": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "B-ATTACK_MOTIVATION": 0,
+    "B-ATTACK_PATTERN": 1,
+    "B-ATTACK_RESOURCE_LEVEL": 2,
+    "B-ATTACK_SOPHISTICATION_LEVEL": 3,
+    "B-ATTACK_TOOL": 4,
+    "B-CAMPAIGN": 5,
+    "B-COURSE_OF_ACTION": 6,
+    "B-GENERAL_IDENTITY": 7,
+    "B-GENERAL_TOOL": 8,
+    "B-IMPACT": 9,
+    "B-INDICATOR": 10,
+    "B-INFRASTRUCTURE": 11,
+    "B-INTRUSION_SET": 12,
+    "B-LOCATION": 13,
+    "B-MALWARE": 14,
+    "B-MALWARE_ANALYSIS": 15,
+    "B-OBSERVED_DATA": 16,
+    "B-THREAT_ACTOR": 17,
+    "B-VICTIM_IDENTITY": 18,
+    "B-VULNERABILITY": 19,
+    "I-ATTACK_MOTIVATION": 20,
+    "I-ATTACK_PATTERN": 21,
+    "I-ATTACK_RESOURCE_LEVEL": 22,
+    "I-ATTACK_SOPHISTICATION_LEVEL": 23,
+    "I-ATTACK_TOOL": 24,
+    "I-CAMPAIGN": 25,
+    "I-COURSE_OF_ACTION": 26,
+    "I-GENERAL_IDENTITY": 27,
+    "I-GENERAL_TOOL": 28,
+    "I-IMPACT": 29,
+    "I-INDICATOR": 30,
+    "I-INFRASTRUCTURE": 31,
+    "I-INTRUSION_SET": 32,
+    "I-LOCATION": 33,
+    "I-MALWARE": 34,
+    "I-MALWARE_ANALYSIS": 35,
+    "I-OBSERVED_DATA": 36,
+    "I-THREAT_ACTOR": 37,
+    "I-VICTIM_IDENTITY": 38,
+    "I-VULNERABILITY": 39,
+    "O": 40
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52000
+}

eval_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 10.0,
+    "eval_accuracy": 0.7295042897998093,
+    "eval_f1": 0.4396501457725948,
+    "eval_loss": 1.6932308673858643,
+    "eval_precision": 0.3931178310740355,
+    "eval_recall": 0.49867724867724866,
+    "eval_runtime": 1.5381,
+    "eval_samples": 304,
+    "eval_samples_per_second": 197.646,
+    "eval_steps_per_second": 98.823
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7ff49705b26e73898eda8bcd7215cf41a43fb293d5ac3de088dc795910061e7
+size 331579292

predict_results.json

@@ -0,0 +1,10 @@
+{
+    "predict_accuracy": 0.7786952931461602,
+    "predict_f1": 0.5012853470437018,
+    "predict_loss": 1.3568669557571411,
+    "predict_precision": 0.4452054794520548,
+    "predict_recall": 0.5735294117647058,
+    "predict_runtime": 0.6383,
+    "predict_samples_per_second": 205.235,
+    "predict_steps_per_second": 103.401
+}

predictions.txt

@@ -0,0 +1,131 @@
+O O O B-ATTACK_TOOL O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O B-THREAT_ACTOR O O O O O O O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O O B-INFRASTRUCTURE O O B-CAMPAIGN O O O O O O O O O O O B-INDICATOR O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O
+O O O O O B-INFRASTRUCTURE I-INTRUSION_SET O O O O O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+O O O O O O O O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O O O O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-INFRASTRUCTURE I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-INDICATOR I-INDICATOR I-COURSE_OF_ACTION O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O I-INDICATOR O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O O O I-MALWARE_ANALYSIS O I-MALWARE_ANALYSIS O O B-MALWARE O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O B-THREAT_ACTOR I-THREAT_ACTOR O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O B-VICTIM_IDENTITY O I-ATTACK_MOTIVATION I-VICTIM_IDENTITY I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+O O O O B-VICTIM_IDENTITY O O O O O B-MALWARE O
+O O O B-MALWARE O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O
+O B-MALWARE O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-THREAT_ACTOR O O O B-ATTACK_MOTIVATION O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-VICTIM_IDENTITY B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O B-VICTIM_IDENTITY O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O
+O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-MALWARE I-MALWARE O O O B-VULNERABILITY I-VULNERABILITY O B-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O B-IMPACT I-IMPACT I-IMPACT B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O I-GENERAL_TOOL O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O O O O O O O O O O O B-CAMPAIGN O O O O O O B-VICTIM_IDENTITY O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O O O I-VICTIM_IDENTITY O
+O O O O O O O O O O O I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-INDICATOR O O O B-INTRUSION_SET I-INTRUSION_SET I-INTRUSION_SET O
+O O O O O O O O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O O
+O O O O O O O B-VICTIM_IDENTITY O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O O O O O O O
+O O O O B-THREAT_ACTOR O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O O O O
+O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O B-THREAT_ACTOR I-THREAT_ACTOR O
+O O O B-INTRUSION_SET I-INTRUSION_SET O B-GENERAL_IDENTITY O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY O
+B-THREAT_ACTOR O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O
+B-MALWARE O O O O O O O B-MALWARE I-MALWARE O O O O O O O O B-ATTACK_TOOL O O O O B-ATTACK_TOOL O O I-ATTACK_TOOL O B-INFRASTRUCTURE I-ATTACK_TOOL I-ATTACK_TOOL O O O O O O
+O O O O O O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O
+O O O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O O O O O O O O O O O O B-THREAT_ACTOR I-GENERAL_IDENTITY O
+O O O O I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O O B-ATTACK_TOOL I-MALWARE I-MALWARE I-MALWARE I-MALWARE O O B-OBSERVED_DATA I-OBSERVED_DATA O O I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-ATTACK_PATTERN I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-ATTACK_PATTERN I-MALWARE_ANALYSIS I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O I-INFRASTRUCTURE I-INFRASTRUCTURE O B-GENERAL_IDENTITY O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O O B-THREAT_ACTOR O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O B-THREAT_ACTOR O O O O O O O B-THREAT_ACTOR O
+O O B-THREAT_ACTOR O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS
+O O O O B-CAMPAIGN I-INTRUSION_SET O O O O O B-INTRUSION_SET I-INTRUSION_SET O O O I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT I-IMPACT O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-GENERAL_IDENTITY O B-THREAT_ACTOR O B-INFRASTRUCTURE I-ATTACK_PATTERN I-ATTACK_PATTERN O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O B-CAMPAIGN O
+O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O O B-INFRASTRUCTURE O I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O O B-ATTACK_PATTERN O I-ATTACK_PATTERN O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O O O O O O O O O O I-IMPACT O I-IMPACT O O O
+B-THREAT_ACTOR O B-MALWARE O O O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-MALWARE_ANALYSIS O O O O O O B-INFRASTRUCTURE O
+B-THREAT_ACTOR O B-GENERAL_TOOL O B-INFRASTRUCTURE I-GENERAL_TOOL O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O I-ATTACK_PATTERN O O O O O I-ATTACK_PATTERN O O O B-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O
+B-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O B-GENERAL_IDENTITY O O O B-ATTACK_PATTERN I-INFRASTRUCTURE I-ATTACK_PATTERN I-ATTACK_PATTERN O O I-ATTACK_PATTERN O
+O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-THREAT_ACTOR I-THREAT_ACTOR B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O B-THREAT_ACTOR I-INTRUSION_SET O O O O O O O O O O O
+O O O O O B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O I-ATTACK_TOOL O
+O O O O O B-THREAT_ACTOR O B-ATTACK_TOOL O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O B-INFRASTRUCTURE O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-ATTACK_PATTERN I-ATTACK_TOOL I-ATTACK_TOOL O O B-INFRASTRUCTURE O
+O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O O O O O B-LOCATION O B-LOCATION O O O O O O O O O O O O O
+B-MALWARE I-MALWARE I-OBSERVED_DATA O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O O O O O O O O O O
+O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-ATTACK_TOOL O O O I-OBSERVED_DATA O I-OBSERVED_DATA O O B-INFRASTRUCTURE O I-INFRASTRUCTURE O O O I-OBSERVED_DATA I-OBSERVED_DATA O O O O O O O O
+O O O B-THREAT_ACTOR I-CAMPAIGN O B-VICTIM_IDENTITY I-VICTIM_IDENTITY
+O O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-MALWARE O O O O O O O O O O O O B-ATTACK_TOOL O B-ATTACK_TOOL O O O O O O O O
+O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O
+O O O O O O B-THREAT_ACTOR O O O O O O O O O O O O O O O O O O O O O O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O I-COURSE_OF_ACTION O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-INFRASTRUCTURE O O
+O O O O O O O O O B-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O O O O O O O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O O O I-MALWARE I-MALWARE O O B-THREAT_ACTOR O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O O O O O O O O O O O O O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O
+B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O
+B-OBSERVED_DATA B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O I-MALWARE O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+B-ATTACK_TOOL O O O O O O O O O O O O O O O
+O O O O O B-GENERAL_IDENTITY O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O B-GENERAL_IDENTITY O O O O
+O O I-OBSERVED_DATA O O O B-MALWARE I-MALWARE O O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O I-ATTACK_PATTERN O O B-MALWARE O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O B-OBSERVED_DATA O O O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O O O O O O O O
+O O O O O O B-THREAT_ACTOR B-MALWARE O O O
+O O O O O O O O O O O O O O O O O O O O O O O O O O O I-ATTACK_SOPHISTICATION_LEVEL I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-IMPACT I-COURSE_OF_ACTION I-IMPACT I-IMPACT I-IMPACT O
+B-GENERAL_IDENTITY O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O O O O O O O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O B-MALWARE O O B-MALWARE O O O O O O O O O O
+B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR B-INTRUSION_SET I-THREAT_ACTOR I-CAMPAIGN I-THREAT_ACTOR O O O O O B-THREAT_ACTOR O B-THREAT_ACTOR O O O O O B-CAMPAIGN O O I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O O O I-GENERAL_IDENTITY O B-ATTACK_MOTIVATION O O O B-THREAT_ACTOR O O O O O O O O O O O I-IMPACT B-VICTIM_IDENTITY I-VICTIM_IDENTITY O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O
+O O O O O O O O I-ATTACK_PATTERN O O O B-ATTACK_PATTERN I-ATTACK_PATTERN O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O
+O O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O O O O O O
+O O O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-GENERAL_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O B-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+O O I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-THREAT_ACTOR O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O I-ATTACK_PATTERN O I-ATTACK_PATTERN I-ATTACK_PATTERN O B-ATTACK_PATTERN I-ATTACK_PATTERN O O B-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O O O O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O
+O O O O O O B-ATTACK_TOOL I-ATTACK_TOOL O O O O
+O O O B-VULNERABILITY O O O O O O O O O B-ATTACK_PATTERN O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O
+B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O B-THREAT_ACTOR O O O O O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O B-MALWARE I-MALWARE O O
+B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O O O O O O O O O O O O
+O O O O O O O O O O O O O B-CAMPAIGN I-CAMPAIGN I-CAMPAIGN I-CAMPAIGN O O O O O O
+B-ATTACK_PATTERN B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O B-LOCATION I-LOCATION O O B-GENERAL_TOOL I-GENERAL_TOOL I-GENERAL_TOOL O O O O O O O O O O B-GENERAL_TOOL I-GENERAL_TOOL O
+O O O O B-VICTIM_IDENTITY I-VICTIM_IDENTITY O B-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION I-ATTACK_MOTIVATION O O O O O O O O O O O O O O O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O O O O B-THREAT_ACTOR O
+O O O O O O O O O O O O O O O O O O O O O O O O O O O
+B-ATTACK_TOOL O B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O
+O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O B-CAMPAIGN O O B-VULNERABILITY I-VULNERABILITY O O O O O O O O O O O O O O O O
+O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O B-GENERAL_TOOL O O O B-GENERAL_TOOL O
+O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE O B-INFRASTRUCTURE O O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE I-INFRASTRUCTURE O B-COURSE_OF_ACTION I-COURSE_OF_ACTION O O B-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR I-INDICATOR O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O O B-GENERAL_TOOL O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+O B-GENERAL_IDENTITY O O O O O O O O O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+O O O O B-MALWARE I-MALWARE O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O O O O O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O O B-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR I-THREAT_ACTOR O
+B-INDICATOR O O B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION B-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION I-COURSE_OF_ACTION O
+B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O
+O O O O O O B-THREAT_ACTOR O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O O O O B-LOCATION I-LOCATION O O O O B-LOCATION O O O
+B-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O O O O O O O O O O O O O O O O B-MALWARE O O O B-MALWARE O B-MALWARE O O O O O B-GENERAL_IDENTITY O O O O O O O O O O O O O O O O O O O O
+O O O B-VULNERABILITY O O O O O O O O O B-ATTACK_PATTERN O
+O O O O O O B-LOCATION I-GENERAL_IDENTITY O O O O O B-GENERAL_IDENTITY I-GENERAL_IDENTITY O O O O O O B-CAMPAIGN O O I-INTRUSION_SET O O O O B-VICTIM_IDENTITY I-ATTACK_PATTERN I-INFRASTRUCTURE I-INFRASTRUCTURE O O B-INFRASTRUCTURE I-INFRASTRUCTURE I-ATTACK_PATTERN I-ATTACK_PATTERN O O O O O O O O O O O O O O O
+O O O O O O O B-ATTACK_PATTERN O O O I-ATTACK_PATTERN O O O O B-ATTACK_MOTIVATION I-ATTACK_PATTERN O I-ATTACK_MOTIVATION O I-ATTACK_MOTIVATION I-ATTACK_PATTERN O O B-ATTACK_MOTIVATION O
+B-THREAT_ACTOR O O O B-ATTACK_TOOL I-ATTACK_PATTERN O O I-ATTACK_PATTERN I-ATTACK_PATTERN O O I-ATTACK_PATTERN I-ATTACK_PATTERN O
+B-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN I-ATTACK_PATTERN O
+O O O O O O O O O O O
+B-GENERAL_IDENTITY B-GENERAL_IDENTITY O O O O O O B-GENERAL_IDENTITY O I-THREAT_ACTOR O O O O O O O O O O O
+O O O O O O O O O B-MALWARE O O O O B-CAMPAIGN I-CAMPAIGN O O B-THREAT_ACTOR I-THREAT_ACTOR O O O O B-ATTACK_TOOL I-ATTACK_TOOL I-ATTACK_TOOL O O O O O O O O O O O
+O O O O O O O O O B-VULNERABILITY I-VULNERABILITY I-VULNERABILITY O O O O O O I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA I-OBSERVED_DATA O
+O O O O O O O O O O O O O O O O O O O
+O O O O B-MALWARE O B-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS I-MALWARE_ANALYSIS O

runs/Dec01_00-18-01_7680f78f04a8/events.out.tfevents.1701389896.7680f78f04a8.29450.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a7438df0e1fd09230b93bfd70fc06d3e129cab3c3b011dc02df094481a06608
+size 22104

runs/Dec01_00-18-01_7680f78f04a8/events.out.tfevents.1701390716.7680f78f04a8.29450.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6f8dcb9c076c1b297c05cd433043876ab8912c11ad776283f4668cba2faafe0
+size 560

runs/Nov21_18-57-58_1fcd30357cc4/events.out.tfevents.1700593089.1fcd30357cc4.33963.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e699cb18dfa534c4761bc7a4b7fe10d89dd0c81bd193b534aff45c2342bd83
+size 34027

runs/Nov21_18-57-58_1fcd30357cc4/events.out.tfevents.1700593877.1fcd30357cc4.33963.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f7b4d5952d42dde8c7e86c0cef7f2a51d8d2e085e63dc277b764697060cacbe
+size 560

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,57 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 10.0,
+    "train_loss": 0.4448028106443542,
+    "train_runtime": 816.9339,
+    "train_samples": 2481,
+    "train_samples_per_second": 30.37,
+    "train_steps_per_second": 15.191
+}

trainer_state.json

@@ -0,0 +1,461 @@
+{
+  "best_metric": 0.4396501457725948,
+  "best_model_checkpoint": "/content/our_data/checkpoint-11500",
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 12410,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.91941982272361e-05,
+      "loss": 1.7927,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_accuracy": 0.6139180171591992,
+      "eval_f1": 0.08594319009468317,
+      "eval_loss": 1.5607472658157349,
+      "eval_precision": 0.09562398703403566,
+      "eval_recall": 0.07804232804232804,
+      "eval_runtime": 1.9857,
+      "eval_samples_per_second": 153.092,
+      "eval_steps_per_second": 76.546,
+      "step": 500
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.83883964544722e-05,
+      "loss": 1.3551,
+      "step": 1000
+    },
+    {
+      "epoch": 0.81,
+      "eval_accuracy": 0.6495471877979028,
+      "eval_f1": 0.22113022113022113,
+      "eval_loss": 1.3530131578445435,
+      "eval_precision": 0.20642201834862386,
+      "eval_recall": 0.23809523809523808,
+      "eval_runtime": 1.4676,
+      "eval_samples_per_second": 207.142,
+      "eval_steps_per_second": 103.571,
+      "step": 1000
+    },
+    {
+      "epoch": 1.21,
+      "learning_rate": 1.75825946817083e-05,
+      "loss": 1.0432,
+      "step": 1500
+    },
+    {
+      "epoch": 1.21,
+      "eval_accuracy": 0.6739752144899904,
+      "eval_f1": 0.26135726303982054,
+      "eval_loss": 1.310741662979126,
+      "eval_precision": 0.22687439143135346,
+      "eval_recall": 0.3082010582010582,
+      "eval_runtime": 1.4924,
+      "eval_samples_per_second": 203.693,
+      "eval_steps_per_second": 101.847,
+      "step": 1500
+    },
+    {
+      "epoch": 1.61,
+      "learning_rate": 1.67767929089444e-05,
+      "loss": 0.8468,
+      "step": 2000
+    },
+    {
+      "epoch": 1.61,
+      "eval_accuracy": 0.6767159199237369,
+      "eval_f1": 0.28364849833147937,
+      "eval_loss": 1.249666690826416,
+      "eval_precision": 0.24472168905950095,
+      "eval_recall": 0.3373015873015873,
+      "eval_runtime": 1.9366,
+      "eval_samples_per_second": 156.977,
+      "eval_steps_per_second": 78.489,
+      "step": 2000
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 1.59709911361805e-05,
+      "loss": 0.7775,
+      "step": 2500
+    },
+    {
+      "epoch": 2.01,
+      "eval_accuracy": 0.6938751191611058,
+      "eval_f1": 0.3260115606936416,
+      "eval_loss": 1.2709521055221558,
+      "eval_precision": 0.28952772073921973,
+      "eval_recall": 0.373015873015873,
+      "eval_runtime": 2.1473,
+      "eval_samples_per_second": 141.572,
+      "eval_steps_per_second": 70.786,
+      "step": 2500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.5165189363416601e-05,
+      "loss": 0.5374,
+      "step": 3000
+    },
+    {
+      "epoch": 2.42,
+      "eval_accuracy": 0.7043612964728313,
+      "eval_f1": 0.34498308906426156,
+      "eval_loss": 1.3020099401474,
+      "eval_precision": 0.3005893909626719,
+      "eval_recall": 0.40476190476190477,
+      "eval_runtime": 1.4555,
+      "eval_samples_per_second": 208.867,
+      "eval_steps_per_second": 104.433,
+      "step": 3000
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.4359387590652701e-05,
+      "loss": 0.5071,
+      "step": 3500
+    },
+    {
+      "epoch": 2.82,
+      "eval_accuracy": 0.7080552907530981,
+      "eval_f1": 0.34189944134078215,
+      "eval_loss": 1.2613815069198608,
+      "eval_precision": 0.29593810444874274,
+      "eval_recall": 0.40476190476190477,
+      "eval_runtime": 1.655,
+      "eval_samples_per_second": 183.688,
+      "eval_steps_per_second": 91.844,
+      "step": 3500
+    },
+    {
+      "epoch": 3.22,
+      "learning_rate": 1.35535858178888e-05,
+      "loss": 0.4237,
+      "step": 4000
+    },
+    {
+      "epoch": 3.22,
+      "eval_accuracy": 0.7166348903717826,
+      "eval_f1": 0.38166189111747856,
+      "eval_loss": 1.3250571489334106,
+      "eval_precision": 0.3367037411526795,
+      "eval_recall": 0.44047619047619047,
+      "eval_runtime": 1.9058,
+      "eval_samples_per_second": 159.516,
+      "eval_steps_per_second": 79.758,
+      "step": 4000
+    },
+    {
+      "epoch": 3.63,
+      "learning_rate": 1.27477840451249e-05,
+      "loss": 0.3597,
+      "step": 4500
+    },
+    {
+      "epoch": 3.63,
+      "eval_accuracy": 0.7124642516682554,
+      "eval_f1": 0.38974358974358975,
+      "eval_loss": 1.3852567672729492,
+      "eval_precision": 0.34234234234234234,
+      "eval_recall": 0.4523809523809524,
+      "eval_runtime": 1.478,
+      "eval_samples_per_second": 205.686,
+      "eval_steps_per_second": 102.843,
+      "step": 4500
+    },
+    {
+      "epoch": 4.03,
+      "learning_rate": 1.1941982272361e-05,
+      "loss": 0.3632,
+      "step": 5000
+    },
+    {
+      "epoch": 4.03,
+      "eval_accuracy": 0.7127025738798856,
+      "eval_f1": 0.39836924868957485,
+      "eval_loss": 1.415600061416626,
+      "eval_precision": 0.3558792924037461,
+      "eval_recall": 0.4523809523809524,
+      "eval_runtime": 1.5214,
+      "eval_samples_per_second": 199.817,
+      "eval_steps_per_second": 99.908,
+      "step": 5000
+    },
+    {
+      "epoch": 4.43,
+      "learning_rate": 1.11361804995971e-05,
+      "loss": 0.2589,
+      "step": 5500
+    },
+    {
+      "epoch": 4.43,
+      "eval_accuracy": 0.717349857006673,
+      "eval_f1": 0.40180586907449206,
+      "eval_loss": 1.4472498893737793,
+      "eval_precision": 0.35039370078740156,
+      "eval_recall": 0.4708994708994709,
+      "eval_runtime": 1.4818,
+      "eval_samples_per_second": 205.152,
+      "eval_steps_per_second": 102.576,
+      "step": 5500
+    },
+    {
+      "epoch": 4.83,
+      "learning_rate": 1.0330378726833199e-05,
+      "loss": 0.323,
+      "step": 6000
+    },
+    {
+      "epoch": 4.83,
+      "eval_accuracy": 0.7222354623450906,
+      "eval_f1": 0.39455782312925164,
+      "eval_loss": 1.399746298789978,
+      "eval_precision": 0.34523809523809523,
+      "eval_recall": 0.4603174603174603,
+      "eval_runtime": 1.4893,
+      "eval_samples_per_second": 204.116,
+      "eval_steps_per_second": 102.058,
+      "step": 6000
+    },
+    {
+      "epoch": 5.24,
+      "learning_rate": 9.5245769540693e-06,
+      "loss": 0.2167,
+      "step": 6500
+    },
+    {
+      "epoch": 5.24,
+      "eval_accuracy": 0.7233079122974261,
+      "eval_f1": 0.39499146272054636,
+      "eval_loss": 1.519398808479309,
+      "eval_precision": 0.34665334665334663,
+      "eval_recall": 0.458994708994709,
+      "eval_runtime": 1.803,
+      "eval_samples_per_second": 168.606,
+      "eval_steps_per_second": 84.303,
+      "step": 6500
+    },
+    {
+      "epoch": 5.64,
+      "learning_rate": 8.7187751813054e-06,
+      "loss": 0.2363,
+      "step": 7000
+    },
+    {
+      "epoch": 5.64,
+      "eval_accuracy": 0.7222354623450906,
+      "eval_f1": 0.4024802705749719,
+      "eval_loss": 1.5585495233535767,
+      "eval_precision": 0.3506876227897839,
+      "eval_recall": 0.4722222222222222,
+      "eval_runtime": 2.0739,
+      "eval_samples_per_second": 146.582,
+      "eval_steps_per_second": 73.291,
+      "step": 7000
+    },
+    {
+      "epoch": 6.04,
+      "learning_rate": 7.9129734085415e-06,
+      "loss": 0.2721,
+      "step": 7500
+    },
+    {
+      "epoch": 6.04,
+      "eval_accuracy": 0.7210438512869399,
+      "eval_f1": 0.4208715596330276,
+      "eval_loss": 1.5420488119125366,
+      "eval_precision": 0.3714574898785425,
+      "eval_recall": 0.48544973544973546,
+      "eval_runtime": 1.4879,
+      "eval_samples_per_second": 204.311,
+      "eval_steps_per_second": 102.155,
+      "step": 7500
+    },
+    {
+      "epoch": 6.45,
+      "learning_rate": 7.107171635777599e-06,
+      "loss": 0.2073,
+      "step": 8000
+    },
+    {
+      "epoch": 6.45,
+      "eval_accuracy": 0.7147283126787417,
+      "eval_f1": 0.40914158305462656,
+      "eval_loss": 1.5877846479415894,
+      "eval_precision": 0.3535645472061657,
+      "eval_recall": 0.48544973544973546,
+      "eval_runtime": 1.4406,
+      "eval_samples_per_second": 211.016,
+      "eval_steps_per_second": 105.508,
+      "step": 8000
+    },
+    {
+      "epoch": 6.85,
+      "learning_rate": 6.301369863013699e-06,
+      "loss": 0.2021,
+      "step": 8500
+    },
+    {
+      "epoch": 6.85,
+      "eval_accuracy": 0.7197330791229742,
+      "eval_f1": 0.42135476463834676,
+      "eval_loss": 1.6636826992034912,
+      "eval_precision": 0.372210953346856,
+      "eval_recall": 0.48544973544973546,
+      "eval_runtime": 1.7337,
+      "eval_samples_per_second": 175.352,
+      "eval_steps_per_second": 87.676,
+      "step": 8500
+    },
+    {
+      "epoch": 7.25,
+      "learning_rate": 5.495568090249799e-06,
+      "loss": 0.1648,
+      "step": 9000
+    },
+    {
+      "epoch": 7.25,
+      "eval_accuracy": 0.7254528122020972,
+      "eval_f1": 0.42339181286549704,
+      "eval_loss": 1.6723591089248657,
+      "eval_precision": 0.37945492662473795,
+      "eval_recall": 0.47883597883597884,
+      "eval_runtime": 2.0842,
+      "eval_samples_per_second": 145.858,
+      "eval_steps_per_second": 72.929,
+      "step": 9000
+    },
+    {
+      "epoch": 7.66,
+      "learning_rate": 4.689766317485899e-06,
+      "loss": 0.1927,
+      "step": 9500
+    },
+    {
+      "epoch": 7.66,
+      "eval_accuracy": 0.7244995233555768,
+      "eval_f1": 0.4298850574712644,
+      "eval_loss": 1.689092993736267,
+      "eval_precision": 0.3800813008130081,
+      "eval_recall": 0.4947089947089947,
+      "eval_runtime": 1.551,
+      "eval_samples_per_second": 196.006,
+      "eval_steps_per_second": 98.003,
+      "step": 9500
+    },
+    {
+      "epoch": 8.06,
+      "learning_rate": 3.883964544721999e-06,
+      "loss": 0.1958,
+      "step": 10000
+    },
+    {
+      "epoch": 8.06,
+      "eval_accuracy": 0.7280743565300286,
+      "eval_f1": 0.4395090590298071,
+      "eval_loss": 1.67740797996521,
+      "eval_precision": 0.393717277486911,
+      "eval_recall": 0.4973544973544973,
+      "eval_runtime": 1.5248,
+      "eval_samples_per_second": 199.375,
+      "eval_steps_per_second": 99.688,
+      "step": 10000
+    },
+    {
+      "epoch": 8.46,
+      "learning_rate": 3.0781627719580986e-06,
+      "loss": 0.1508,
+      "step": 10500
+    },
+    {
+      "epoch": 8.46,
+      "eval_accuracy": 0.7259294566253575,
+      "eval_f1": 0.4272409778812573,
+      "eval_loss": 1.7378581762313843,
+      "eval_precision": 0.3814968814968815,
+      "eval_recall": 0.48544973544973546,
+      "eval_runtime": 1.5008,
+      "eval_samples_per_second": 202.555,
+      "eval_steps_per_second": 101.278,
+      "step": 10500
+    },
+    {
+      "epoch": 8.86,
+      "learning_rate": 2.2723609991941985e-06,
+      "loss": 0.184,
+      "step": 11000
+    },
+    {
+      "epoch": 8.86,
+      "eval_accuracy": 0.7277168732125834,
+      "eval_f1": 0.43638457109959694,
+      "eval_loss": 1.700131893157959,
+      "eval_precision": 0.38634046890927626,
+      "eval_recall": 0.5013227513227513,
+      "eval_runtime": 1.5178,
+      "eval_samples_per_second": 200.287,
+      "eval_steps_per_second": 100.144,
+      "step": 11000
+    },
+    {
+      "epoch": 9.27,
+      "learning_rate": 1.4665592264302982e-06,
+      "loss": 0.1696,
+      "step": 11500
+    },
+    {
+      "epoch": 9.27,
+      "eval_accuracy": 0.7295042897998093,
+      "eval_f1": 0.4396501457725948,
+      "eval_loss": 1.6932308673858643,
+      "eval_precision": 0.3931178310740355,
+      "eval_recall": 0.49867724867724866,
+      "eval_runtime": 2.1423,
+      "eval_samples_per_second": 141.904,
+      "eval_steps_per_second": 70.952,
+      "step": 11500
+    },
+    {
+      "epoch": 9.67,
+      "learning_rate": 6.607574536663981e-07,
+      "loss": 0.1425,
+      "step": 12000
+    },
+    {
+      "epoch": 9.67,
+      "eval_accuracy": 0.7275977121067684,
+      "eval_f1": 0.433886662850601,
+      "eval_loss": 1.7137079238891602,
+      "eval_precision": 0.38244197780020184,
+      "eval_recall": 0.5013227513227513,
+      "eval_runtime": 2.2155,
+      "eval_samples_per_second": 137.215,
+      "eval_steps_per_second": 68.608,
+      "step": 12000
+    },
+    {
+      "epoch": 10.0,
+      "step": 12410,
+      "total_flos": 243842156652198.0,
+      "train_loss": 0.4448028106443542,
+      "train_runtime": 816.9339,
+      "train_samples_per_second": 30.37,
+      "train_steps_per_second": 15.191
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 12410,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "total_flos": 243842156652198.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a01377110804395519c90d1af4fb5afeb3515f17bf7f70061a6fd4c723c59944
+size 4664