Hugging Face's logo

osiria
/
bert-italian-cased-question-answering

Question Answering
Transformers
PyTorch
Safetensors
Italian
bert
Eval Results
Inference Endpoints
Model card Files Community
1
osiria commited on
Commit
07a09dc
·
1 Parent(s): cef0960

Delete osiria_bert_italian_cased_qa_evaluation.ipynb

Browse files
Files changed (1) hide show
  1. osiria_bert_italian_cased_qa_evaluation.ipynb +0 -788
osiria_bert_italian_cased_qa_evaluation.ipynb DELETED
@@ -1,788 +0,0 @@
1
- {
2
- "nbformat": 4,
3
- "nbformat_minor": 0,
4
- "metadata": {
5
- "colab": {
6
- "provenance": [],
7
- "gpuType": "T4"
8
- },
9
- "kernelspec": {
10
- "name": "python3",
11
- "display_name": "Python 3"
12
- },
13
- "language_info": {
14
- "name": "python"
15
- },
16
- "accelerator": "GPU"
17
- },
18
- "cells": [
19
- {
20
- "cell_type": "markdown",
21
- "source": [
22
- "# osiria/bert-italian-cased-question-answering\n",
23
- "## test set metrics"
24
- ],
25
- "metadata": {
26
- "id": "oub2ir8ZDkrH"
27
- }
28
- },
29
- {
30
- "cell_type": "code",
31
- "execution_count": 1,
32
- "metadata": {
33
- "colab": {
34
- "base_uri": "https://localhost:8080/"
35
- },
36
- "id": "02K9xO6Gvto1",
37
- "outputId": "d913515b-49cc-437e-a9d9-a4ccc4fed2d4"
38
- },
39
- "outputs": [
40
- {
41
- "output_type": "stream",
42
- "name": "stdout",
43
- "text": [
44
- "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
45
- "Collecting git+https://github.com/huggingface/transformers\n",
46
- " Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-0dr9nm3r\n",
47
- " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-0dr9nm3r\n",
48
- " Resolved https://github.com/huggingface/transformers to commit 8f093fb799246f7dd9104ff44728da0c53a9f67a\n",
49
- " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
50
- " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
51
- " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
52
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (3.12.0)\n",
53
- "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (0.15.1)\n",
54
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (1.22.4)\n",
55
- "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (23.1)\n",
56
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (6.0)\n",
57
- "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (2022.10.31)\n",
58
- "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (2.27.1)\n",
59
- "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (0.13.3)\n",
60
- "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (0.3.1)\n",
61
- "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31.0.dev0) (4.65.0)\n",
62
- "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0.dev0) (2023.4.0)\n",
63
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0.dev0) (4.5.0)\n",
64
- "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31.0.dev0) (1.26.15)\n",
65
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31.0.dev0) (2022.12.7)\n",
66
- "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31.0.dev0) (2.0.12)\n",
67
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31.0.dev0) (3.4)\n"
68
- ]
69
- }
70
- ],
71
- "source": [
72
- "!pip install git+https://github.com/huggingface/transformers"
73
- ]
74
- },
75
- {
76
- "cell_type": "code",
77
- "source": [
78
- "!pip install datasets"
79
- ],
80
- "metadata": {
81
- "colab": {
82
- "base_uri": "https://localhost:8080/"
83
- },
84
- "id": "VKgybrUswa1q",
85
- "outputId": "eaf295cd-277a-4112-e4d2-2083661fbeff"
86
- },
87
- "execution_count": 2,
88
- "outputs": [
89
- {
90
- "output_type": "stream",
91
- "name": "stdout",
92
- "text": [
93
- "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
94
- "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.12.0)\n",
95
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.22.4)\n",
96
- "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
97
- "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.6)\n",
98
- "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
99
- "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.27.1)\n",
100
- "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.65.0)\n",
101
- "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.2.0)\n",
102
- "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.14)\n",
103
- "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.4.0)\n",
104
- "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.4)\n",
105
- "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.15.1)\n",
106
- "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.1)\n",
107
- "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.18.0)\n",
108
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0)\n",
109
- "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
110
- "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.0.12)\n",
111
- "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
112
- "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.2)\n",
113
- "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
114
- "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.3)\n",
115
- "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
116
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (3.12.0)\n",
117
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.11.0->datasets) (4.5.0)\n",
118
- "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
119
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
120
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
121
- "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
122
- "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2022.7.1)\n",
123
- "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n"
124
- ]
125
- }
126
- ]
127
- },
128
- {
129
- "cell_type": "code",
130
- "source": [
131
- "!pip install accelerate"
132
- ],
133
- "metadata": {
134
- "colab": {
135
- "base_uri": "https://localhost:8080/"
136
- },
137
- "id": "xYEW0Hjqz2lo",
138
- "outputId": "e1485ec9-7436-479b-c6ea-c332f5e8742f"
139
- },
140
- "execution_count": 3,
141
- "outputs": [
142
- {
143
- "output_type": "stream",
144
- "name": "stdout",
145
- "text": [
146
- "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
147
- "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.20.3)\n",
148
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.22.4)\n",
149
- "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n",
150
- "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
151
- "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0)\n",
152
- "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
153
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (3.12.0)\n",
154
- "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (4.5.0)\n",
155
- "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (1.11.1)\n",
156
- "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (3.1)\n",
157
- "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (3.1.2)\n",
158
- "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->accelerate) (2.0.0)\n",
159
- "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->accelerate) (3.25.2)\n",
160
- "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->accelerate) (16.0.5)\n",
161
- "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->accelerate) (2.1.2)\n",
162
- "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->accelerate) (1.3.0)\n"
163
- ]
164
- }
165
- ]
166
- },
167
- {
168
- "cell_type": "code",
169
- "source": [
170
- "!pip install evaluate"
171
- ],
172
- "metadata": {
173
- "colab": {
174
- "base_uri": "https://localhost:8080/"
175
- },
176
- "id": "9OFz3JdR0cQ_",
177
- "outputId": "d19842dd-32c0-4c32-a222-44eaad205ba1"
178
- },
179
- "execution_count": 4,
180
- "outputs": [
181
- {
182
- "output_type": "stream",
183
- "name": "stdout",
184
- "text": [
185
- "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
186
- "Requirement already satisfied: evaluate in /usr/local/lib/python3.10/dist-packages (0.4.0)\n",
187
- "Requirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.12.0)\n",
188
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.22.4)\n",
189
- "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.3.6)\n",
190
- "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.5.3)\n",
191
- "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.27.1)\n",
192
- "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from evaluate) (4.65.0)\n",
193
- "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate) (3.2.0)\n",
194
- "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.70.14)\n",
195
- "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2023.4.0)\n",
196
- "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.15.1)\n",
197
- "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate) (23.1)\n",
198
- "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.18.0)\n",
199
- "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
200
- "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (3.8.4)\n",
201
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
202
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.0)\n",
203
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n",
204
- "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (1.26.15)\n",
205
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2022.12.7)\n",
206
- "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2.0.12)\n",
207
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.4)\n",
208
- "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2.8.2)\n",
209
- "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2022.7.1)\n",
210
- "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n",
211
- "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
212
- "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.2)\n",
213
- "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\n",
214
- "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.3)\n",
215
- "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
216
- "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n"
217
- ]
218
- }
219
- ]
220
- },
221
- {
222
- "cell_type": "code",
223
- "source": [
224
- "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/run_qa.py\n",
225
- "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/trainer_qa.py\n",
226
- "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/utils_qa.py"
227
- ],
228
- "metadata": {
229
- "colab": {
230
- "base_uri": "https://localhost:8080/"
231
- },
232
- "id": "JJhpBNwpxOTK",
233
- "outputId": "cfa59bca-e9fd-4ea8-a008-8bf30cef3a9d"
234
- },
235
- "execution_count": 5,
236
- "outputs": [
237
- {
238
- "output_type": "stream",
239
- "name": "stdout",
240
- "text": [
241
- "--2023-06-11 20:53:20-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/run_qa.py\n",
242
- "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.110.133, ...\n",
243
- "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
244
- "HTTP request sent, awaiting response... 200 OK\n",
245
- "Length: 31717 (31K) [text/plain]\n",
246
- "Saving to: ‘run_qa.py.2’\n",
247
- "\n",
248
- "\rrun_qa.py.2 0%[ ] 0 --.-KB/s \rrun_qa.py.2 100%[===================>] 30.97K --.-KB/s in 0.002s \n",
249
- "\n",
250
- "2023-06-11 20:53:20 (13.6 MB/s) - ‘run_qa.py.2’ saved [31717/31717]\n",
251
- "\n",
252
- "--2023-06-11 20:53:20-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/trainer_qa.py\n",
253
- "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
254
- "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
255
- "HTTP request sent, awaiting response... 200 OK\n",
256
- "Length: 6253 (6.1K) [text/plain]\n",
257
- "Saving to: ‘trainer_qa.py.2’\n",
258
- "\n",
259
- "trainer_qa.py.2 100%[===================>] 6.11K --.-KB/s in 0s \n",
260
- "\n",
261
- "2023-06-11 20:53:20 (86.0 MB/s) - ‘trainer_qa.py.2’ saved [6253/6253]\n",
262
- "\n",
263
- "--2023-06-11 20:53:20-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/utils_qa.py\n",
264
- "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
265
- "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
266
- "HTTP request sent, awaiting response... 200 OK\n",
267
- "Length: 22777 (22K) [text/plain]\n",
268
- "Saving to: ‘utils_qa.py.2’\n",
269
- "\n",
270
- "utils_qa.py.2 100%[===================>] 22.24K --.-KB/s in 0.002s \n",
271
- "\n",
272
- "2023-06-11 20:53:20 (14.0 MB/s) - ‘utils_qa.py.2’ saved [22777/22777]\n",
273
- "\n"
274
- ]
275
- }
276
- ]
277
- },
278
- {
279
- "cell_type": "code",
280
- "source": [
281
- "!ls"
282
- ],
283
- "metadata": {
284
- "colab": {
285
- "base_uri": "https://localhost:8080/"
286
- },
287
- "id": "Loa6y9XqzXJ4",
288
- "outputId": "b39fa79a-5cd7-4fa2-9426-522048f4b76f"
289
- },
290
- "execution_count": 6,
291
- "outputs": [
292
- {
293
- "output_type": "stream",
294
- "name": "stdout",
295
- "text": [
296
- "eval_results run_qa.py.1 trainer_qa.py utils_qa.py\n",
297
- "__pycache__ run_qa.py.2 trainer_qa.py.1 utils_qa.py.1\n",
298
- "run_qa.py sample_data trainer_qa.py.2 utils_qa.py.2\n"
299
- ]
300
- }
301
- ]
302
- },
303
- {
304
- "cell_type": "code",
305
- "source": [
306
- "!mkdir eval_results"
307
- ],
308
- "metadata": {
309
- "id": "QDNxV31P0reW",
310
- "colab": {
311
- "base_uri": "https://localhost:8080/"
312
- },
313
- "outputId": "f2054dd3-7097-46c9-f192-ea87d32285f7"
314
- },
315
- "execution_count": 7,
316
- "outputs": [
317
- {
318
- "output_type": "stream",
319
- "name": "stdout",
320
- "text": [
321
- "mkdir: cannot create directory ‘eval_results’: File exists\n"
322
- ]
323
- }
324
- ]
325
- },
326
- {
327
- "cell_type": "code",
328
- "source": [
329
- "!python -m run_qa --model_name_or_path osiria/bert-italian-cased-question-answering --dataset_name squad_it --do_predict --per_device_eval_batch_size=5 --output_dir eval_results"
330
- ],
331
- "metadata": {
332
- "colab": {
333
- "base_uri": "https://localhost:8080/"
334
- },
335
- "id": "B5O5Lom8x_c4",
336
- "outputId": "e565d920-7313-43dd-f5bb-3d3d454fb377"
337
- },
338
- "execution_count": 8,
339
- "outputs": [
340
- {
341
- "output_type": "stream",
342
- "name": "stdout",
343
- "text": [
344
- "2023-06-11 20:53:25.728657: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
345
- "06/11/2023 20:53:27 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False\n",
346
- "06/11/2023 20:53:27 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n",
347
- "_n_gpu=1,\n",
348
- "adafactor=False,\n",
349
- "adam_beta1=0.9,\n",
350
- "adam_beta2=0.999,\n",
351
- "adam_epsilon=1e-08,\n",
352
- "auto_find_batch_size=False,\n",
353
- "bf16=False,\n",
354
- "bf16_full_eval=False,\n",
355
- "data_seed=None,\n",
356
- "dataloader_drop_last=False,\n",
357
- "dataloader_num_workers=0,\n",
358
- "dataloader_pin_memory=True,\n",
359
- "ddp_backend=None,\n",
360
- "ddp_bucket_cap_mb=None,\n",
361
- "ddp_find_unused_parameters=None,\n",
362
- "ddp_timeout=1800,\n",
363
- "debug=[],\n",
364
- "deepspeed=None,\n",
365
- "disable_tqdm=False,\n",
366
- "do_eval=False,\n",
367
- "do_predict=True,\n",
368
- "do_train=False,\n",
369
- "eval_accumulation_steps=None,\n",
370
- "eval_delay=0,\n",
371
- "eval_steps=None,\n",
372
- "evaluation_strategy=no,\n",
373
- "fp16=False,\n",
374
- "fp16_backend=auto,\n",
375
- "fp16_full_eval=False,\n",
376
- "fp16_opt_level=O1,\n",
377
- "fsdp=[],\n",
378
- "fsdp_config={'fsdp_min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n",
379
- "fsdp_min_num_params=0,\n",
380
- "fsdp_transformer_layer_cls_to_wrap=None,\n",
381
- "full_determinism=False,\n",
382
- "gradient_accumulation_steps=1,\n",
383
- "gradient_checkpointing=False,\n",
384
- "greater_is_better=None,\n",
385
- "group_by_length=False,\n",
386
- "half_precision_backend=auto,\n",
387
- "hub_model_id=None,\n",
388
- "hub_private_repo=False,\n",
389
- "hub_strategy=every_save,\n",
390
- "hub_token=<HUB_TOKEN>,\n",
391
- "ignore_data_skip=False,\n",
392
- "include_inputs_for_metrics=False,\n",
393
- "jit_mode_eval=False,\n",
394
- "label_names=None,\n",
395
- "label_smoothing_factor=0.0,\n",
396
- "learning_rate=5e-05,\n",
397
- "length_column_name=length,\n",
398
- "load_best_model_at_end=False,\n",
399
- "local_rank=0,\n",
400
- "log_level=passive,\n",
401
- "log_level_replica=warning,\n",
402
- "log_on_each_node=True,\n",
403
- "logging_dir=eval_results/runs/Jun11_20-53-27_6b4f0ceaae37,\n",
404
- "logging_first_step=False,\n",
405
- "logging_nan_inf_filter=True,\n",
406
- "logging_steps=500,\n",
407
- "logging_strategy=steps,\n",
408
- "lr_scheduler_type=linear,\n",
409
- "max_grad_norm=1.0,\n",
410
- "max_steps=-1,\n",
411
- "metric_for_best_model=None,\n",
412
- "mp_parameters=,\n",
413
- "no_cuda=False,\n",
414
- "num_train_epochs=3.0,\n",
415
- "optim=adamw_hf,\n",
416
- "optim_args=None,\n",
417
- "output_dir=eval_results,\n",
418
- "overwrite_output_dir=False,\n",
419
- "past_index=-1,\n",
420
- "per_device_eval_batch_size=5,\n",
421
- "per_device_train_batch_size=8,\n",
422
- "prediction_loss_only=False,\n",
423
- "push_to_hub=False,\n",
424
- "push_to_hub_model_id=None,\n",
425
- "push_to_hub_organization=None,\n",
426
- "push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
427
- "ray_scope=last,\n",
428
- "remove_unused_columns=True,\n",
429
- "report_to=['tensorboard'],\n",
430
- "resume_from_checkpoint=None,\n",
431
- "run_name=eval_results,\n",
432
- "save_on_each_node=False,\n",
433
- "save_safetensors=False,\n",
434
- "save_steps=500,\n",
435
- "save_strategy=steps,\n",
436
- "save_total_limit=None,\n",
437
- "seed=42,\n",
438
- "sharded_ddp=[],\n",
439
- "skip_memory_metrics=True,\n",
440
- "tf32=None,\n",
441
- "torch_compile=False,\n",
442
- "torch_compile_backend=None,\n",
443
- "torch_compile_mode=None,\n",
444
- "torchdynamo=None,\n",
445
- "tpu_metrics_debug=False,\n",
446
- "tpu_num_cores=None,\n",
447
- "use_ipex=False,\n",
448
- "use_legacy_prediction_loop=False,\n",
449
- "use_mps_device=False,\n",
450
- "warmup_ratio=0.0,\n",
451
- "warmup_steps=0,\n",
452
- "weight_decay=0.0,\n",
453
- "xpu_backend=None,\n",
454
- ")\n",
455
- "06/11/2023 20:53:28 - INFO - datasets.info - Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/squad_it/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71\n",
456
- "06/11/2023 20:53:28 - INFO - datasets.builder - Overwrite dataset info from restored data version if exists.\n",
457
- "06/11/2023 20:53:28 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71\n",
458
- "06/11/2023 20:53:28 - WARNING - datasets.builder - Found cached dataset squad_it (/root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71)\n",
459
- "06/11/2023 20:53:28 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71\n",
460
- "100% 2/2 [00:00<00:00, 540.12it/s]\n",
461
- "[INFO|configuration_utils.py:669] 2023-06-11 20:53:28,527 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-cased-question-answering/snapshots/d289e6216d2c615b2e1eb5dd25cb7239c51b5088/config.json\n",
462
- "[INFO|configuration_utils.py:725] 2023-06-11 20:53:28,530 >> Model config BertConfig {\n",
463
- " \"_name_or_path\": \"osiria/bert-italian-cased-question-answering\",\n",
464
- " \"architectures\": [\n",
465
- " \"BertForQuestionAnswering\"\n",
466
- " ],\n",
467
- " \"attention_probs_dropout_prob\": 0.1,\n",
468
- " \"classifier_dropout\": null,\n",
469
- " \"directionality\": \"bidi\",\n",
470
- " \"hidden_act\": \"gelu\",\n",
471
- " \"hidden_dropout_prob\": 0.1,\n",
472
- " \"hidden_size\": 768,\n",
473
- " \"initializer_range\": 0.02,\n",
474
- " \"intermediate_size\": 3072,\n",
475
- " \"layer_norm_eps\": 1e-12,\n",
476
- " \"max_position_embeddings\": 512,\n",
477
- " \"model_type\": \"bert\",\n",
478
- " \"num_attention_heads\": 12,\n",
479
- " \"num_hidden_layers\": 12,\n",
480
- " \"pad_token_id\": 0,\n",
481
- " \"pooler_fc_size\": 768,\n",
482
- " \"pooler_num_attention_heads\": 12,\n",
483
- " \"pooler_num_fc_layers\": 3,\n",
484
- " \"pooler_size_per_head\": 128,\n",
485
- " \"pooler_type\": \"first_token_transform\",\n",
486
- " \"position_embedding_type\": \"absolute\",\n",
487
- " \"torch_dtype\": \"float32\",\n",
488
- " \"transformers_version\": \"4.31.0.dev0\",\n",
489
- " \"type_vocab_size\": 2,\n",
490
- " \"use_cache\": true,\n",
491
- " \"vocab_size\": 30785\n",
492
- "}\n",
493
- "\n",
494
- "[INFO|tokenization_utils_base.py:1823] 2023-06-11 20:53:29,081 >> loading file vocab.txt from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-cased-question-answering/snapshots/d289e6216d2c615b2e1eb5dd25cb7239c51b5088/vocab.txt\n",
495
- "[INFO|tokenization_utils_base.py:1823] 2023-06-11 20:53:29,082 >> loading file tokenizer.json from cache at None\n",
496
- "[INFO|tokenization_utils_base.py:1823] 2023-06-11 20:53:29,082 >> loading file added_tokens.json from cache at None\n",
497
- "[INFO|tokenization_utils_base.py:1823] 2023-06-11 20:53:29,082 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-cased-question-answering/snapshots/d289e6216d2c615b2e1eb5dd25cb7239c51b5088/special_tokens_map.json\n",
498
- "[INFO|tokenization_utils_base.py:1823] 2023-06-11 20:53:29,082 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-cased-question-answering/snapshots/d289e6216d2c615b2e1eb5dd25cb7239c51b5088/tokenizer_config.json\n",
499
- "[INFO|modeling_utils.py:2578] 2023-06-11 20:53:29,866 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-cased-question-answering/snapshots/d289e6216d2c615b2e1eb5dd25cb7239c51b5088/pytorch_model.bin\n",
500
- "[INFO|modeling_utils.py:3295] 2023-06-11 20:53:31,723 >> All model checkpoint weights were used when initializing BertForQuestionAnswering.\n",
501
- "\n",
502
- "[INFO|modeling_utils.py:3303] 2023-06-11 20:53:31,723 >> All the weights of BertForQuestionAnswering were initialized from the model checkpoint at osiria/bert-italian-cased-question-answering.\n",
503
- "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForQuestionAnswering for predictions without further training.\n",
504
- "Running tokenizer on prediction dataset: 0% 0/7609 [00:00<?, ? examples/s]06/11/2023 20:53:32 - INFO - datasets.arrow_dataset - Caching processed dataset at /root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71/cache-f05ac3c41e20dd08.arrow\n",
505
- "06/11/2023 20:53:41 - INFO - __main__ - *** Predict ***\n",
506
- "[INFO|trainer.py:776] 2023-06-11 20:53:41,522 >> The following columns in the test set don't have a corresponding argument in `BertForQuestionAnswering.forward` and have been ignored: offset_mapping, example_id. If offset_mapping, example_id are not expected by `BertForQuestionAnswering.forward`, you can safely ignore this message.\n",
507
- "[INFO|trainer.py:3200] 2023-06-11 20:53:41,526 >> ***** Running Prediction *****\n",
508
- "[INFO|trainer.py:3202] 2023-06-11 20:53:41,526 >> Num examples = 7992\n",
509
- "[INFO|trainer.py:3205] 2023-06-11 20:53:41,526 >> Batch size = 5\n",
510
- "100% 1598/1599 [03:13<00:00, 7.89it/s]06/11/2023 20:57:07 - INFO - utils_qa - Post-processing 7609 example predictions split into 7992 features.\n",
511
- "\n",
512
- " 0% 0/7609 [00:00<?, ?it/s]\u001b[A\n",
513
- " 0% 20/7609 [00:00<00:38, 195.14it/s]\u001b[A\n",
514
- " 1% 41/7609 [00:00<00:37, 203.05it/s]\u001b[A\n",
515
- " 1% 72/7609 [00:00<00:30, 249.92it/s]\u001b[A\n",
516
- " 1% 102/7609 [00:00<00:27, 268.69it/s]\u001b[A\n",
517
- " 2% 133/7609 [00:00<00:26, 281.18it/s]\u001b[A\n",
518
- " 2% 164/7609 [00:00<00:25, 289.10it/s]\u001b[A\n",
519
- " 3% 196/7609 [00:00<00:24, 298.41it/s]\u001b[A\n",
520
- " 3% 232/7609 [00:00<00:23, 315.67it/s]\u001b[A\n",
521
- " 3% 264/7609 [00:00<00:23, 313.05it/s]\u001b[A\n",
522
- " 4% 297/7609 [00:01<00:23, 316.84it/s]\u001b[A\n",
523
- " 4% 330/7609 [00:01<00:22, 317.64it/s]\u001b[A\n",
524
- " 5% 362/7609 [00:01<00:24, 301.76it/s]\u001b[A\n",
525
- " 5% 397/7609 [00:01<00:23, 312.78it/s]\u001b[A\n",
526
- " 6% 429/7609 [00:01<00:23, 311.35it/s]\u001b[A\n",
527
- " 6% 461/7609 [00:01<00:22, 312.95it/s]\u001b[A\n",
528
- " 7% 495/7609 [00:01<00:22, 318.76it/s]\u001b[A\n",
529
- " 7% 527/7609 [00:01<00:23, 306.27it/s]\u001b[A\n",
530
- " 7% 558/7609 [00:01<00:23, 303.87it/s]\u001b[A\n",
531
- " 8% 590/7609 [00:01<00:22, 306.50it/s]\u001b[A\n",
532
- " 8% 621/7609 [00:02<00:22, 305.90it/s]\u001b[A\n",
533
- " 9% 655/7609 [00:02<00:22, 315.47it/s]\u001b[A\n",
534
- " 9% 689/7609 [00:02<00:21, 321.92it/s]\u001b[A\n",
535
- " 9% 722/7609 [00:02<00:21, 321.69it/s]\u001b[A\n",
536
- " 10% 755/7609 [00:02<00:21, 312.94it/s]\u001b[A\n",
537
- " 10% 787/7609 [00:02<00:21, 310.60it/s]\u001b[A\n",
538
- " 11% 819/7609 [00:02<00:22, 295.48it/s]\u001b[A\n",
539
- " 11% 853/7609 [00:02<00:21, 307.31it/s]\u001b[A\n",
540
- " 12% 886/7609 [00:02<00:21, 312.94it/s]\u001b[A\n",
541
- " 12% 918/7609 [00:03<00:21, 307.77it/s]\u001b[A\n",
542
- " 12% 949/7609 [00:03<00:21, 307.49it/s]\u001b[A\n",
543
- " 13% 982/7609 [00:03<00:21, 314.01it/s]\u001b[A\n",
544
- " 13% 1014/7609 [00:03<00:20, 314.12it/s]\u001b[A\n",
545
- " 14% 1050/7609 [00:03<00:20, 326.69it/s]\u001b[A\n",
546
- " 14% 1086/7609 [00:03<00:19, 332.63it/s]\u001b[A\n",
547
- " 15% 1120/7609 [00:03<00:20, 314.31it/s]\u001b[A\n",
548
- " 15% 1152/7609 [00:03<00:20, 309.50it/s]\u001b[A\n",
549
- " 16% 1184/7609 [00:03<00:21, 294.61it/s]\u001b[A\n",
550
- " 16% 1218/7609 [00:03<00:20, 305.58it/s]\u001b[A\n",
551
- " 16% 1252/7609 [00:04<00:20, 312.62it/s]\u001b[A\n",
552
- " 17% 1284/7609 [00:04<00:20, 312.65it/s]\u001b[A\n",
553
- " 17% 1317/7609 [00:04<00:19, 316.39it/s]\u001b[A\n",
554
- " 18% 1352/7609 [00:04<00:19, 324.19it/s]\u001b[A\n",
555
- " 18% 1385/7609 [00:04<00:19, 323.06it/s]\u001b[A\n",
556
- " 19% 1418/7609 [00:04<00:19, 310.79it/s]\u001b[A\n",
557
- " 19% 1450/7609 [00:04<00:19, 312.09it/s]\u001b[A\n",
558
- " 19% 1482/7609 [00:04<00:20, 301.57it/s]\u001b[A\n",
559
- " 20% 1513/7609 [00:04<00:20, 296.75it/s]\u001b[A\n",
560
- " 20% 1545/7609 [00:05<00:20, 302.42it/s]\u001b[A\n",
561
- " 21% 1581/7609 [00:05<00:19, 316.37it/s]\u001b[A\n",
562
- " 21% 1613/7609 [00:05<00:19, 303.34it/s]\u001b[A\n",
563
- " 22% 1646/7609 [00:05<00:19, 310.08it/s]\u001b[A\n",
564
- " 22% 1681/7609 [00:05<00:18, 318.67it/s]\u001b[A\n",
565
- " 23% 1715/7609 [00:05<00:18, 323.25it/s]\u001b[A\n",
566
- " 23% 1748/7609 [00:05<00:18, 323.99it/s]\u001b[A\n",
567
- " 23% 1781/7609 [00:05<00:18, 315.58it/s]\u001b[A\n",
568
- " 24% 1815/7609 [00:05<00:18, 320.26it/s]\u001b[A\n",
569
- " 24% 1849/7609 [00:05<00:17, 324.51it/s]\u001b[A\n",
570
- " 25% 1882/7609 [00:06<00:17, 325.18it/s]\u001b[A\n",
571
- " 25% 1915/7609 [00:06<00:17, 321.37it/s]\u001b[A\n",
572
- " 26% 1949/7609 [00:06<00:17, 326.45it/s]\u001b[A\n",
573
- " 26% 1982/7609 [00:06<00:20, 280.92it/s]\u001b[A\n",
574
- " 26% 2012/7609 [00:06<00:23, 241.28it/s]\u001b[A\n",
575
- " 27% 2038/7609 [00:06<00:25, 218.91it/s]\u001b[A\n",
576
- " 27% 2064/7609 [00:06<00:24, 227.96it/s]\u001b[A\n",
577
- " 27% 2088/7609 [00:07<00:32, 171.75it/s]\u001b[A\n",
578
- " 28% 2108/7609 [00:07<00:33, 164.67it/s]\u001b[A\n",
579
- " 28% 2142/7609 [00:07<00:27, 201.85it/s]\u001b[A\n",
580
- " 29% 2179/7609 [00:07<00:22, 240.40it/s]\u001b[A\n",
581
- " 29% 2213/7609 [00:07<00:20, 265.02it/s]\u001b[A\n",
582
- " 29% 2242/7609 [00:07<00:20, 262.30it/s]\u001b[A\n",
583
- " 30% 2270/7609 [00:07<00:20, 264.20it/s]\u001b[A\n",
584
- " 30% 2304/7609 [00:07<00:18, 284.39it/s]\u001b[A\n",
585
- " 31% 2337/7609 [00:07<00:17, 296.90it/s]\u001b[A\n",
586
- " 31% 2373/7609 [00:08<00:16, 314.24it/s]\u001b[A\n",
587
- " 32% 2407/7609 [00:08<00:16, 320.90it/s]\u001b[A\n",
588
- " 32% 2441/7609 [00:08<00:15, 324.50it/s]\u001b[A\n",
589
- " 33% 2474/7609 [00:08<00:16, 305.55it/s]\u001b[A\n",
590
- " 33% 2509/7609 [00:08<00:16, 317.58it/s]\u001b[A\n",
591
- " 33% 2543/7609 [00:08<00:15, 322.81it/s]\u001b[A\n",
592
- " 34% 2576/7609 [00:08<00:15, 321.99it/s]\u001b[A\n",
593
- " 34% 2609/7609 [00:08<00:16, 304.24it/s]\u001b[A\n",
594
- " 35% 2642/7609 [00:08<00:15, 311.37it/s]\u001b[A\n",
595
- " 35% 2676/7609 [00:09<00:15, 318.53it/s]\u001b[A\n",
596
- " 36% 2710/7609 [00:09<00:15, 323.02it/s]\u001b[A\n",
597
- " 36% 2743/7609 [00:09<00:15, 321.88it/s]\u001b[A\n",
598
- " 37% 2778/7609 [00:09<00:14, 328.50it/s]\u001b[A\n",
599
- " 37% 2811/7609 [00:09<00:14, 327.81it/s]\u001b[A\n",
600
- " 37% 2845/7609 [00:09<00:14, 330.00it/s]\u001b[A\n",
601
- " 38% 2881/7609 [00:09<00:14, 337.70it/s]\u001b[A\n",
602
- " 38% 2915/7609 [00:09<00:14, 327.57it/s]\u001b[A\n",
603
- " 39% 2951/7609 [00:09<00:13, 334.48it/s]\u001b[A\n",
604
- " 39% 2986/7609 [00:09<00:13, 336.64it/s]\u001b[A\n",
605
- " 40% 3020/7609 [00:10<00:13, 335.05it/s]\u001b[A\n",
606
- " 40% 3055/7609 [00:10<00:13, 338.07it/s]\u001b[A\n",
607
- " 41% 3089/7609 [00:10<00:15, 299.60it/s]\u001b[A\n",
608
- " 41% 3120/7609 [00:10<00:18, 238.88it/s]\u001b[A\n",
609
- " 41% 3147/7609 [00:10<00:20, 222.56it/s]\u001b[A\n",
610
- " 42% 3171/7609 [00:10<00:21, 207.53it/s]\u001b[A\n",
611
- " 42% 3193/7609 [00:10<00:25, 173.59it/s]\u001b[A\n",
612
- " 42% 3212/7609 [00:11<00:26, 163.35it/s]\u001b[A\n",
613
- " 42% 3230/7609 [00:11<00:26, 165.02it/s]\u001b[A\n",
614
- " 43% 3250/7609 [00:11<00:25, 171.98it/s]\u001b[A\n",
615
- " 43% 3268/7609 [00:11<00:25, 167.09it/s]\u001b[A\n",
616
- " 43% 3288/7609 [00:11<00:24, 174.21it/s]\u001b[A\n",
617
- " 43% 3307/7609 [00:11<00:24, 177.85it/s]\u001b[A\n",
618
- " 44% 3326/7609 [00:11<00:25, 167.85it/s]\u001b[A\n",
619
- " 44% 3345/7609 [00:11<00:24, 172.38it/s]\u001b[A\n",
620
- " 44% 3364/7609 [00:11<00:24, 176.78it/s]\u001b[A\n",
621
- " 44% 3382/7609 [00:12<00:24, 173.27it/s]\u001b[A\n",
622
- " 45% 3401/7609 [00:12<00:23, 177.82it/s]\u001b[A\n",
623
- " 45% 3421/7609 [00:12<00:22, 183.82it/s]\u001b[A\n",
624
- " 45% 3441/7609 [00:12<00:22, 186.31it/s]\u001b[A\n",
625
- " 45% 3460/7609 [00:12<00:22, 186.86it/s]\u001b[A\n",
626
- " 46% 3480/7609 [00:12<00:21, 189.06it/s]\u001b[A\n",
627
- " 46% 3500/7609 [00:12<00:21, 192.11it/s]\u001b[A\n",
628
- " 46% 3520/7609 [00:12<00:21, 190.39it/s]\u001b[A\n",
629
- " 47% 3540/7609 [00:12<00:21, 192.97it/s]\u001b[A\n",
630
- " 47% 3560/7609 [00:13<00:21, 191.51it/s]\u001b[A\n",
631
- " 47% 3580/7609 [00:13<00:20, 193.29it/s]\u001b[A\n",
632
- " 47% 3600/7609 [00:13<00:20, 193.66it/s]\u001b[A\n",
633
- " 48% 3620/7609 [00:13<00:20, 192.90it/s]\u001b[A\n",
634
- " 48% 3640/7609 [00:13<00:21, 187.99it/s]\u001b[A\n",
635
- " 48% 3661/7609 [00:13<00:20, 193.31it/s]\u001b[A\n",
636
- " 48% 3681/7609 [00:13<00:20, 188.25it/s]\u001b[A\n",
637
- " 49% 3702/7609 [00:13<00:20, 191.63it/s]\u001b[A\n",
638
- " 49% 3722/7609 [00:13<00:20, 190.32it/s]\u001b[A\n",
639
- " 49% 3742/7609 [00:13<00:20, 189.32it/s]\u001b[A\n",
640
- " 49% 3762/7609 [00:14<00:20, 191.77it/s]\u001b[A\n",
641
- " 50% 3782/7609 [00:14<00:19, 192.69it/s]\u001b[A\n",
642
- " 50% 3802/7609 [00:14<00:19, 193.14it/s]\u001b[A\n",
643
- " 50% 3831/7609 [00:14<00:17, 220.54it/s]\u001b[A\n",
644
- " 51% 3867/7609 [00:14<00:14, 260.43it/s]\u001b[A\n",
645
- " 51% 3898/7609 [00:14<00:13, 275.01it/s]\u001b[A\n",
646
- " 52% 3933/7609 [00:14<00:12, 296.05it/s]\u001b[A\n",
647
- " 52% 3967/7609 [00:14<00:11, 308.86it/s]\u001b[A\n",
648
- " 53% 3998/7609 [00:14<00:11, 306.03it/s]\u001b[A\n",
649
- " 53% 4033/7609 [00:14<00:11, 316.89it/s]\u001b[A\n",
650
- " 53% 4065/7609 [00:15<00:11, 315.00it/s]\u001b[A\n",
651
- " 54% 4097/7609 [00:15<00:11, 308.17it/s]\u001b[A\n",
652
- " 54% 4131/7609 [00:15<00:10, 316.47it/s]\u001b[A\n",
653
- " 55% 4166/7609 [00:15<00:10, 324.79it/s]\u001b[A\n",
654
- " 55% 4201/7609 [00:15<00:10, 330.88it/s]\u001b[A\n",
655
- " 56% 4235/7609 [00:15<00:10, 332.20it/s]\u001b[A\n",
656
- " 56% 4270/7609 [00:15<00:09, 336.21it/s]\u001b[A\n",
657
- " 57% 4307/7609 [00:15<00:09, 343.82it/s]\u001b[A\n",
658
- " 57% 4342/7609 [00:15<00:09, 336.95it/s]\u001b[A\n",
659
- " 58% 4376/7609 [00:16<00:09, 335.66it/s]\u001b[A\n",
660
- " 58% 4412/7609 [00:16<00:09, 340.46it/s]\u001b[A\n",
661
- " 58% 4447/7609 [00:16<00:09, 322.13it/s]\u001b[A\n",
662
- " 59% 4484/7609 [00:16<00:09, 333.16it/s]\u001b[A\n",
663
- " 59% 4520/7609 [00:16<00:09, 338.43it/s]\u001b[A\n",
664
- " 60% 4556/7609 [00:16<00:08, 342.73it/s]\u001b[A\n",
665
- " 60% 4591/7609 [00:16<00:08, 340.68it/s]\u001b[A\n",
666
- " 61% 4626/7609 [00:16<00:08, 342.85it/s]\u001b[A\n",
667
- " 61% 4663/7609 [00:16<00:08, 349.40it/s]\u001b[A\n",
668
- " 62% 4698/7609 [00:16<00:08, 336.93it/s]\u001b[A\n",
669
- " 62% 4736/7609 [00:17<00:08, 347.12it/s]\u001b[A\n",
670
- " 63% 4771/7609 [00:17<00:08, 342.80it/s]\u001b[A\n",
671
- " 63% 4806/7609 [00:17<00:08, 340.14it/s]\u001b[A\n",
672
- " 64% 4843/7609 [00:17<00:07, 346.67it/s]\u001b[A\n",
673
- " 64% 4879/7609 [00:17<00:07, 349.23it/s]\u001b[A\n",
674
- " 65% 4914/7609 [00:17<00:07, 340.33it/s]\u001b[A\n",
675
- " 65% 4949/7609 [00:17<00:07, 333.04it/s]\u001b[A\n",
676
- " 65% 4983/7609 [00:17<00:07, 332.52it/s]\u001b[A\n",
677
- " 66% 5017/7609 [00:17<00:08, 319.67it/s]\u001b[A\n",
678
- " 66% 5050/7609 [00:18<00:07, 320.93it/s]\u001b[A\n",
679
- " 67% 5084/7609 [00:18<00:07, 324.47it/s]\u001b[A\n",
680
- " 67% 5118/7609 [00:18<00:07, 326.48it/s]\u001b[A\n",
681
- " 68% 5153/7609 [00:18<00:07, 331.63it/s]\u001b[A\n",
682
- " 68% 5187/7609 [00:18<00:07, 333.97it/s]\u001b[A\n",
683
- " 69% 5222/7609 [00:18<00:07, 337.50it/s]\u001b[A\n",
684
- " 69% 5258/7609 [00:18<00:06, 341.40it/s]\u001b[A\n",
685
- " 70% 5293/7609 [00:18<00:06, 340.17it/s]\u001b[A\n",
686
- " 70% 5328/7609 [00:18<00:06, 342.70it/s]\u001b[A\n",
687
- " 70% 5363/7609 [00:18<00:06, 334.83it/s]\u001b[A\n",
688
- " 71% 5397/7609 [00:19<00:06, 334.27it/s]\u001b[A\n",
689
- " 71% 5431/7609 [00:19<00:06, 333.55it/s]\u001b[A\n",
690
- " 72% 5466/7609 [00:19<00:06, 338.03it/s]\u001b[A\n",
691
- " 72% 5500/7609 [00:19<00:06, 335.44it/s]\u001b[A\n",
692
- " 73% 5535/7609 [00:19<00:06, 337.65it/s]\u001b[A\n",
693
- " 73% 5570/7609 [00:19<00:05, 341.28it/s]\u001b[A\n",
694
- " 74% 5605/7609 [00:19<00:05, 338.55it/s]\u001b[A\n",
695
- " 74% 5640/7609 [00:19<00:05, 340.98it/s]\u001b[A\n",
696
- " 75% 5676/7609 [00:19<00:05, 345.49it/s]\u001b[A\n",
697
- " 75% 5711/7609 [00:19<00:05, 337.68it/s]\u001b[A\n",
698
- " 76% 5747/7609 [00:20<00:05, 342.84it/s]\u001b[A\n",
699
- " 76% 5785/7609 [00:20<00:05, 353.49it/s]\u001b[A\n",
700
- " 77% 5821/7609 [00:20<00:05, 352.63it/s]\u001b[A\n",
701
- " 77% 5857/7609 [00:20<00:04, 352.02it/s]\u001b[A\n",
702
- " 77% 5893/7609 [00:20<00:04, 353.00it/s]\u001b[A\n",
703
- " 78% 5929/7609 [00:20<00:04, 345.39it/s]\u001b[A\n",
704
- " 78% 5964/7609 [00:20<00:04, 330.71it/s]\u001b[A\n",
705
- " 79% 6000/7609 [00:20<00:04, 338.18it/s]\u001b[A\n",
706
- " 79% 6034/7609 [00:20<00:04, 327.30it/s]\u001b[A\n",
707
- " 80% 6068/7609 [00:21<00:04, 328.53it/s]\u001b[A\n",
708
- " 80% 6104/7609 [00:21<00:04, 336.55it/s]\u001b[A\n",
709
- " 81% 6138/7609 [00:21<00:04, 304.04it/s]\u001b[A\n",
710
- " 81% 6170/7609 [00:21<00:05, 280.05it/s]\u001b[A\n",
711
- " 81% 6200/7609 [00:21<00:04, 283.64it/s]\u001b[A\n",
712
- " 82% 6236/7609 [00:21<00:04, 302.84it/s]\u001b[A\n",
713
- " 82% 6274/7609 [00:21<00:04, 322.67it/s]\u001b[A\n",
714
- " 83% 6309/7609 [00:21<00:03, 329.48it/s]\u001b[A\n",
715
- " 83% 6346/7609 [00:21<00:03, 340.00it/s]\u001b[A\n",
716
- " 84% 6381/7609 [00:21<00:03, 341.59it/s]\u001b[A\n",
717
- " 84% 6416/7609 [00:22<00:03, 340.91it/s]\u001b[A\n",
718
- " 85% 6451/7609 [00:22<00:03, 343.01it/s]\u001b[A\n",
719
- " 85% 6489/7609 [00:22<00:03, 351.88it/s]\u001b[A\n",
720
- " 86% 6525/7609 [00:22<00:03, 349.91it/s]\u001b[A\n",
721
- " 86% 6562/7609 [00:22<00:02, 354.01it/s]\u001b[A\n",
722
- " 87% 6598/7609 [00:22<00:02, 355.09it/s]\u001b[A\n",
723
- " 87% 6634/7609 [00:22<00:02, 338.17it/s]\u001b[A\n",
724
- " 88% 6670/7609 [00:22<00:02, 342.02it/s]\u001b[A\n",
725
- " 88% 6705/7609 [00:22<00:02, 334.34it/s]\u001b[A\n",
726
- " 89% 6739/7609 [00:23<00:02, 323.10it/s]\u001b[A\n",
727
- " 89% 6772/7609 [00:23<00:02, 323.96it/s]\u001b[A\n",
728
- " 89% 6805/7609 [00:23<00:02, 311.85it/s]\u001b[A\n",
729
- " 90% 6840/7609 [00:23<00:02, 321.24it/s]\u001b[A\n",
730
- " 90% 6873/7609 [00:23<00:02, 316.97it/s]\u001b[A\n",
731
- " 91% 6907/7609 [00:23<00:02, 321.75it/s]\u001b[A\n",
732
- " 91% 6942/7609 [00:23<00:02, 328.07it/s]\u001b[A\n",
733
- " 92% 6976/7609 [00:23<00:01, 331.30it/s]\u001b[A\n",
734
- " 92% 7010/7609 [00:23<00:01, 330.95it/s]\u001b[A\n",
735
- " 93% 7045/7609 [00:23<00:01, 335.28it/s]\u001b[A\n",
736
- " 93% 7079/7609 [00:24<00:01, 306.99it/s]\u001b[A\n",
737
- " 93% 7111/7609 [00:24<00:01, 302.98it/s]\u001b[A\n",
738
- " 94% 7142/7609 [00:24<00:01, 290.42it/s]\u001b[A\n",
739
- " 94% 7172/7609 [00:24<00:01, 254.75it/s]\u001b[A\n",
740
- " 95% 7199/7609 [00:24<00:01, 219.10it/s]\u001b[A\n",
741
- " 95% 7223/7609 [00:24<00:01, 209.03it/s]\u001b[A\n",
742
- " 95% 7245/7609 [00:24<00:01, 204.13it/s]\u001b[A\n",
743
- " 95% 7266/7609 [00:25<00:01, 187.45it/s]\u001b[A\n",
744
- " 96% 7286/7609 [00:25<00:01, 188.71it/s]\u001b[A\n",
745
- " 96% 7306/7609 [00:25<00:01, 190.24it/s]\u001b[A\n",
746
- " 96% 7326/7609 [00:25<00:01, 179.65it/s]\u001b[A\n",
747
- " 97% 7346/7609 [00:25<00:01, 183.20it/s]\u001b[A\n",
748
- " 97% 7365/7609 [00:25<00:01, 182.16it/s]\u001b[A\n",
749
- " 97% 7385/7609 [00:25<00:01, 184.72it/s]\u001b[A\n",
750
- " 97% 7404/7609 [00:25<00:01, 186.02it/s]\u001b[A\n",
751
- " 98% 7424/7609 [00:25<00:00, 187.70it/s]\u001b[A\n",
752
- " 98% 7443/7609 [00:26<00:00, 187.57it/s]\u001b[A\n",
753
- " 98% 7462/7609 [00:26<00:00, 179.01it/s]\u001b[A\n",
754
- " 98% 7480/7609 [00:26<00:00, 172.84it/s]\u001b[A\n",
755
- " 99% 7498/7609 [00:26<00:00, 167.57it/s]\u001b[A\n",
756
- " 99% 7517/7609 [00:26<00:00, 172.12it/s]\u001b[A\n",
757
- " 99% 7535/7609 [00:26<00:00, 164.33it/s]\u001b[A\n",
758
- " 99% 7552/7609 [00:26<00:00, 154.64it/s]\u001b[A\n",
759
- " 99% 7569/7609 [00:26<00:00, 157.53it/s]\u001b[A\n",
760
- "100% 7588/7609 [00:26<00:00, 165.34it/s]\u001b[A\n",
761
- "100% 7609/7609 [00:27<00:00, 281.47it/s]\n",
762
- "06/11/2023 20:57:34 - INFO - utils_qa - Saving predictions to eval_results/predict_predictions.json.\n",
763
- "06/11/2023 20:57:34 - INFO - utils_qa - Saving nbest_preds to eval_results/predict_nbest_predictions.json.\n",
764
- "***** predict metrics *****\n",
765
- " predict_samples = 7992\n",
766
- " test_exact_match = 65.7248\n",
767
- " test_f1 = 77.0614\n",
768
- " test_runtime = 0:03:14.17\n",
769
- " test_samples_per_second = 41.158\n",
770
- " test_steps_per_second = 8.235\n",
771
- "[INFO|modelcard.py:451] 2023-06-11 20:57:38,434 >> Dropping the following result as it does not have all the necessary fields:\n",
772
- "{'task': {'name': 'Question Answering', 'type': 'question-answering'}, 'dataset': {'name': 'squad_it', 'type': 'squad_it'}}\n",
773
- "100% 1599/1599 [03:56<00:00, 6.77it/s]\n"
774
- ]
775
- }
776
- ]
777
- },
778
- {
779
- "cell_type": "code",
780
- "source": [],
781
- "metadata": {
782
- "id": "rMyhUFpnItIQ"
783
- },
784
- "execution_count": null,
785
- "outputs": []
786
- }
787
- ]
788
- }