Upload 7 files

TrainLlTRA.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":14916,"status":"ok","timestamp":1704900816235,"user":{"displayName":"Esmail Atta","userId":"01595736708541341728"},"user_tz":-180},"id":"u40Mwfxw3KgY","outputId":"be4f239a-c096-40e8-b182-3d13d150bc59"},"outputs":[{"name":"stdout","output_type":"stream","text":["Collecting datasets\n","  Downloading datasets-2.16.1-py3-none-any.whl (507 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m507.1/507.1 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n","Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n","Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n","Collecting dill<0.3.8,>=0.3.0 (from datasets)\n","  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n","Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n","Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n","Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n","Collecting multiprocess (from datasets)\n","  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n","Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n","Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.2)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n","Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n","Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.5.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.11.17)\n","Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n","Installing collected packages: dill, multiprocess, datasets\n","Successfully installed datasets-2.16.1 dill-0.3.7 multiprocess-0.70.15\n","Collecting torchmetrics\n","  Downloading torchmetrics-1.2.1-py3-none-any.whl (806 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m806.1/806.1 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy>1.20.0 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (1.23.5)\n","Requirement already satisfied: packaging>17.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (23.2)\n","Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from torchmetrics) (2.1.0+cu121)\n","Collecting lightning-utilities>=0.8.0 (from torchmetrics)\n","  Downloading lightning_utilities-0.10.0-py3-none-any.whl (24 kB)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (67.7.2)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from lightning-utilities>=0.8.0->torchmetrics) (4.5.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.13.1)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (1.12)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.2.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (3.1.2)\n","Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (2023.6.0)\n","Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.1->torchmetrics) (2.1.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8.1->torchmetrics) (2.1.3)\n","Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8.1->torchmetrics) (1.3.0)\n","Installing collected packages: lightning-utilities, torchmetrics\n","Successfully installed lightning-utilities-0.10.0 torchmetrics-1.2.1\n"]}],"source":["!pip install datasets\n","!pip install torchmetrics"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["e7ca2b775f364535b8680690debe2474","ef05b268e2474f4c85568fbdde4b6438","4ca3067006f94781b4ba3bccb5ba06dc","3c8a2090fcbf4852ba9725567276f075","2417416e2fac4a128e2f1a9c9d7ec954","31327824719549bf8efa2d5552a5bbb6","87a904f6c0104c6f9a7db082115add8e","4f2aae26e0134d5893bf6d1e5df3e8fa","a0d08f72e8944ac3974ead823bbb77eb","4527d0722a5945ef971c84501a83cb80","42ab868cbda24d0ba8c9edc830ebf0e2","c72162b3d21546dabe5106c1b5a0bc05","488f4ee309054452abbf1723c464a655","7f98711a78904150b33a22304b385790","d02174a09e7a4940b4036d33da452f14","df3b97420e9f44a587e6cce4af770609","a07af88443414f498a391d322448b0a4","99b49687e4644218a4ac3a6e1384c373","8eecef66de34438784767d6f9a85da03","a58c1a159c7144bcae7ad01354915db9","9a7ddd319fb046a5bc84deaa2471f2d5","66f389563fec4f37b40c63b56880fbc7"]},"executionInfo":{"elapsed":13029424,"status":"ok","timestamp":1704913851683,"user":{"displayName":"Esmail Atta","userId":"01595736708541341728"},"user_tz":-180},"id":"J0x_z-BA3R7z","outputId":"97e67505-5e6f-4839-e516-0a0bb515a10f"},"outputs":[{"name":"stdout","output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n","Using device: cuda\n","Device name: Tesla T4\n","Device memory: 14.74810791015625 GB\n"]},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"e7ca2b775f364535b8680690debe2474","version_major":2,"version_minor":0},"text/plain":["Downloading data:   0%|          | 0.00/2.78M [00:00<?, ?B/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"c72162b3d21546dabe5106c1b5a0bc05","version_major":2,"version_minor":0},"text/plain":["Generating train split:   0%|          | 0/50769 [00:00<?, ? examples/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["Max length of source sentence: 79\n","Max length of target sentence: 83\n","Preloading model opus_infopankki_weights/tmodel_00.pt\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 01: 100%|██████████| 5712/5712 [11:16<00:00,  8.45it/s, loss=1.870]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: العائلات الناطقة بلغة أجنبية لديها الحق في خدمات الترجمة عند اللزوم.\n","    TARGET: A foreign-language family is entitled to interpreting services as necessary.\n"," PREDICTED: in a native language is provided by the services of the services for the elderly .\n","--------------------------------------------------------------------------------\n","    SOURCE: يمكن لمواطني الاتحاد الأوروبي (EU) والمنطقة الاقتصادية الأوروبية (ETA) أن يعلنوا عن أنفسهم كباحثين عن العمل وذلك بواسطة الخدمة الشبكية لمكتب العمل والموارد المعيشية في قسم \"Oma asiointi\" المعاملات الشخصية.\n","    TARGET: To find web pages for jobs on the Internet, write “avoimet työpaikat” (vacancies) in the search engine’s text field. Many web pages for jobs allow you to fill in and send a job application and to enclose your CV.\n"," PREDICTED: of : ( ) and , , and .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 02: 100%|██████████| 5712/5712 [11:17<00:00,  8.43it/s, loss=2.823]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: عندما تلتحق بالدراسة، فستحصل على الحق في إنجاز كلتا الدرجتين العلميتين.\n","    TARGET: When you are accepted into an institute of higher education, you receive the right to complete both degrees.\n"," PREDICTED: When you have a of residence , you will receive a higher education degree .\n","--------------------------------------------------------------------------------\n","    SOURCE: عندما استقلت فنلندا سنة 1917، أصبحت هلسنكي العاصمة لجمهورية فنلندا.\n","    TARGET: When Finland gained its independence in 1917, Helsinki became the capital of the republic.\n"," PREDICTED: When gained its independence in , the became the capital of .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 03: 100%|██████████| 5712/5712 [11:18<00:00,  8.42it/s, loss=2.646]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: مركز الضمان التقاعدي يقدم النصيحة لك، عندما تطلب التقاعد من الخارج.\n","    TARGET: The Finnish Centre for Pension will give you advice for applying for pension abroad.\n"," PREDICTED: The for will apply for a when you apply for .\n","--------------------------------------------------------------------------------\n","    SOURCE: اللجنة لا تتداول حالات التهميش والتمييز المتعلقة بالعمل.\n","    TARGET: The Tribunal does not handle cases of employment-related discrimination.\n"," PREDICTED: The does not have to pay and the work .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 04: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=2.014]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: القاسم المشترك لهذه الغرف هو عمل القهوة وإمكانية استعمال الكومبيوتر المجهز باشتراك الإنترنت مجاناً كذلك يمكن في عديد من هذه الغرف التمتع بالوجبات المحضرة في الغرفة بسعر معقول والتي تقدم في أيام العمل.\n","    TARGET: All centres have a cafe and the opportunity to use a computer with a free-of-charge Internet connection. Most resident centres also offer the opportunity to enjoy an affordable lunch prepared at the centre that is served on weekdays.\n"," PREDICTED: All centres are open to and at a . Many centres also offer an opportunity to use a at a centre in the centre where are .\n","--------------------------------------------------------------------------------\n","    SOURCE: تساعد في مشاكل علاقات الشراكة والمشاكل العائلية أيضاً الاستشارات العائلية لأبرشية فانتا (Vantaa).\n","    TARGET: The family guidance of the Vantaa Parish Union also helps with relationship and family issues.\n"," PREDICTED: and family also help with the .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 05: 100%|██████████| 5712/5712 [11:19<00:00,  8.40it/s, loss=1.728]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: اللجنة الإستشارية لتعدد الثقافاتالفنلندية\n","    TARGET: Advisory board for multicultural affairsFinnish\n"," PREDICTED: of\n","--------------------------------------------------------------------------------\n","    SOURCE: يتم تنظيم تعليم تحضيري للمهاجرين للتعليم المهني في معهد التعليم المهني وتعليم الكبار لستادي.\n","    TARGET: Helsinki Vocational College organises training that prepares immigrants for vocational education.\n"," PREDICTED: education for education and training training for training and .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 06: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=2.187]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: Metsolakoditالفنلندية\n","    TARGET: MetsolakoditFinnish\n"," PREDICTED: \n","--------------------------------------------------------------------------------\n","    SOURCE: يجب عليك أيضاً أن تستطيع إثبات على سبيل المثال بالوصفة الطبية أو بالتقرير الطبي بأن الغرض من الدواء هو استخدامك أنت الشخصي.\n","    TARGET: In addition, you must be able to prove with a prescription or medical certificate, for example, that the medicine is intended for your personal use.\n"," PREDICTED: You must also have to prove your identity with a friend or friend , for example , that the medicine is intended for your personal use .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 07: 100%|██████████| 5712/5712 [11:17<00:00,  8.44it/s, loss=1.719]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: قمت بارتكاب جرائم وتُعتبر بأنك خطير على النظام أو الأمن العام.\n","    TARGET: you have committed crimes and are considered a danger to public order or safety\n"," PREDICTED: you have committed crimes and are considered a danger to public order or safety\n","--------------------------------------------------------------------------------\n","    SOURCE: إذا كان لديك ترخيص إقامة في فنلندا، ولكن لم تُمنح ترخيص إقامة استمراري، فسوف تصدر دائرة شؤون الهجرة قراراً بالترحيل.\n","    TARGET: If you already have a residence permit in Finland but are not granted a residence permit extension, the Finnish Immigration Service makes a deportation decision.\n"," PREDICTED: If you have a residence permit in but are not granted a residence permit , the Service makes a decision .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 08: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=1.682]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: العنف الجنسي\n","    TARGET: Sexual violence\n"," PREDICTED: violence\n","--------------------------------------------------------------------------------\n","    SOURCE: فنادقالفنلندية _ السويدية _ الإنجليزية _ الروسية\n","    TARGET: HotelsFinnish _ Swedish _ English _ Russian\n"," PREDICTED: _ Swedish _ English _ Russian\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 09: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=1.484]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: الأسر التي لديها أطفال\n","    TARGET: families with children\n"," PREDICTED: families with children\n","--------------------------------------------------------------------------------\n","    SOURCE: من الممكن أن يحتاج إلى المساعدة أي شخص في الحالات الحرجة.\n","    TARGET: Anyone in a challenging situation in life can be in need of help.\n"," PREDICTED: in a challenging situation in a family can be in a challenging situation .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 10: 100%|██████████| 5712/5712 [11:18<00:00,  8.42it/s, loss=1.445]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: Welcome Guideالإنجليزية\n","    TARGET: Welcome GuideEnglish\n"," PREDICTED: \n","--------------------------------------------------------------------------------\n","    SOURCE: اذهبي إلى الطبيب\n","    TARGET: See a doctor\n"," PREDICTED: See a doctor\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 11: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=1.821]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: هذا يُعرف باسم معاملات دبلن (Dublin).\n","    TARGET: This is called the Dublin procedure.\n"," PREDICTED: This is called the .\n","--------------------------------------------------------------------------------\n","    SOURCE: على سبيل المثال، مؤسسة التقاعد الوطني الكيلا ودائرة شئون الهجرة (Maahanmuuttovirasto) تقوم بحجز مترجم شفوي للزبون في بعض الحالات.\n","    TARGET: Kela and the Finnish Immigration service (Maahanmuuttovirasto), for example, book interpreters for their customers in some cases.\n"," PREDICTED: and the service ( ), for example , book for their in some cases .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 12: 100%|██████████| 5712/5712 [11:15<00:00,  8.46it/s, loss=1.528]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: مثلاً. يجب أرفاق توضيح شخصي عن الوضع أيضاً كملحق لطلب ترخيص الإقامة.\n","    TARGET: You should also attach your account of the circumstances to the residence permit application.\n"," PREDICTED: You should also attach your account of the circumstances to the residence permit application .\n","--------------------------------------------------------------------------------\n","    SOURCE: تقوم باستقبال طلبات ترخيص الإقامة إلى فنلندا\n","    TARGET: accept residence permit applications for Finland\n"," PREDICTED: accept residence permit for\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 13: 100%|██████████| 5712/5712 [11:16<00:00,  8.44it/s, loss=1.461]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: ذا بدأت العمل أو التعليم عندما تكون عاطلًا عن العمل، فأبلغ عن ذلك إلى مكتب العمل والموارد المعيشية.\n","    TARGET: If you start working or studying while you are unemployed, please notify the TE Office.\n"," PREDICTED: If you start working or studying while you are , please the Office .\n","--------------------------------------------------------------------------------\n","    SOURCE: يتأقلم الشخص الذي يقطن السكن المدعوم مع أعماله اليومة بصفة مستقلة تقريباً.\n","    TARGET: A person living in supported housing copes with everyday tasks almost independently.\n"," PREDICTED: A person living in supported with everyday tasks almost independently .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 14: 100%|██████████| 5712/5712 [11:19<00:00,  8.41it/s, loss=1.442]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: تقعُ إستشارية التنظيم الأسري في المركز الصحي الرئيسي في مانتومكي (Mäntymäki).\n","    TARGET: The Contraception Clinic is located at the main health centre in Mäntymäki.\n"," PREDICTED: The is in the largest cities and in .\n","--------------------------------------------------------------------------------\n","    SOURCE: إذا كانت هناك حاجة لتوضيحات إضافية لطلبك، فسوف يتم إبلاغك بذلك عن طريق حسابك على - Enter Finland.\n","    TARGET: If further clarifications are needed, you will be informed through your account.\n"," PREDICTED: If further are needed , you will be informed through your account .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 15: 100%|██████████| 5712/5712 [11:20<00:00,  8.39it/s, loss=1.403]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: _ البلغارية\n","    TARGET: _ Bulgarian\n"," PREDICTED: _\n","--------------------------------------------------------------------------------\n","    SOURCE: منظومة المساعدة الوطنية الفنلندية لضحايا الإتجار بالبشر\n","    TARGET: The Finnish national system for assisting victims of human trafficking\n"," PREDICTED: The national national system for assisting victims of human\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 16: 100%|██████████| 5712/5712 [11:16<00:00,  8.45it/s, loss=1.457]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: من الممكن أن يحتاج إلى المساعدة أي شخص في الحالات الحرجة.\n","    TARGET: Anyone in a challenging situation in life can be in need of help.\n"," PREDICTED: in a challenging situation in life can be in need of help .\n","--------------------------------------------------------------------------------\n","    SOURCE: شروط نقود البدء هي على سبيل المثال:\n","    TARGET: Prerequisites for getting a start-up grant:\n"," PREDICTED: for getting a start - up grant :\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 17: 100%|██████████| 5712/5712 [11:15<00:00,  8.46it/s, loss=1.453]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: هاتف: 0800 414 004\n","    TARGET: Tel. 0800 414 004\n"," PREDICTED: .\n","--------------------------------------------------------------------------------\n","    SOURCE: اقرأ المزيد: الصحة الجنسية.\n","    TARGET: Read more: Sexual health.\n"," PREDICTED: Read more : health .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 18: 100%|██████████| 5712/5712 [11:15<00:00,  8.46it/s, loss=1.452]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: حرر إعلان إلغاء عقد الإيجار كتابياً دائماً، إذ يجب عليك أن تكون على استعداد للبرهان بأنك قد بلغت المؤجِّر عن الإلغاء.\n","    TARGET: Always make the notice of termination in writing. You must be able to prove that you have given the notice to the landlord.\n"," PREDICTED: Always make the notice of termination in writing . You must be able to prove that you have given the notice to the landlord .\n","--------------------------------------------------------------------------------\n","    SOURCE: إذا كنت تشك بأنك وقعت في مكان تمييز فبإمكانك التواصل مع سلطات الحماية في العمل وبالإتحاد المهني الخاص بك.\n","    TARGET: If you suspect that you have been the object of discrimination at work, you can contact the occupational safety and health authorities or your own trade union.\n"," PREDICTED: If you suspect that you have been the object of at work , you can contact the safety and health authorities or your own trade union .\n","--------------------------------------------------------------------------------\n"]},{"name":"stderr","output_type":"stream","text":["Processing Epoch 19: 100%|██████████| 5712/5712 [11:18<00:00,  8.42it/s, loss=1.437]\n"]},{"name":"stdout","output_type":"stream","text":["--------------------------------------------------------------------------------\n","    SOURCE: متزوج أو في علاقة مفتوحة الفنلندية\n","    TARGET: DivorcingFinnish\n"," PREDICTED: \n","--------------------------------------------------------------------------------\n","    SOURCE: الخدمة تقدم أيضاً خدمات دعم على أشكال مجموعات وكذلك إمكانية المقابلة المدعومة/الكائنة تحت المراقبة وكذلك في تعليم كيفية النوم.\n","    TARGET: The service also offers group support services and opportunities for supported/supervised meetings and sleep training.\n"," PREDICTED: The service also offers group support services and opportunities for supported / meetings and sleep training .\n","--------------------------------------------------------------------------------\n"]}],"source":["from google.colab import drive\n","\n","drive.mount('/content/drive')\n","\n","import os \n","\n","os.chdir('/content/drive/MyDrive/LlTRA')\n","\n","%run train.py"]}],"metadata":{"accelerator":"GPU","colab":{"authorship_tag":"ABX9TyOETClsPsvovjgdHvEjmFPX","gpuType":"T4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"2417416e2fac4a128e2f1a9c9d7ec954":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"31327824719549bf8efa2d5552a5bbb6":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3c8a2090fcbf4852ba9725567276f075":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4527d0722a5945ef971c84501a83cb80","placeholder":"​","style":"IPY_MODEL_42ab868cbda24d0ba8c9edc830ebf0e2","value":" 2.78M/2.78M [00:00&lt;00:00, 8.24MB/s]"}},"42ab868cbda24d0ba8c9edc830ebf0e2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4527d0722a5945ef971c84501a83cb80":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"488f4ee309054452abbf1723c464a655":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a07af88443414f498a391d322448b0a4","placeholder":"​","style":"IPY_MODEL_99b49687e4644218a4ac3a6e1384c373","value":"Generating train split: 100%"}},"4ca3067006f94781b4ba3bccb5ba06dc":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_4f2aae26e0134d5893bf6d1e5df3e8fa","max":2775475,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a0d08f72e8944ac3974ead823bbb77eb","value":2775475}},"4f2aae26e0134d5893bf6d1e5df3e8fa":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"66f389563fec4f37b40c63b56880fbc7":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7f98711a78904150b33a22304b385790":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8eecef66de34438784767d6f9a85da03","max":50769,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a58c1a159c7144bcae7ad01354915db9","value":50769}},"87a904f6c0104c6f9a7db082115add8e":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8eecef66de34438784767d6f9a85da03":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"99b49687e4644218a4ac3a6e1384c373":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"9a7ddd319fb046a5bc84deaa2471f2d5":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a07af88443414f498a391d322448b0a4":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a0d08f72e8944ac3974ead823bbb77eb":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"a58c1a159c7144bcae7ad01354915db9":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"c72162b3d21546dabe5106c1b5a0bc05":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_488f4ee309054452abbf1723c464a655","IPY_MODEL_7f98711a78904150b33a22304b385790","IPY_MODEL_d02174a09e7a4940b4036d33da452f14"],"layout":"IPY_MODEL_df3b97420e9f44a587e6cce4af770609"}},"d02174a09e7a4940b4036d33da452f14":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9a7ddd319fb046a5bc84deaa2471f2d5","placeholder":"​","style":"IPY_MODEL_66f389563fec4f37b40c63b56880fbc7","value":" 50769/50769 [00:00&lt;00:00, 398935.49 examples/s]"}},"df3b97420e9f44a587e6cce4af770609":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e7ca2b775f364535b8680690debe2474":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ef05b268e2474f4c85568fbdde4b6438","IPY_MODEL_4ca3067006f94781b4ba3bccb5ba06dc","IPY_MODEL_3c8a2090fcbf4852ba9725567276f075"],"layout":"IPY_MODEL_2417416e2fac4a128e2f1a9c9d7ec954"}},"ef05b268e2474f4c85568fbdde4b6438":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_31327824719549bf8efa2d5552a5bbb6","placeholder":"​","style":"IPY_MODEL_87a904f6c0104c6f9a7db082115add8e","value":"Downloading data: 100%"}}}}},"nbformat":4,"nbformat_minor":0}

configuration.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+def Get_configuration():
+    return {
+        "batch_size": 8,
+        "num_epochs": 20,
+        "lr": 10**-4,
+        "sequence_length": 100,
+        "d_model": 512,
+        "datasource": 'opus_infopankki',
+        "source_language": "ar",
+        "target_language": "en",
+        "model_folder": "weights",
+        "model_basename": "tmodel_",
+        "preload": "latest",
+        "tokenizer_file": "tokenizer_{0}.json",
+        "experiment_name": "runs/tmodel"
+    }
+
+def Get_weights_file_path(config, epoch: str):
+    model_folder = f"{config['datasource']}_{config['model_folder']}"
+    model_filename = f"{config['model_basename']}{epoch}.pt"
+    return str(Path('.') / model_folder / model_filename)
+
+def latest_weights_file_path(config):
+    model_folder = f"{config['datasource']}_{config['model_folder']}"
+    model_filename = f"{config['model_basename']}*"
+    weights_files = list(Path(model_folder).glob(model_filename))
+    if len(weights_files) == 0:
+        return None
+    weights_files.sort()
+    return str(weights_files[-1])

dataset.py

@@ -0,0 +1,78 @@
+import torch 
+from torch.utils.data import Dataset
+
+class BilingualDataset(Dataset):
+
+    def __init__(self, dataset, source_tokenizer, target_tokenizer, source_language, target_language, sequence_length):
+        super().__init__()
+        self.dataset = dataset
+        self.source_tokenizer = source_tokenizer
+        self.target_tokenizer = target_tokenizer
+        self.source_language = source_language
+        self.target_language = target_language
+        self.sequence_length = sequence_length
+
+        self.SOS_token = torch.tensor([target_tokenizer.token_to_id("[SOS]")], dtype=torch.int64)
+        self.PAD_token = torch.tensor([target_tokenizer.token_to_id("[PAD]")], dtype= torch.int64)
+        self.EOS_token = torch.tensor([target_tokenizer.token_to_id("[EOS]")], dtype= torch.int64)
+
+    def __len__(self):
+        return len(self.dataset)
+
+    def __getitem__(self, index) :
+        source_target_dataset = self.dataset[index]
+        source_text = source_target_dataset['translation'][self.source_language]
+        target_text = source_target_dataset['translation'][self.target_language]
+
+        encode_source_tokenizer = self.source_tokenizer.encode(source_text).ids 
+        encode_target_tokenizer = self.target_tokenizer.encode(target_text).ids 
+
+        encode_source_padding = self.sequence_length - len(encode_source_tokenizer) - 2 
+        encode_target_padding = self.sequence_length - len(encode_target_tokenizer) - 1 
+
+        if encode_source_padding < 0 or encode_target_padding < 0:
+            raise ValueError("sequence is too long")
+
+        encoder_input = torch.cat(
+            [
+                self.SOS_token,
+                torch.tensor(encode_source_tokenizer, dtype=torch.int64),
+                self.EOS_token,
+                torch.tensor([self.PAD_token] * encode_source_padding, dtype=torch.int64)
+            ]
+        )
+
+        decoder_input = torch.cat(
+            [
+                self.SOS_token,
+                torch.tensor(encode_target_tokenizer, dtype=torch.int64),
+                torch.tensor([self.PAD_token] * encode_target_padding, dtype=torch.int64)
+            ]
+        )
+
+        Target = torch.cat(
+            [
+                torch.tensor(encode_target_tokenizer, dtype=torch.int64),
+                torch.tensor([self.PAD_token] * encode_target_padding, dtype=torch.int64),
+                self.EOS_token
+            ]
+        )
+
+        assert encoder_input.size(0) == self.sequence_length 
+        assert decoder_input.size(0) == self.sequence_length 
+        assert Target.size(0) == self.sequence_length
+
+        return {
+            "encoder_input": encoder_input,
+            "decoder_input": decoder_input,
+            "encoder_input_mask": (encoder_input != self.PAD_token).unsqueeze(0).unsqueeze(0).int(),
+            "decoder_input_mask": (decoder_input != self.PAD_token).unsqueeze(0).int() & casual_mask(decoder_input.size(0)),
+            "Target": Target,
+            "source_text": source_text,
+            "target_text": target_text 
+        }
+
+
+def casual_mask(size):
+    mask = torch.triu(torch.ones((1, size, size)), diagonal=1).type(torch.int)
+    return mask == 0

model.py

@@ -0,0 +1,221 @@
+#LlTRA = Language to Language Transformer model.
+import math
+import torch 
+import torch.nn as nn
+
+class InputEmbeddingsLayer(nn.Module): 
+    def __init__(self, d_model: int, vocab_size: int) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.vocab_size = vocab_size
+        self.embedding = nn.Embedding(vocab_size, d_model)
+    def forward(self, x):
+        return self.embedding(x) * math.sqrt(self.d_model)
+
+class PositionalEncodingLayer(nn.Module):
+    def __init__(self, d_model: int, sequence_length: int, dropout: float) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.sequence_length = sequence_length
+        self.dropout = nn.Dropout(dropout)
+
+        PE = torch.zeros(sequence_length, d_model)
+        Position = torch.arange(0, sequence_length, dtype=torch.float).unsqueeze(1)
+        deviation_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+
+        PE[:, 0::2] = torch.sin(Position * deviation_term)
+        PE[:, 1::2] = torch.cos(Position * deviation_term)
+        PE = PE.unsqueeze(0)
+        self.register_buffer('PE', PE)
+    def forward(self, x):
+        x = x + (self.PE[:, :x.shape[1], :]).requires_grad_(False)
+        return self.dropout(x)
+
+class NormalizationLayer(nn.Module):
+    def __init__(self, Epslone: float = 10**-6) -> None:
+        super().__init__()
+        self.Epslone = Epslone
+        self.Alpha = nn.Parameter(torch.ones(1)) 
+        self.Bias = nn.Parameter(torch.ones(1)) 
+    def forward(self, x):
+        mean = x.mean(dim = -1, keepdim = True)
+        std = x.std(dim = -1, keepdim = True)
+
+        return self.Alpha * (x - mean) / (std + self.Epslone) + self.Bias
+
+class FeedForwardBlock(nn.Module):
+    def __init__(self, d_model: int, d_ff: int, dropout: float) -> None:
+        super().__init__()
+        self.Linear_1 = nn.Linear(d_model, d_ff)
+        self.dropout = nn.Dropout(dropout)
+        self.Linear_2 = nn.Linear(d_ff, d_model)
+    def forward(self, x):
+        return self.Linear_2(self.dropout(torch.relu(self.Linear_1(x))))
+
+class MultiHeadAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, heads: int, dropout: float) -> None:
+        super().__init__()
+        self.d_model = d_model
+        self.heads = heads 
+
+        assert d_model % heads == 0 , "d_model is not divisible by heads"
+
+        self.d_k = d_model // heads 
+        
+        self.W_Q = nn.Linear(d_model, d_model) 
+        self.W_K = nn.Linear(d_model, d_model) 
+        self.W_V = nn.Linear(d_model, d_model) 
+
+        self.W_O = nn.Linear(d_model, d_model)
+        self.dropout = nn.Dropout(dropout)
+
+    @staticmethod
+    def Attention(query, key, value, mask, dropout: nn.Dropout):
+        d_k = query.shape[-1]
+        self_attention_scores = (query @ key.transpose(-2, -1)) / math.sqrt(d_k)
+        if mask is not None:
+            self_attention_scores.masked_fill_(mask == 0, -1e9)
+
+        self_attention_scores = self_attention_scores.softmax(dim=-1) 
+        if dropout is not None:
+            self_attention_scores = dropout(self_attention_scores)
+
+        return (self_attention_scores @ value), self_attention_scores
+    def forward(self, query, key, value, mask):
+        Query = self.W_Q(query)
+        Key = self.W_K(key)
+        Value = self.W_V(value) 
+
+        Query = Query.view(Query.shape[0], Query.shape[1], self.heads, self.d_k).transpose(1,2)
+        Key = Key.view(Key.shape[0], Key.shape[1], self.heads, self.d_k).transpose(1,2)
+        Value = Value.view(Value.shape[0], Value.shape[1], self.heads, self.d_k).transpose(1,2)
+
+        x, self.self_attention_scores = MultiHeadAttentionBlock.Attention(Query, Key, Value, mask, self.dropout)
+
+        x = x.transpose(1,2).contiguous().view(x.shape[0], -1, self.heads * self.d_k)
+
+        return self.W_O(x)
+
+class ResidualConnection(nn.Module):
+    def __init__(self, dropout: float) -> None:
+        super().__init__()
+        self.dropout = nn.Dropout(dropout)
+        self.normalization = NormalizationLayer()
+    def forward(self, x, subLayer):
+        return x + self.dropout(subLayer(self.normalization(x)))
+
+class EncoderBlock(nn.Module):
+    def __init__(self, encoder_self_attention_block: MultiHeadAttentionBlock, encoder_feed_forward_block: FeedForwardBlock, dropout: float) -> None:
+        super().__init__()
+        self.encoder_self_attention_block = encoder_self_attention_block
+        self.encoder_feed_forward_block = encoder_feed_forward_block
+        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(2)])
+    def forward(self, x, source_mask):
+        x = self.residual_connection[0](x, lambda x: self.encoder_self_attention_block(x, x, x, source_mask))
+        x = self.residual_connection[1](x, self.encoder_feed_forward_block)
+
+        return x
+
+class Encoder(nn.Module):
+    def __init__(self, Layers: nn.ModuleList) -> None:
+        super().__init__()
+        self.Layers = Layers
+        self.normalization = NormalizationLayer()
+    def forward(self, x, source_mask):
+        for layer in self.Layers:
+            x = layer(x, source_mask)
+
+        return self.normalization(x)
+
+class DecoderBlock(nn.Module):
+    def __init__(self, decoder_self_attention_block: MultiHeadAttentionBlock, decoder_cross_attention_block: MultiHeadAttentionBlock, decoder_feed_forward_block: FeedForwardBlock, dropout: float) -> None:
+        super().__init__()
+        self.decoder_self_attention_block = decoder_self_attention_block
+        self.decoder_cross_attention_block = decoder_cross_attention_block
+        self.decoder_feed_forward_block = decoder_feed_forward_block
+        self.residual_connection = nn.ModuleList([ResidualConnection(dropout) for _ in range(3)])
+    def forward(self, x, Encoder_output, maks, target_mask):
+        x = self.residual_connection[0](x, lambda x: self.decoder_self_attention_block(x, x, x, target_mask))
+        x = self.residual_connection[1](x, lambda x: self.decoder_cross_attention_block(x, Encoder_output, Encoder_output, target_mask))
+        x = self.residual_connection[2](x, self.decoder_feed_forward_block)
+
+        return x
+
+class Decoder(nn.Module):
+    def __init__(self, Layers: nn.ModuleList) -> None:
+        super().__init__()
+        self.Layers = Layers 
+        self.normalization = NormalizationLayer()
+    def forward(self, x, Encoder_output, mask, target_mask):
+        for layer in self.Layers:
+            x = layer(x, Encoder_output, mask, target_mask)
+
+        return self.normalization(x)
+
+class LinearLayer(nn.Module):
+    def __init__(self, d_model: int, vocab_size: int) -> None:
+        super().__init__()
+        self.Linear = nn.Linear(d_model, vocab_size)
+    def forward(self, x):
+        return self.Linear(x)
+
+
+class TransformerBlock(nn.Module):
+    def __init__(self, encoder: Encoder, decoder: Decoder, source_embedding: InputEmbeddingsLayer, target_embedding: InputEmbeddingsLayer, source_position: PositionalEncodingLayer, target_position: PositionalEncodingLayer, Linear: LinearLayer) -> None:
+        super().__init__()
+        self.encoder = encoder 
+        self.decoder = decoder 
+        self.source_embedding = source_embedding
+        self.target_embedding = target_embedding
+        self.source_position = source_position
+        self.target_position = target_position
+        self.Linear = Linear
+    def encode(self, source_language, source_mask):
+        source_language = self.source_embedding(source_language)
+        source_language = self.source_position(source_language)
+        return self.encoder(source_language, source_mask)
+    def decode(self, Encoder_output, mask, target_language, target_mask):
+        target_language = self.target_embedding(target_language)
+        target_language = self.target_position(target_language)
+        return self.decoder(target_language, Encoder_output, mask, target_mask)
+    def linear(self, x):
+        return self.Linear(x)
+
+
+def TransformerModel(source_vocab_size: int, target_vocab_size: int, source_sequence_length: int, target_sequence_length: int, d_model: int = 512, Layers: int = 6, heads: int = 8, dropout: float = 0.1, d_ff: int = 2048)->TransformerBlock:
+
+    source_embedding = InputEmbeddingsLayer(d_model, source_vocab_size)
+    source_position = PositionalEncodingLayer(d_model, source_sequence_length, dropout)
+    
+    target_embedding = InputEmbeddingsLayer(d_model, target_vocab_size)
+    target_position = PositionalEncodingLayer(d_model, target_sequence_length, dropout)
+
+    EncoderBlocks = []
+    for _ in range(Layers):
+        encoder_self_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)
+        encoder_feed_forward_block = FeedForwardBlock(d_model, d_ff, dropout)
+        encoder_block = EncoderBlock(encoder_self_attention_block, encoder_feed_forward_block, dropout)
+        EncoderBlocks.append(encoder_block)
+
+    
+    DecoderBlocks = []
+    for _ in range(Layers):
+        decoder_self_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)
+        decoder_cross_attention_block = MultiHeadAttentionBlock(d_model, heads, dropout)
+        decoder_feed_forward_block = FeedForwardBlock(d_model, d_ff, dropout)
+        decoder_block = DecoderBlock(decoder_self_attention_block, decoder_cross_attention_block, decoder_feed_forward_block, dropout)
+        DecoderBlocks.append(decoder_block)
+
+
+    encoder = Encoder(nn.ModuleList(EncoderBlocks))
+    decoder = Decoder(nn.ModuleList(DecoderBlocks))
+
+    linear = LinearLayer(d_model, target_vocab_size)
+
+    Transformer = TransformerBlock(encoder, decoder, source_embedding, target_embedding, source_position, target_position, linear)
+
+    for T in Transformer.parameters():
+        if T.dim() > 1:
+            nn.init.xavier_uniform(T)
+
+    return Transformer

train.py

@@ -0,0 +1,203 @@
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader, random_split
+from torch.utils.tensorboard import SummaryWriter
+
+from model import TransformerModel
+from dataset import BilingualDataset, casual_mask
+from configuration import Get_configuration, Get_weights_file_path, latest_weights_file_path
+
+from datasets import load_dataset
+
+from tokenizers import Tokenizer
+from tokenizers.models import WordLevel
+from tokenizers.pre_tokenizers import Whitespace
+from tokenizers.trainers import WordLevelTrainer
+
+from pathlib import Path 
+
+import warnings
+from tqdm import tqdm
+import os 
+
+def greedy_search(model, source, source_mask, source_tokenizer, target_tokenizer, max_len, device):
+    sos_idx = target_tokenizer.token_to_id('[SOS]')
+    eos_idx = target_tokenizer.token_to_id('[EOS]')
+
+    encoder_output = model.encode(source, source_mask)
+
+    decoder_input = torch.empty(1, 1).fill_(sos_idx).type_as(source).to(device)
+    while True:
+        if decoder_input.size(1) == max_len:
+            break
+
+        decoder_mask = casual_mask(decoder_input.size(1)).type_as(source_mask).to(device)
+
+        out = model.decode(encoder_output, source_mask, decoder_input, decoder_mask)
+
+        # get next token (get the token with the maximum probabilty)
+        prob = model.linear(out[:, -1])
+        _, next_word = torch.max(prob, dim=1)
+        decoder_input = torch.cat(
+            [decoder_input, torch.empty(1, 1).type_as(source).fill_(next_word.item()).to(device)], dim=1
+        )
+
+        if next_word == eos_idx:
+            break
+
+    return decoder_input.squeeze(0)
+
+
+def run_validation(model, validation_ds, source_tokenizer, target_tokenizer, max_len, device, print_msg, global_step, writer, num_examples=2):
+    model.eval()
+    count = 0
+    console_width = 80
+
+    with torch.no_grad():
+        for batch in validation_ds:
+            count += 1
+            encoder_input = batch["encoder_input"].to(device) 
+            encoder_mask = batch["encoder_input_mask"].to(device) 
+
+            assert encoder_input.size(0) == 1, "Batch size must be 1 for validation"
+
+            model_out = greedy_search(model, encoder_input, encoder_mask, source_tokenizer, target_tokenizer, max_len, device)
+
+            source_text = batch["target_text"][0]
+            target_text = batch["target_text"][0]
+            model_out_text = target_tokenizer.decode(model_out.detach().cpu().numpy())
+
+            print_msg('-'*console_width)
+            print_msg(f"{f'SOURCE: ':>12}{source_text}")
+            print_msg(f"{f'TARGET: ':>12}{target_text}")
+            print_msg(f"{f'PREDICTED: ':>12}{model_out_text}")
+
+            if count == num_examples:
+                break
+
+def Get_All_Sentences(dataset, language):
+    for lang in dataset:
+        yield lang['translation'][language]
+
+def Build_Tokenizer(configuration, dataset, language):
+    tokenizer_path = Path(configuration['tokenizer_file'].format(language))
+    if not Path.exists(tokenizer_path):
+        tokenizer = Tokenizer(WordLevel(unk_token= "[UNK]"))
+        tokenizer.pre_tokenizer = Whitespace()
+        trainer = WordLevelTrainer(special_tokens = ["[UNK]", "[PAD]", "[SOS]", "[EOS]"], min_frequency = 2)
+        tokenizer.train_from_iterator(Get_All_Sentences(dataset, language), trainer=trainer)
+        tokenizer.save(str(tokenizer_path))
+    else:
+        tokenizer = Tokenizer.from_file(str(tokenizer_path))
+    return tokenizer 
+
+def Get_dataset(configuration):
+    dataset_Raw = load_dataset(f"{configuration['datasource']}", f"{configuration['source_language']}-{configuration['target_language']}", split="train")
+
+    source_tokenizer = Build_Tokenizer(configuration, dataset_Raw, configuration['source_language'])
+    target_tokenizer = Build_Tokenizer(configuration, dataset_Raw, configuration['target_language'])
+
+    train_dataset_Size = int(0.9 * len(dataset_Raw)) 
+    validation_dataset_Size = len(dataset_Raw) - train_dataset_Size
+
+    train_dataset_Raw, validation_dataset_Raw = random_split(dataset_Raw, [train_dataset_Size, validation_dataset_Size])
+
+    train_dataset = BilingualDataset(train_dataset_Raw, source_tokenizer, target_tokenizer, configuration['source_language'], configuration['target_language'], configuration['sequence_length'])
+    validation_dataset = BilingualDataset(validation_dataset_Raw, source_tokenizer, target_tokenizer, configuration['source_language'], configuration['target_language'], configuration['sequence_length'])
+
+    maximum_source_sequence_length = 0 
+    maximum_target_sequence_length = 0 
+
+    for item in dataset_Raw:
+        source_id = source_tokenizer.encode(item['translation'][configuration['source_language']]).ids
+        target_id = target_tokenizer.encode(item['translation'][configuration['target_language']]).ids
+        maximum_source_sequence_length = max(maximum_source_sequence_length, len(source_id))
+        maximum_target_sequence_length = max(maximum_target_sequence_length, len(target_id))
+
+    print(f"maximum_source_sequence_length : {maximum_source_sequence_length}")
+    print(f"maximum_target_sequence_length: {maximum_target_sequence_length}")
+
+    train_dataLoader = DataLoader(train_dataset, batch_size= configuration['batch_size'], shuffle=True)
+    validation_dataLoader = DataLoader(validation_dataset, batch_size= 1, shuffle=True)
+  
+    return train_dataLoader, validation_dataLoader, source_tokenizer, target_tokenizer
+
+def Get_model(configuration, source_vocab_size, target_vocab_size):
+    model = TransformerModel(source_vocab_size, target_vocab_size, configuration['sequence_length'], configuration['sequence_length'], configuration['d_model'])
+    return model 
+
+def train_model(configuration):
+    device = "cuda" if torch.cuda.is_available() else "mps" if torch.has_mps or torch.backends.mps.is_available() else "cpu"
+    print("Using device:", device)
+
+    Path(f"{configuration['datasource']}_{configuration['model_folder']}").mkdir(parents=True, exist_ok=True)
+
+    train_dataLoader, validation_dataLoader, source_tokenizer, target_tokenizer = Get_dataset(configuration)
+    model = Get_model(configuration, source_tokenizer.get_vocab_size(), target_tokenizer.get_vocab_size()).to(device)
+
+    writer = SummaryWriter(configuration['experiment_name'])
+
+    optimizer = torch.optim.Adam(model.parameters(), lr=configuration['lr'], eps=1e-9)
+
+    initial_epoch = 0
+    global_step = 0
+    preload = configuration['preload']
+    model_filename = latest_weights_file_path(configuration) if preload == 'latest' else Get_weights_file_path(configuration, preload) if preload else None
+    if model_filename:
+        print(f'Preloading model {model_filename}')
+        state = torch.load(model_filename)
+        model.load_state_dict(state['model_state_dict'])
+        initial_epoch = state['epoch'] + 1
+        optimizer.load_state_dict(state['optimizer_state_dict'])
+        global_step = state['global_step']
+    else:
+        print('No model to preload, starting from scratch')
+
+    loss_fn = nn.CrossEntropyLoss(ignore_index=source_tokenizer.token_to_id('[PAD]'), label_smoothing=0.1).to(device)
+
+    for epoch in range(initial_epoch, configuration['num_epochs']):
+        torch.cuda.empty_cache()
+        batch_iterator = tqdm(train_dataLoader, desc=f"Processing Epoch {epoch:02d}")
+        for batch in batch_iterator:
+            model.train()
+            encoder_input = batch['encoder_input'].to(device) 
+            decoder_input = batch['decoder_input'].to(device) 
+            encoder_mask = batch['encoder_input_mask'].to(device) 
+            decoder_mask = batch['encoder_input_mask'].to(device) 
+
+            encoder_output = model.encode(encoder_input, encoder_mask) 
+            decoder_output = model.decode(encoder_output, encoder_mask, decoder_input, decoder_mask) 
+            proj_output = model.linear(decoder_output) 
+
+            Target = batch['Target'].to(device) 
+
+            loss = loss_fn(proj_output.view(-1, target_tokenizer.get_vocab_size()), Target.view(-1))
+            batch_iterator.set_postfix({"loss": f"{loss.item():6.3f}"})
+
+            writer.add_scalar('train loss', loss.item(), global_step)
+            writer.flush()
+
+            loss.backward()
+
+            optimizer.step()
+            optimizer.zero_grad(set_to_none=True)
+
+            # run_validation(model, validation_dataLoader, source_tokenizer, target_tokenizer, configuration['sequence_length'], device, lambda msg: batch_iterator.write(msg), global_step, writer)
+
+            global_step += 1
+
+
+        model_filename = Get_weights_file_path(configuration, f"{epoch:02d}")
+        torch.save({
+            'epoch': epoch,
+            'model_state_dict': model.state_dict(),
+            'optimizer_state_dict': optimizer.state_dict(),
+            'global_step': global_step
+        }, model_filename)
+
+
+
+if __name__ == '__main__':
+    warnings.filterwarnings("ignore")
+    configuration = Get_configuration()
+    train_model(configuration)