Upload 4 files

Dockerfile

@@ -11,4 +11,4 @@ COPY requirements.txt /deneme/requirements.txt
 RUN pip install --no-cache-dir --upgrade -r /deneme/requirements.txt
 
 # Uygulamanızı başlatma komutu
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+CMD ["uvicorn", "app:app2", "--host", "0.0.0.0", "--port", "7860"]

load2.ipynb

@@ -9,23 +9,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'datasets'",
+     "ename": "ValueError",
+     "evalue": "numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject",
      "output_type": "error",
      "traceback": [
       "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[1], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dataset\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m \n",
-      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'datasets'"
+      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[5], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdatasets\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m load_dataset\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m \n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\datasets\\__init__.py:17\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[0;32m     15\u001b[0m __version__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2.20.0\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m---> 17\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrow_dataset\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Dataset\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrow_reader\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ReadInstruction\n\u001b[0;32m     19\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbuilder\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\datasets\\arrow_dataset.py:59\u001b[0m\n\u001b[0;32m     57\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mfsspec\u001b[39;00m\n\u001b[0;32m     58\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m---> 59\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpa\u001b[39;00m\n\u001b[0;32m     61\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpyarrow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcompute\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpc\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\pandas\\__init__.py:50\u001b[0m\n\u001b[0;32m     43\u001b[0m     _module \u001b[38;5;241m=\u001b[39m _err\u001b[38;5;241m.\u001b[39mname\n\u001b[0;32m     44\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[0;32m     45\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mC extension: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m_module\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m not built. If you want to import \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     46\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpandas from the source directory, you may need to run \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     47\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpython setup.py build_ext\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m to build the C extensions first.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m     48\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m_err\u001b[39;00m\n\u001b[1;32m---> 50\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     51\u001b[0m     get_option,\n\u001b[0;32m     52\u001b[0m     set_option,\n\u001b[0;32m     53\u001b[0m     reset_option,\n\u001b[0;32m     54\u001b[0m     describe_option,\n\u001b[0;32m     55\u001b[0m     option_context,\n\u001b[0;32m     56\u001b[0m     options,\n\u001b[0;32m     57\u001b[0m )\n\u001b[0;32m     59\u001b[0m \u001b[38;5;66;03m# let init-time option registration happen\u001b[39;00m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig_init\u001b[39;00m  \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport] # noqa: F401\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\pandas\\_config\\__init__.py:20\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;124;03mpandas._config is considered explicitly upstream of everything else in pandas,\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;124;03mshould have no intra-pandas dependencies.\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[38;5;124;03mare initialized.\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m      8\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m      9\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mconfig\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     10\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdetect_console_encoding\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwarn_copy_on_write\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     19\u001b[0m ]\n\u001b[1;32m---> 20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m config\n\u001b[0;32m     21\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dates  \u001b[38;5;66;03m# pyright: ignore[reportUnusedImport]  # noqa: F401\u001b[39;00m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_config\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconfig\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     23\u001b[0m     _global_config,\n\u001b[0;32m     24\u001b[0m     describe_option,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     29\u001b[0m     set_option,\n\u001b[0;32m     30\u001b[0m )\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\pandas\\_config\\config.py:68\u001b[0m\n\u001b[0;32m     58\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     59\u001b[0m     TYPE_CHECKING,\n\u001b[0;32m     60\u001b[0m     Any,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     64\u001b[0m     cast,\n\u001b[0;32m     65\u001b[0m )\n\u001b[0;32m     66\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwarnings\u001b[39;00m\n\u001b[1;32m---> 68\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_typing\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     69\u001b[0m     F,\n\u001b[0;32m     70\u001b[0m     T,\n\u001b[0;32m     71\u001b[0m )\n\u001b[0;32m     72\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutil\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_exceptions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m find_stack_level\n\u001b[0;32m     74\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m TYPE_CHECKING:\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\pandas\\_typing.py:198\u001b[0m\n\u001b[0;32m    192\u001b[0m Frequency \u001b[38;5;241m=\u001b[39m Union[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mBaseOffset\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m    193\u001b[0m Axes \u001b[38;5;241m=\u001b[39m ListLike\n\u001b[0;32m    195\u001b[0m RandomState \u001b[38;5;241m=\u001b[39m Union[\n\u001b[0;32m    196\u001b[0m     \u001b[38;5;28mint\u001b[39m,\n\u001b[0;32m    197\u001b[0m     np\u001b[38;5;241m.\u001b[39mndarray,\n\u001b[1;32m--> 198\u001b[0m     \u001b[43mnp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrandom\u001b[49m\u001b[38;5;241m.\u001b[39mGenerator,\n\u001b[0;32m    199\u001b[0m     np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mBitGenerator,\n\u001b[0;32m    200\u001b[0m     np\u001b[38;5;241m.\u001b[39mrandom\u001b[38;5;241m.\u001b[39mRandomState,\n\u001b[0;32m    201\u001b[0m ]\n\u001b[0;32m    203\u001b[0m \u001b[38;5;66;03m# dtypes\u001b[39;00m\n\u001b[0;32m    204\u001b[0m NpDtype \u001b[38;5;241m=\u001b[39m Union[\u001b[38;5;28mstr\u001b[39m, np\u001b[38;5;241m.\u001b[39mdtype, type_t[Union[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mcomplex\u001b[39m, \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mobject\u001b[39m]]]\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\numpy\\__init__.py:351\u001b[0m, in \u001b[0;36m__getattr__\u001b[1;34m(attr)\u001b[0m\n\u001b[0;32m    350\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__dir__\u001b[39m():\n\u001b[1;32m--> 351\u001b[0m     public_symbols \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mglobals\u001b[39m()\u001b[38;5;241m.\u001b[39mkeys() \u001b[38;5;241m|\u001b[39m {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtesting\u001b[39m\u001b[38;5;124m'\u001b[39m}\n\u001b[0;32m    352\u001b[0m     public_symbols \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m    353\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcore\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmatrixlib\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    354\u001b[0m         \u001b[38;5;66;03m# These were moved in 1.25 and may be deprecated eventually:\u001b[39;00m\n\u001b[0;32m    355\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mModuleDeprecationWarning\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mVisibleDeprecationWarning\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m    356\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mComplexWarning\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTooHardError\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAxisError\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    357\u001b[0m     }\n\u001b[0;32m    358\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(public_symbols)\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\numpy\\random\\__init__.py:180\u001b[0m\n\u001b[0;32m    126\u001b[0m __all__ \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m    127\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbeta\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m    128\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbinomial\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    176\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mzipf\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m    177\u001b[0m ]\n\u001b[0;32m    179\u001b[0m \u001b[38;5;66;03m# add these for module-freeze analysis (like PyInstaller)\u001b[39;00m\n\u001b[1;32m--> 180\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _pickle\n\u001b[0;32m    181\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _common\n\u001b[0;32m    182\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m _bounded_integers\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\numpy\\random\\_pickle.py:1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmtrand\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RandomState\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_philox\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Philox\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_pcg64\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m PCG64, PCG64DXSM\n",
+      "File \u001b[1;32mnumpy\\\\random\\\\mtrand.pyx:1\u001b[0m, in \u001b[0;36minit numpy.random.mtrand\u001b[1;34m()\u001b[0m\n",
+      "\u001b[1;31mValueError\u001b[0m: numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject"
      ]
     }
    ],
    "source": [
-    "import datasets\n",
     "from datasets import load_dataset\n",
     "import pandas as pd \n"
    ]
@@ -111,7 +120,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -121,7 +130,7 @@
      "traceback": [
       "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[9], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m#train  ve test dosyaları oluşturma \u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mtrain_df\u001b[49m\u001b[38;5;241m.\u001b[39mto_parquet(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mgitProjects\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mdeneme\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124megitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mtrain_Egitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mmerged_train.parquet\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      3\u001b[0m test_df\u001b[38;5;241m.\u001b[39mto_parquet(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mgitProjects\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mdeneme\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mtest_Egitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mmerged_train.parquet\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "Cell \u001b[1;32mIn[2], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m#train  ve test dosyaları oluşturma \u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[43mtrain_df\u001b[49m\u001b[38;5;241m.\u001b[39mto_parquet(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mgitProjects\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mdeneme\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124megitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mtrain_Egitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mmerged_train.parquet\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m      3\u001b[0m test_df\u001b[38;5;241m.\u001b[39mto_parquet(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mC:\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mgitProjects\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mdeneme\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mtest_Egitim\u001b[39m\u001b[38;5;130;01m\\\\\u001b[39;00m\u001b[38;5;124mmerged_train.parquet\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
       "\u001b[1;31mNameError\u001b[0m: name 'train_df' is not defined"
      ]
     }
@@ -134,7 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -226,7 +235,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -236,7 +245,7 @@
      "traceback": [
       "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
-      "Cell \u001b[1;32mIn[13], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AutoModel,AutoTokenizer,AutoModelForSeq2SeqLM\n\u001b[0;32m      2\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m AutoTokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mphilschmid/bart-large-cnn-samsum\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m      3\u001b[0m model \u001b[38;5;241m=\u001b[39m AutoModelForSeq2SeqLM\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mphilschmid/bart-large-cnn-samsum\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
+      "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m T5Tokenizer, T5ForConditionalGeneration\n\u001b[0;32m      3\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m T5Tokenizer\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgoogle/flan-t5-xxl\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m      4\u001b[0m model \u001b[38;5;241m=\u001b[39m T5ForConditionalGeneration\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgoogle/flan-t5-xxl\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
       "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\__init__.py:26\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TYPE_CHECKING\n\u001b[0;32m     25\u001b[0m \u001b[38;5;66;03m# Check the dependencies satisfy the minimal versions required.\u001b[39;00m\n\u001b[1;32m---> 26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dependency_versions_check\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     28\u001b[0m     OptionalDependencyNotAvailable,\n\u001b[0;32m     29\u001b[0m     _LazyModule,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     48\u001b[0m     logging,\n\u001b[0;32m     49\u001b[0m )\n\u001b[0;32m     52\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mget_logger(\u001b[38;5;18m__name__\u001b[39m)  \u001b[38;5;66;03m# pylint: disable=invalid-name\u001b[39;00m\n",
       "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\dependency_versions_check.py:16\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Copyright 2020 The HuggingFace Team. All rights reserved.\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdependency_versions_table\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m deps\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mversions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m require_version, require_version_core\n\u001b[0;32m     19\u001b[0m \u001b[38;5;66;03m# define which module versions we always want to check at run time\u001b[39;00m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;66;03m# (usually the ones defined in `install_requires` in setup.py)\u001b[39;00m\n\u001b[0;32m     21\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;66;03m# order specific notes:\u001b[39;00m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;66;03m# - tqdm must be checked before tokenizers\u001b[39;00m\n\u001b[0;32m     25\u001b[0m pkgs_to_check_at_runtime \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     26\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     27\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtqdm\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpyyaml\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     38\u001b[0m ]\n",
       "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\utils\\__init__.py:34\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdoc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     27\u001b[0m     add_code_sample_docstrings,\n\u001b[0;32m     28\u001b[0m     add_end_docstrings,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     32\u001b[0m     replace_return_docstrings,\n\u001b[0;32m     33\u001b[0m )\n\u001b[1;32m---> 34\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgeneric\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     35\u001b[0m     ContextManagers,\n\u001b[0;32m     36\u001b[0m     ExplicitEnum,\n\u001b[0;32m     37\u001b[0m     ModelOutput,\n\u001b[0;32m     38\u001b[0m     PaddingStrategy,\n\u001b[0;32m     39\u001b[0m     TensorType,\n\u001b[0;32m     40\u001b[0m     add_model_info_to_auto_map,\n\u001b[0;32m     41\u001b[0m     add_model_info_to_custom_pipelines,\n\u001b[0;32m     42\u001b[0m     cached_property,\n\u001b[0;32m     43\u001b[0m     can_return_loss,\n\u001b[0;32m     44\u001b[0m     expand_dims,\n\u001b[0;32m     45\u001b[0m     filter_out_non_signature_kwargs,\n\u001b[0;32m     46\u001b[0m     find_labels,\n\u001b[0;32m     47\u001b[0m     flatten_dict,\n\u001b[0;32m     48\u001b[0m     infer_framework,\n\u001b[0;32m     49\u001b[0m     is_jax_tensor,\n\u001b[0;32m     50\u001b[0m     is_numpy_array,\n\u001b[0;32m     51\u001b[0m     is_tensor,\n\u001b[0;32m     52\u001b[0m     is_tf_symbolic_tensor,\n\u001b[0;32m     53\u001b[0m     is_tf_tensor,\n\u001b[0;32m     54\u001b[0m     is_torch_device,\n\u001b[0;32m     55\u001b[0m     is_torch_dtype,\n\u001b[0;32m     56\u001b[0m     is_torch_tensor,\n\u001b[0;32m     57\u001b[0m     reshape,\n\u001b[0;32m     58\u001b[0m     squeeze,\n\u001b[0;32m     59\u001b[0m     strtobool,\n\u001b[0;32m     60\u001b[0m     tensor_size,\n\u001b[0;32m     61\u001b[0m     to_numpy,\n\u001b[0;32m     62\u001b[0m     to_py_obj,\n\u001b[0;32m     63\u001b[0m     torch_float,\n\u001b[0;32m     64\u001b[0m     torch_int,\n\u001b[0;32m     65\u001b[0m     transpose,\n\u001b[0;32m     66\u001b[0m     working_or_temp_dir,\n\u001b[0;32m     67\u001b[0m )\n\u001b[0;32m     68\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     69\u001b[0m     CLOUDFRONT_DISTRIB_PREFIX,\n\u001b[0;32m     70\u001b[0m     HF_MODULES_CACHE,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     96\u001b[0m     try_to_load_from_cache,\n\u001b[0;32m     97\u001b[0m )\n\u001b[0;32m     98\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mimport_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     99\u001b[0m     ACCELERATE_MIN_VERSION,\n\u001b[0;32m    100\u001b[0m     ENV_VARS_TRUE_AND_AUTO_VALUES,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    219\u001b[0m     torch_only_method,\n\u001b[0;32m    220\u001b[0m )\n",
@@ -247,9 +256,10 @@
     }
    ],
    "source": [
-    "from transformers import AutoModel,AutoTokenizer,AutoModelForSeq2SeqLM\n",
-    "tokenizer = AutoTokenizer.from_pretrained(\"philschmid/bart-large-cnn-samsum\")\n",
-    "model = AutoModelForSeq2SeqLM.from_pretrained(\"philschmid/bart-large-cnn-samsum\")\n"
+    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
+    "\n",
+    "tokenizer = T5Tokenizer.from_pretrained(\"google/flan-t5-xxl\")\n",
+    "model = T5ForConditionalGeneration.from_pretrained(\"google/flan-t5-xxl\")"
    ]
   },
   {
@@ -260,11 +270,11 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
-   "source": [
-    "Train verilerinin moongodb yhe yüklenmesi "
-   ]
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "code",
@@ -314,11 +324,11 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
-   "source": [
-    "Test verilerinin monngodb yhe eklenmesi "
-   ]
+   "outputs": [],
+   "source": []
   },
   {
    "cell_type": "code",
@@ -367,70 +377,42 @@
     "source_collection = dataset_read()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Model eğitimi için tokenleştirme "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pymongo import MongoClient\n",
-    "\n",
-    "def get_mongodb():\n",
-    "    # MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmalıdır.\n",
-    "    return 'mongodb://localhost:27017/', 'yeniDatabase', 'test'\n",
-    "\n",
-    "def get_average_prompt_token_length():\n",
-    "    # MongoDB bağlantı bilgilerini alma\n",
-    "    mongo_url, db_name, collection_name = get_mongodb()\n",
-    "\n",
-    "    # MongoDB'ye bağlanma\n",
-    "    client = MongoClient(mongo_url)\n",
-    "    db = client[db_name]\n",
-    "    collection = db[collection_name]\n",
-    "\n",
-    "    # Tüm dökümanları çekme ve 'prompt_token_length' alanını alma\n",
-    "    docs = collection.find({}, {'Prompt_token_length': 1})\n",
-    "\n",
-    "    # 'prompt_token_length' değerlerini toplama ve sayma\n",
-    "    total_length = 0\n",
-    "    count = 0\n",
-    "\n",
-    "    for doc in docs:\n",
-    "        if 'Prompt_token_length' in doc:\n",
-    "            total_length += doc['Prompt_token_length']\n",
-    "            count += 1\n",
-    "    \n",
-    "    # Ortalama hesaplama\n",
-    "    if count > 0:\n",
-    "        average_length = total_length / count\n",
-    "    else:\n",
-    "        average_length = 0  # Eğer 'prompt_token_length' alanı olan döküman yoksa\n",
-    "\n",
-    "    return int(average_length)\n",
-    "\n",
-    "# Ortalama prompt token uzunluğunu al ve yazdır\n",
-    "average_length = get_average_prompt_token_length()\n",
-    "print(f\"Ortalama prompt token uzunluğu: {average_length}\")\n"
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "ename": "OSError",
+     "evalue": "[WinError 126] Belirtilen modül bulunamadı. Error loading \"c:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\torch\\lib\\fbgemm.dll\" or one of its dependencies.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
+      "Cell \u001b[1;32mIn[1], line 2\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# uygulama için kullanılcak olan özelliklerin tanımlanması\u001b[39;00m\n\u001b[1;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtransformers\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m BertTokenizer,BertForQuestionAnswering,BertConfig\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mQA\u001b[39;00m:\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m,model_path: \u001b[38;5;28mstr\u001b[39m):\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\__init__.py:26\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TYPE_CHECKING\n\u001b[0;32m     25\u001b[0m \u001b[38;5;66;03m# Check the dependencies satisfy the minimal versions required.\u001b[39;00m\n\u001b[1;32m---> 26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m dependency_versions_check\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     28\u001b[0m     OptionalDependencyNotAvailable,\n\u001b[0;32m     29\u001b[0m     _LazyModule,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     48\u001b[0m     logging,\n\u001b[0;32m     49\u001b[0m )\n\u001b[0;32m     52\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mget_logger(\u001b[38;5;18m__name__\u001b[39m)  \u001b[38;5;66;03m# pylint: disable=invalid-name\u001b[39;00m\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\dependency_versions_check.py:16\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m# Copyright 2020 The HuggingFace Team. All rights reserved.\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;66;03m# Licensed under the Apache License, Version 2.0 (the \"License\");\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[38;5;66;03m# See the License for the specific language governing permissions and\u001b[39;00m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;66;03m# limitations under the License.\u001b[39;00m\n\u001b[0;32m     15\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdependency_versions_table\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m deps\n\u001b[1;32m---> 16\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mversions\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m require_version, require_version_core\n\u001b[0;32m     19\u001b[0m \u001b[38;5;66;03m# define which module versions we always want to check at run time\u001b[39;00m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;66;03m# (usually the ones defined in `install_requires` in setup.py)\u001b[39;00m\n\u001b[0;32m     21\u001b[0m \u001b[38;5;66;03m#\u001b[39;00m\n\u001b[0;32m     22\u001b[0m \u001b[38;5;66;03m# order specific notes:\u001b[39;00m\n\u001b[0;32m     23\u001b[0m \u001b[38;5;66;03m# - tqdm must be checked before tokenizers\u001b[39;00m\n\u001b[0;32m     25\u001b[0m pkgs_to_check_at_runtime \u001b[38;5;241m=\u001b[39m [\n\u001b[0;32m     26\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpython\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     27\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtqdm\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     37\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpyyaml\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m     38\u001b[0m ]\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\utils\\__init__.py:34\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mconstants\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD\n\u001b[0;32m     26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdoc\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     27\u001b[0m     add_code_sample_docstrings,\n\u001b[0;32m     28\u001b[0m     add_end_docstrings,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     32\u001b[0m     replace_return_docstrings,\n\u001b[0;32m     33\u001b[0m )\n\u001b[1;32m---> 34\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgeneric\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     35\u001b[0m     ContextManagers,\n\u001b[0;32m     36\u001b[0m     ExplicitEnum,\n\u001b[0;32m     37\u001b[0m     ModelOutput,\n\u001b[0;32m     38\u001b[0m     PaddingStrategy,\n\u001b[0;32m     39\u001b[0m     TensorType,\n\u001b[0;32m     40\u001b[0m     add_model_info_to_auto_map,\n\u001b[0;32m     41\u001b[0m     add_model_info_to_custom_pipelines,\n\u001b[0;32m     42\u001b[0m     cached_property,\n\u001b[0;32m     43\u001b[0m     can_return_loss,\n\u001b[0;32m     44\u001b[0m     expand_dims,\n\u001b[0;32m     45\u001b[0m     filter_out_non_signature_kwargs,\n\u001b[0;32m     46\u001b[0m     find_labels,\n\u001b[0;32m     47\u001b[0m     flatten_dict,\n\u001b[0;32m     48\u001b[0m     infer_framework,\n\u001b[0;32m     49\u001b[0m     is_jax_tensor,\n\u001b[0;32m     50\u001b[0m     is_numpy_array,\n\u001b[0;32m     51\u001b[0m     is_tensor,\n\u001b[0;32m     52\u001b[0m     is_tf_symbolic_tensor,\n\u001b[0;32m     53\u001b[0m     is_tf_tensor,\n\u001b[0;32m     54\u001b[0m     is_torch_device,\n\u001b[0;32m     55\u001b[0m     is_torch_dtype,\n\u001b[0;32m     56\u001b[0m     is_torch_tensor,\n\u001b[0;32m     57\u001b[0m     reshape,\n\u001b[0;32m     58\u001b[0m     squeeze,\n\u001b[0;32m     59\u001b[0m     strtobool,\n\u001b[0;32m     60\u001b[0m     tensor_size,\n\u001b[0;32m     61\u001b[0m     to_numpy,\n\u001b[0;32m     62\u001b[0m     to_py_obj,\n\u001b[0;32m     63\u001b[0m     torch_float,\n\u001b[0;32m     64\u001b[0m     torch_int,\n\u001b[0;32m     65\u001b[0m     transpose,\n\u001b[0;32m     66\u001b[0m     working_or_temp_dir,\n\u001b[0;32m     67\u001b[0m )\n\u001b[0;32m     68\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     69\u001b[0m     CLOUDFRONT_DISTRIB_PREFIX,\n\u001b[0;32m     70\u001b[0m     HF_MODULES_CACHE,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     96\u001b[0m     try_to_load_from_cache,\n\u001b[0;32m     97\u001b[0m )\n\u001b[0;32m     98\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mimport_utils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[0;32m     99\u001b[0m     ACCELERATE_MIN_VERSION,\n\u001b[0;32m    100\u001b[0m     ENV_VARS_TRUE_AND_AUTO_VALUES,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    219\u001b[0m     torch_only_method,\n\u001b[0;32m    220\u001b[0m )\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\transformers\\utils\\generic.py:462\u001b[0m\n\u001b[0;32m    458\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m(\u001b[38;5;28mself\u001b[39m[k] \u001b[38;5;28;01mfor\u001b[39;00m k \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkeys())\n\u001b[0;32m    461\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_torch_available():\n\u001b[1;32m--> 462\u001b[0m     \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_pytree\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01m_torch_pytree\u001b[39;00m\n\u001b[0;32m    464\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_model_output_flatten\u001b[39m(output: ModelOutput) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[List[Any], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_torch_pytree.Context\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[0;32m    465\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mlist\u001b[39m(output\u001b[38;5;241m.\u001b[39mvalues()), \u001b[38;5;28mlist\u001b[39m(output\u001b[38;5;241m.\u001b[39mkeys())\n",
+      "File \u001b[1;32mc:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\torch\\__init__.py:148\u001b[0m\n\u001b[0;32m    146\u001b[0m                 err \u001b[38;5;241m=\u001b[39m ctypes\u001b[38;5;241m.\u001b[39mWinError(ctypes\u001b[38;5;241m.\u001b[39mget_last_error())\n\u001b[0;32m    147\u001b[0m                 err\u001b[38;5;241m.\u001b[39mstrerror \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m Error loading \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdll\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m or one of its dependencies.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m--> 148\u001b[0m                 \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[0;32m    150\u001b[0m     kernel32\u001b[38;5;241m.\u001b[39mSetErrorMode(prev_error_mode)\n\u001b[0;32m    153\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_preload_cuda_deps\u001b[39m(lib_folder, lib_name):\n",
+      "\u001b[1;31mOSError\u001b[0m: [WinError 126] Belirtilen modül bulunamadı. Error loading \"c:\\gitProjects\\deneme\\.venv\\Lib\\site-packages\\torch\\lib\\fbgemm.dll\" or one of its dependencies."
+     ]
+    }
+   ],
    "source": [
     "# uygulama için kullanılcak olan özelliklerin tanımlanması\n",
     "from transformers import BertTokenizer,BertForQuestionAnswering,BertConfig\n",
     "class QA:\n",
     "    def __init__(self,model_path: str):\n",
-    "        self.max_seq_length = 200 #max seq\n",
+    "        self.max_seq_length = 384 #max seq\n",
     "        self.doc_stride = 128 #stride \n",
     "        self.do_lower_case = False\n",
     "        self.max_query_length = 30\n",
@@ -449,18 +431,11 @@
     "        \n",
     "        # This function is used to load the model\n",
     "    def load_model(self,model_path: str,do_lower_case=False):\n",
-    "        config = BertConfig.from_pretrained(model_path + \"C:\\\\gitProjects\\\\train_Egitim\")\n",
+    "        config = BertConfig.from_pretrained(model_path + \"C:\\\\gitProjects\\\\deneme\\\\egitim\\\\train_Egitim\")\n",
     "        tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=do_lower_case)\n",
     "        model = BertForQuestionAnswering.from_pretrained(model_path, from_tf=False, config=config)\n",
-    "        return model, tokenizer"
+    "        return model, tokenizer\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

model.py

@@ -0,0 +1,179 @@
+from datasets import load_dataset
+import pandas as pd
+import torch
+from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
+from transformers import BertTokenizer, BertForQuestionAnswering, BertConfig
+from pymongo import MongoClient
+
+class Database:
+    @staticmethod
+    def get_mongodb():
+        # MongoDB bağlantı bilgilerini döndürecek şekilde tanımlanmalıdır.
+        return 'mongodb://localhost:27017/', 'yeniDatabase', 'train'
+
+    @staticmethod
+    def get_input_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Prompt": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Prompt": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        input_texts_from_db = list(cursor)
+        # Input text'leri döndürme
+        return input_texts_from_db
+    
+    @staticmethod
+    def get_output_texts():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Sorguyu tanımlama
+        query = {"Response": {"$exists": True}}
+        # Sorguyu çalıştırma ve dökümanları çekme
+        cursor = collection.find(query, {"Response": 1, "_id": 0})
+        # Cursor'ı döküman listesine dönüştürme
+        output_texts_from_db = list(cursor)
+        # Input text'leri döndürme
+        return output_texts_from_db
+
+    @staticmethod
+    def get_average_prompt_token_length():
+        # MongoDB bağlantı bilgilerini alma
+        mongo_url, db_name, collection_name = Database.get_mongodb()
+        # MongoDB'ye bağlanma
+        client = MongoClient(mongo_url)
+        db = client[db_name]
+        collection = db[collection_name]
+        # Tüm dökümanları çekme ve 'prompt_token_length' alanını alma
+        docs = collection.find({}, {'Prompt_token_length': 1})
+        # 'prompt_token_length' değerlerini toplama ve sayma
+        total_length = 0
+        count = 0
+        for doc in docs:
+            if 'Prompt_token_length' in doc:
+                total_length += doc['Prompt_token_length']
+                count += 1
+        # Ortalama hesaplama
+        average_length = total_length / count if count > 0 else 0
+        return int(average_length)
+
+# Tokenizer ve Modeli yükleme
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+# Encode işlemi
+def tokenize_and_encode(train_df,doc):
+
+    input_texts_from_db = Database.get_input_texts()
+    output_texts_from_db= Database.get_output_texts()
+    input_texts = [doc["Prompt"] for doc in input_texts_from_db]
+    output_texts= [doc["Response"] for doc in output_texts_from_db]
+    encoded = tokenizer.batch_encode_plus(
+
+        #doc['Prompt'].tolist(),
+        #text_pair= doc['Response'].tolist(),
+        input_texts,
+        output_texts,
+        padding=True,
+        truncation=True,
+        max_length=100,
+        return_attention_mask=True,
+        return_tensors='pt'
+)
+    return encoded
+encoded_data=tokenize_and_encode()
+
+class QA:
+
+    #buradaki verilerin değeri değiştirilmeli 
+    def __init__(self, model_path: str):
+        self.max_seq_length = 384
+        self.doc_stride = 128
+        self.do_lower_case = False
+        self.max_query_length = 30
+        self.n_best_size = 3
+        self.max_answer_length = 30
+        self.version_2_with_negative = False
+        self.model, self.tokenizer = self.load_model(model_path)
+        self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.model.to(self.device)
+        self.model.eval()
+
+    def load_model(self, model_path: str, do_lower_case=False):
+        config = BertConfig.from_pretrained(model_path)
+        tokenizer = BertTokenizer.from_pretrained(model_path, do_lower_case=do_lower_case)
+        model = BertForQuestionAnswering.from_pretrained(model_path, from_tf=False, config=config)
+        return model, tokenizer
+
+    def extract_features_from_dataset(self, train_df):
+        def get_max_length(examples):
+            return {
+                'max_seq_length': max(len(e) for e in examples),
+                'max_query_length': max(len(q) for q in examples)
+            }
+        # Örnek bir kullanım
+        features = get_max_length(train_df)
+        return features
+
+# Ortalama prompt token uzunluğunu al ve yazdır
+average_length = Database.get_average_prompt_token_length()
+print(f"Ortalama prompt token uzunluğu: {average_length}")
+
+# QA sınıfını oluştur
+qa = QA(model_path='bert-base-uncased')
+
+
+#tensor veri setini koda entegre etme 
+
+"""# Tensor veri kümesi oluşturma
+input_ids = encoded_data['input_ids']
+attention_mask = encoded_data['attention_mask']
+token_type_ids = encoded_data['token_type_ids']
+labels = torch.tensor(data['Response'].tolist())  # Cevapları etiket olarak kullanın
+
+# TensorDataset oluşturma
+dataset = TensorDataset(input_ids, attention_mask, token_type_ids, labels)
+
+# DataLoader oluşturma
+batch_size = 16
+dataloader = DataLoader(
+    dataset,
+    sampler=RandomSampler(dataset),
+    batch_size=batch_size
+)"""
+
+#modelin için epoch sayısının tanımlaması 
+"""# Eğitim için optimizer
+optimizer = AdamW(model.parameters(), lr=5e-5)
+
+# Eğitim döngüsü
+model.train()
+for epoch in range(3):  # Örnek olarak 3 epoch
+    for batch in dataloader:
+        input_ids, attention_mask, token_type_ids, labels = [t.to(device) for t in batch]
+        optimizer.zero_grad()
+        outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, start_positions=labels, end_positions=labels)
+        loss = outputs.loss
+        loss.backward()
+        optimizer.step()
+    print(f"Epoch {epoch+1} loss: {loss.item()}")"""
+
+
+#sonuçların sınıflandırılması
+"""# Modeli değerlendirme aşamasına getirme
+model.eval()
+
+# Örnek tahmin
+with torch.no_grad():
+    for batch in dataloader:
+        input_ids, attention_mask, token_type_ids, _ = [t.to(device) for t in batch]
+        outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
+        # Çıktıları kullanarak başlık, alt başlık ve anahtar kelimeler belirleyebilirsiniz
+"""

requirements.txt

@@ -2,5 +2,5 @@ gradio==4.40.0.*
 pymongo==4.8.0.*
 pandas==2.2.2.*
 datasets==2.20.0.*
-torch==2.4.0.*
+torch
 transformers==4.43.4.*