Upload Tweet_Financial_News_Classification.ipynb

Tweet_Financial_News_Classification.ipynb

@@ -0,0 +1,549 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "TPU"
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "import warnings\n",
+        "warnings.filterwarnings('ignore')\n",
+        "\n",
+        "import transformers\n",
+        "transformers_version = transformers.__version__\n",
+        "\n",
+        "if transformers_version > '4.31.1':\n",
+        "  !pip uninstall transformers\n",
+        "  !pip install transformers==4.31\n",
+        "else:\n",
+        "  print(\"transformers version:\", transformers.__version__)"
+      ],
+      "metadata": {
+        "id": "2RcFPIqQJ6CY",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "8030dedf-b9f5-4687-ef87-1c5a4d8ee9b9"
+      },
+      "execution_count": 1,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Found existing installation: transformers 4.31.0\n",
+            "Uninstalling transformers-4.31.0:\n",
+            "  Would remove:\n",
+            "    /usr/local/bin/transformers-cli\n",
+            "    /usr/local/lib/python3.10/dist-packages/transformers-4.31.0.dist-info/*\n",
+            "    /usr/local/lib/python3.10/dist-packages/transformers/*\n",
+            "Proceed (Y/n)? n\n",
+            "\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/usr/local/lib/python3.10/dist-packages)\u001b[0m\u001b[33m\n",
+            "\u001b[0mRequirement already satisfied: transformers==4.31 in /usr/local/lib/python3.10/dist-packages (4.31.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (3.13.4)\n",
+            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (0.20.3)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (1.25.2)\n",
+            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (24.0)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (6.0.1)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (2023.12.25)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (2.31.0)\n",
+            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (0.13.3)\n",
+            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (0.4.3)\n",
+            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.31) (4.66.2)\n",
+            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31) (2023.6.0)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31) (4.11.0)\n",
+            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31) (3.3.2)\n",
+            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31) (3.7)\n",
+            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31) (2.0.7)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.31) (2024.2.2)\n",
+            "\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/usr/local/lib/python3.10/dist-packages)\u001b[0m\u001b[33m\n",
+            "\u001b[0m"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import tensorflow as tf\n",
+        "print(\"TensorFlow version:\", tf.__version__)\n",
+        "\n",
+        "import keras\n",
+        "print(\"Keras version:\", keras.__version__)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "b_0OPx3WukSi",
+        "outputId": "0d205aa3-33b4-4a34-9055-d670cc5ac049"
+      },
+      "execution_count": 2,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "TensorFlow version: 2.15.0\n",
+            "Keras version: 2.15.0\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "id": "WkzyTQGqzbPS",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "9bc0c671-8557-4b3c-a120-0237d7f96253"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Mounted at /content/drive\n"
+          ]
+        }
+      ],
+      "source": [
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Loading the Data ###"
+      ],
+      "metadata": {
+        "id": "BKn5EaROLKeX"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import pandas as pd\n",
+        "\n",
+        "# Load the CSV file in memory\n",
+        "train_path = '/content/drive/MyDrive/dataset/Twitter_Financial_News_Sentiment/train.csv'\n",
+        "test_path = '/content/drive/MyDrive/dataset/Twitter_Financial_News_Sentiment/test.csv'\n",
+        "\n",
+        "train_df = pd.read_csv(train_path, usecols=['text', 'label'])\n",
+        "test_df = pd.read_csv(test_path, usecols=['text', 'label'])"
+      ],
+      "metadata": {
+        "id": "QztIz9VOKLuV"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Show example"
+      ],
+      "metadata": {
+        "id": "hn5ONAwkNeFS"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "train_df.head()"
+      ],
+      "metadata": {
+        "id": "zwYzU-dANpJ-"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "source": [
+        "#import matplotlib library\n",
+        "from matplotlib import pyplot as plt\n",
+        "\n",
+        "#Histogram of \"Label\" column in train datset\n",
+        "train_df['label'].plot(kind='hist', title='Label')\n",
+        "plt.gca().spines[['top', 'right']].set_visible(False)"
+      ],
+      "cell_type": "code",
+      "execution_count": null,
+      "outputs": [],
+      "metadata": {
+        "id": "2M1XLsAeN2GN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "test_df.head()"
+      ],
+      "metadata": {
+        "id": "g5_oGvo1NvON"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Pritn theshape of datasets\n",
+        "print(f'train_df shape: {train_df.shape}')\n",
+        "print(f'test_df shape: {test_df.shape}')"
+      ],
+      "metadata": {
+        "id": "kCFupI1FQlMF"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Removing the Special Characters ###"
+      ],
+      "metadata": {
+        "id": "zRcmc15aSNx6"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "\n",
+        "!pip install text_hammer\n",
+        "\n",
+        "import text_hammer as th\n",
+        "\n",
+        "def text_proccessing(df, col_name):\n",
+        "  \"\"\"\n",
+        "  Process text data in a DataFrame column by performing the following operations:\n",
+        "\n",
+        "  1. Convert text to lowercase.\n",
+        "  2. Remove emails from the text.\n",
+        "  3. Remove accented characters from the text.\n",
+        "  4. Remove URLs from the text.\n",
+        "\n",
+        "  Parameters:\n",
+        "  df (DataFrame): Input DataFrame containing text data.\n",
+        "  col_name (str): Name of the column in the DataFrame containing text data.\n",
+        "\n",
+        "  Returns:\n",
+        "  DataFrame: Processed DataFrame with text data after applying the specified operations.\n",
+        "  \"\"\"\n",
+        "\n",
+        "  # df[col_name] = df[col_name].apply(lambda x:str(x).lower())\n",
+        "  df[col_name] = df[col_name].apply(lambda x: th.remove_emails(x))\n",
+        "  df[col_name] = df[col_name].apply(lambda x: th.remove_accented_chars(x))\n",
+        "  df[col_name] = df[col_name].apply(lambda x: th.remove_urls(x))\n",
+        "\n",
+        "  return df\n",
+        "\n",
+        "train_df = text_proccessing(train_df, 'text')\n"
+      ],
+      "metadata": {
+        "id": "YEMq7SUiS28e"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Print the first sample after cleaning data\n",
+        "train_df['text'].iloc[0:10]"
+      ],
+      "metadata": {
+        "id": "VD92IEhPZQHm"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "###Loading PreTrained BERT Model###"
+      ],
+      "metadata": {
+        "id": "YfH0H1W6c0Bb"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from transformers import AutoTokenizer, TFBertModel\n",
+        "tokenizer =  AutoTokenizer.from_pretrained('bert-base-uncased')\n",
+        "bert = TFBertModel.from_pretrained('bert-base-uncased')\n"
+      ],
+      "metadata": {
+        "id": "ejMMzCOecze9"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "tokenizer(train_df['text'].iloc[0])"
+      ],
+      "metadata": {
+        "id": "PVWkIfE5gLOV"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "max_len = max([len(x.split()) for x in train_df.text])\n",
+        "print(f'Max len of tweets: {max_len}')"
+      ],
+      "metadata": {
+        "id": "dGANUQVdhHH7"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "x_train = tokenizer(\n",
+        "                text = train_df.text.tolist(),\n",
+        "                padding = True,\n",
+        "                max_length= 36,\n",
+        "                truncation= True,\n",
+        "                return_tensors = 'tf')\n",
+        "\n",
+        "print(x_train)"
+      ],
+      "metadata": {
+        "id": "q9b4iDZ0jW5-"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "print(x_train['input_ids'].shape)\n",
+        "print(x_train['attention_mask'].shape)"
+      ],
+      "metadata": {
+        "id": "PUMeXfO8lgNd"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "print(train_df.label.value_counts())"
+      ],
+      "metadata": {
+        "id": "RMM1QI3DlpmD"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "y_train = train_df.label.values\n",
+        "y_train\n"
+      ],
+      "metadata": {
+        "id": "4zFkagLml80z"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Building the Model Architecture ###"
+      ],
+      "metadata": {
+        "id": "fFQNe5Cimwxn"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from keras import layers, Model\n",
+        "\n",
+        "max_length = 36\n",
+        "\n",
+        "input_ids = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"input_ids\")\n",
+        "input_mask = layers.Input(shape=(max_length,), dtype=tf.int32, name=\"attention_mask\")\n",
+        "\n",
+        "embeddings = bert(input_ids,attention_mask = input_mask)[1] #(0 is the last hidden states,1 means pooler_output)\n",
+        "\n",
+        "out = layers.Dropout(0.1)(embeddings)\n",
+        "out = layers.Dense(128, activation='relu')(out)\n",
+        "out = layers.Dropout(0.1)(out)\n",
+        "out = layers.Dense(32,activation = 'relu')(out)\n",
+        "\n",
+        "y = layers.Dense(3,activation = 'softmax')(out)\n",
+        "\n",
+        "model = tf.keras.Model(inputs=[input_ids, input_mask], outputs=y)\n",
+        "model.layers[2].trainable = False"
+      ],
+      "metadata": {
+        "id": "DE1XbnVomwMc"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "model.summary()"
+      ],
+      "metadata": {
+        "id": "GuxGCjYjrTyY"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from keras.optimizers import Adam\n",
+        "\n",
+        "optimizer = Adam(\n",
+        "    learning_rate = 6e-06, # this learning rate is for bert model , taken from huggingface website\n",
+        "    epsilon=1e-08,\n",
+        "    weight_decay=0.01)\n",
+        "\n",
+        "# Compile the model\n",
+        "model.compile(\n",
+        "    optimizer = optimizer,\n",
+        "    loss = 'sparse_categorical_crossentropy',\n",
+        "    metrics = [\"sparse_categorical_accuracy\"])"
+      ],
+      "metadata": {
+        "id": "FyyNrAAf7QMP"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "train_history = model.fit(\n",
+        "    x = {'input_ids':x_train['input_ids'], 'attention_mask':x_train['attention_mask']} ,\n",
+        "    y = y_train,\n",
+        "    validation_split = 0.1,\n",
+        "    epochs= 3,\n",
+        "    batch_size= 32)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "bEnttT2rA8Yw",
+        "outputId": "644c03fd-0cc0-40ff-8108-e059e3a4a0dd"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Epoch 1/3\n",
+            "118/269 [============>.................] - ETA: 10:10 - loss: 0.9140 - sparse_categorical_accuracy: 0.6261"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#### TESTING PHASE\n",
+        "on this phase we will make predictions out of our model"
+      ],
+      "metadata": {
+        "id": "hgiDVRwSBtCN"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "x_test = tokenizer(\n",
+        "    text = test_df.text.tolist(),\n",
+        "    padding= True,\n",
+        "    max_length= 36,\n",
+        "    truncation = True,\n",
+        "    return_tensors= 'tf')"
+      ],
+      "metadata": {
+        "id": "xaKYd2PRBySe"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "y_test = test_df.label.values\n",
+        "y_test"
+      ],
+      "metadata": {
+        "id": "OpvHTg3atflb"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "predicted = model.predict({'input_ids':x_test['input_ids'],'attention_mask':x_test['attention_mask']})"
+      ],
+      "metadata": {
+        "id": "nWgCdpKvCSWm"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from sklearn.metrics import confusion_matrix\n",
+        "import seaborn as sns\n",
+        "\n",
+        "# Convert the predictions to binary values (0 or 1)\n",
+        "y_pred_binary = [int(round(x[0])) for x in predicted]\n",
+        "\n",
+        "# Generate the confusion matrix\n",
+        "cm = confusion_matrix(test_df['label'], y_pred_binary)\n",
+        "\n",
+        "# Create a heatmap of the confusion matrix\n",
+        "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
+        "plt.xlabel(\"Predicted Label\")\n",
+        "plt.ylabel(\"True Label\")\n",
+        "plt.title(\"Confusion Matrix\")\n",
+        "plt.show()"
+      ],
+      "metadata": {
+        "id": "-BICUoNs_8qI"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}