Upload model

config.json

@@ -0,0 +1,15 @@
+{
+  "architectures": [
+    "ThaiLightWeightEncoderModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration.ThaiLightWeightEncoderConfig",
+    "AutoModel": "encoder.ThaiLightWeightEncoderModel"
+  },
+  "dropout": 0.2,
+  "final_embedding_dim": 512,
+  "input_embedding_dim": 300,
+  "torch_dtype": "float32",
+  "transformers_version": "4.28.1",
+  "word_vector_model_name": "thai2fit_wv"
+}

configuration.py

@@ -0,0 +1,19 @@
+from transformers import PretrainedConfig
+from typing import List
+
+
+class ThaiLightWeightEncoderConfig(PretrainedConfig):
+
+    def __init__(
+        self,
+        input_embedding_dim: int = 300,
+        final_embedding_dim: int = 512,
+        dropout: float = 0.2,
+        word_vector_model_name: str = "thai2fit_wv",
+        **kwargs,
+    ):
+        self.input_embedding_dim = input_embedding_dim
+        self.final_embedding_dim = final_embedding_dim
+        self.word_vector_model_name = word_vector_model_name
+        self.dropout = dropout
+        super().__init__(**kwargs)

encoder.py

@@ -0,0 +1,26 @@
+from transformers import PreTrainedModel
+
+from pythainlp import word_vector
+import torch
+
+from .configuration import ThaiLightWeightEncoderConfig
+from .projector import Projector
+
+
+class ThaiLightWeightEncoderModel(PreTrainedModel):
+    config_class = ThaiLightWeightEncoderConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.wv = word_vector.WordVector(model_name=config.word_vector_model_name)
+        self.projector = Projector(
+            input_embedding_dim=config.input_embedding_dim,
+            final_embedding_dim=config.final_embedding_dim,
+            dropout=config.dropout
+        )
+    
+    def forward(self, text: str):
+        embed = self.wv.sentence_vectorizer(text, use_mean=True)[0]
+        proj_embed = self.projector(torch.from_numpy(embed).float())
+        proj_embed = proj_embed.to("cpu").detach().numpy()
+        return proj_embed

projector.py

@@ -0,0 +1,29 @@
+import torch
+from torch import nn
+import numpy as np
+
+
+class Projector(nn.Module):
+    def __init__(
+        self,
+        input_embedding_dim: int = 300,
+        final_embedding_dim: int = 512,
+        dropout: float = 0.2
+    ):
+        super().__init__()
+        self.fc1 = nn.Linear(input_embedding_dim, 512)
+        self.fn1 = nn.LeakyReLU()
+        self.fc2 = nn.Linear(512, final_embedding_dim)
+        self.fn2 = nn.LeakyReLU()
+        self.dropout = nn.Dropout(dropout)
+        self.layer_norm = nn.LayerNorm(final_embedding_dim)
+        self.temperature = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.fn1(x)
+        x = self.dropout(x)
+        x = self.fc2(x)
+        x = self.fn2(x)
+        x = self.layer_norm(x)
+        return x

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba76acb7fec448de261d0b0af611fe61f1a7a2b49be1b2cecfd0828d90d8d628
+size 1673609