Initial commit

flagged/log.csv

@@ -0,0 +1,3 @@
+'name','output','flag','username','timestamp'
+'Hello','Hello Hello!!','','','2022-05-28 16:06:47.684383'
+'Hello','Hello Hello!!','','','2022-05-28 16:06:49.325378'

main.py

@@ -0,0 +1,100 @@
+import copy
+
+import torch
+import numpy as np
+import gradio as gr
+from spoter_mod.skeleton_extractor import obtain_pose_data
+from spoter_mod.normalization.body_normalization import normalize_single_dict as normalize_single_body_dict, BODY_IDENTIFIERS
+from spoter_mod.normalization.hand_normalization import normalize_single_dict as normalize_single_hand_dict, HAND_IDENTIFIERS
+
+
+model = torch.load("spoter-checkpoint.pth", map_location=torch.device('cpu'))
+model.train(False)
+
+HAND_IDENTIFIERS = [id + "_Left" for id in HAND_IDENTIFIERS] + [id + "_Right" for id in HAND_IDENTIFIERS]
+GLOSS = ['book', 'drink', 'computer', 'before', 'chair', 'go', 'clothes', 'who', 'candy', 'cousin', 'deaf', 'fine',
+         'help', 'no', 'thin', 'walk', 'year', 'yes', 'all', 'black', 'cool', 'finish', 'hot', 'like', 'many', 'mother',
+         'now', 'orange', 'table', 'thanksgiving', 'what', 'woman', 'bed', 'blue', 'bowling', 'can', 'dog', 'family',
+         'fish', 'graduate', 'hat', 'hearing', 'kiss', 'language', 'later', 'man', 'shirt', 'study', 'tall', 'white',
+         'wrong', 'accident', 'apple', 'bird', 'change', 'color', 'corn', 'cow', 'dance', 'dark', 'doctor', 'eat',
+         'enjoy', 'forget', 'give', 'last', 'meet', 'pink', 'pizza', 'play', 'school', 'secretary', 'short', 'time',
+         'want', 'work', 'africa', 'basketball', 'birthday', 'brown', 'but', 'cheat', 'city', 'cook', 'decide', 'full',
+         'how', 'jacket', 'letter', 'medicine', 'need', 'paint', 'paper', 'pull', 'purple', 'right', 'same', 'son',
+         'tell', 'thursday']
+
+device = torch.device("cpu")
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+
+
+def tensor_to_dictionary(landmarks_tensor: torch.Tensor) -> dict:
+
+    data_array = landmarks_tensor.numpy()
+    output = {}
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+        output[identifier] = data_array[:, landmark_index]
+
+    return output
+
+
+def dictionary_to_tensor(landmarks_dict: dict) -> torch.Tensor:
+
+    output = np.empty(shape=(len(landmarks_dict["leftEar"]), len(BODY_IDENTIFIERS + HAND_IDENTIFIERS), 2))
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+        output[:, landmark_index, 0] = [frame[0] for frame in landmarks_dict[identifier]]
+        output[:, landmark_index, 1] = [frame[1] for frame in landmarks_dict[identifier]]
+
+    return torch.from_numpy(output)
+
+
+def greet(video):
+
+    data = obtain_pose_data(video)
+
+    depth_map = np.empty(shape=(len(data.data_hub["nose_X"]), len(BODY_IDENTIFIERS + HAND_IDENTIFIERS), 2))
+
+    for index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+        depth_map[:, index, 0] = data.data_hub[identifier + "_X"]
+        depth_map[:, index, 1] = data.data_hub[identifier + "_Y"]
+
+    depth_map = torch.from_numpy(np.copy(depth_map))
+    print(depth_map.shape)
+
+    depth_map = tensor_to_dictionary(depth_map)
+
+    keys = copy.copy(list(depth_map.keys()))
+    for key in keys:
+        data = depth_map[key]
+        del depth_map[key]
+        depth_map[key.replace("_Left", "_0").replace("_Right", "_1")] = data
+
+    depth_map = normalize_single_body_dict(depth_map)
+    depth_map = normalize_single_hand_dict(depth_map)
+
+    keys = copy.copy(list(depth_map.keys()))
+    for key in keys:
+        data = depth_map[key]
+        del depth_map[key]
+        depth_map[key.replace("_0", "_Left").replace("_1", "_Right")] = data
+
+    depth_map = dictionary_to_tensor(depth_map)
+
+    depth_map = depth_map - 0.5
+
+    inputs = depth_map.squeeze(0).to(device)
+    outputs = model(inputs).expand(1, -1, -1)
+    results = torch.nn.functional.softmax(outputs, dim=2).detach().numpy()[0, 0]
+
+    results = {GLOSS[i]: float(results[i]) for i in range(100)}
+    print(outputs.shape)
+    print(results)
+
+    return results
+
+
+label = gr.outputs.Label(num_top_classes=5)
+demo = gr.Interface(fn=greet, inputs="playable_video", outputs=label)
+
+demo.launch()

spoter-checkpoint.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4f0927fbaddf11da6762ca76474a7bbc049565599e3fc6f081caa5cc00fb53a
+size 23764668

spoter/gaussian_noise.py

@@ -0,0 +1,18 @@
+
+import torch
+
+
+class GaussianNoise(object):
+    def __init__(self, mean=0., std=1.):
+        self.std = std
+        self.mean = mean
+
+    def __call__(self, tensor):
+        return tensor + torch.randn(tensor.size()) * self.std + self.mean
+
+    def __repr__(self):
+        return self.__class__.__name__ + '(mean={0}, std={1})'.format(self.mean, self.std)
+
+
+if __name__ == "__main__":
+    pass

spoter/spoter_model.py

@@ -0,0 +1,70 @@
+
+import copy
+import torch
+
+import torch.nn as nn
+from typing import Optional
+
+
+def _get_clones(mod, n):
+    return nn.ModuleList([copy.deepcopy(mod) for _ in range(n)])
+
+
+class SPOTERTransformerDecoderLayer(nn.TransformerDecoderLayer):
+    """
+    Edited TransformerDecoderLayer implementation omitting the redundant self-attention operation as opposed to the
+    standard implementation.
+    """
+
+    def __init__(self, d_model, nhead, dim_feedforward, dropout, activation):
+        super(SPOTERTransformerDecoderLayer, self).__init__(d_model, nhead, dim_feedforward, dropout, activation)
+
+        del self.self_attn
+
+    def forward(self, tgt: torch.Tensor, memory: torch.Tensor, tgt_mask: Optional[torch.Tensor] = None,
+                memory_mask: Optional[torch.Tensor] = None, tgt_key_padding_mask: Optional[torch.Tensor] = None,
+                memory_key_padding_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+
+        tgt = tgt + self.dropout1(tgt)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(tgt, memory, memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+
+        return tgt
+
+
+class SPOTER(nn.Module):
+    """
+    Implementation of the SPOTER (Sign POse-based TransformER) architecture for sign language recognition from sequence
+    of skeletal data.
+    """
+
+    def __init__(self, num_classes, hidden_dim=55):
+        super().__init__()
+
+        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim))
+        self.pos = nn.Parameter(torch.cat([self.row_embed[0].unsqueeze(0).repeat(1, 1, 1)], dim=-1).flatten(0, 1).unsqueeze(0))
+        self.class_query = nn.Parameter(torch.rand(1, hidden_dim))
+        self.transformer = nn.Transformer(hidden_dim, 9, 6, 6)
+        self.linear_class = nn.Linear(hidden_dim, num_classes)
+
+        # Deactivate the initial attention decoder mechanism
+        custom_decoder_layer = SPOTERTransformerDecoderLayer(self.transformer.d_model, self.transformer.nhead, 2048,
+                                                             0.1, "relu")
+        self.transformer.decoder.layers = _get_clones(custom_decoder_layer, self.transformer.decoder.num_layers)
+
+    def forward(self, inputs):
+        h = torch.unsqueeze(inputs.flatten(start_dim=1), 1).float()
+        h = self.transformer(self.pos + h, self.class_query.unsqueeze(0)).transpose(0, 1)
+        res = self.linear_class(h)
+
+        return res
+
+
+if __name__ == "__main__":
+    pass

spoter/utils.py

@@ -0,0 +1,81 @@
+
+import logging
+import torch
+
+
+def train_epoch(model, dataloader, criterion, optimizer, device, scheduler=None):
+
+    pred_correct, pred_all = 0, 0
+    running_loss = 0.0
+
+    for i, data in enumerate(dataloader):
+        inputs, labels = data
+        inputs = inputs.squeeze(0).to(device)
+        labels = labels.to(device, dtype=torch.long)
+
+        optimizer.zero_grad()
+        outputs = model(inputs).expand(1, -1, -1)
+
+        loss = criterion(outputs[0], labels[0])
+        loss.backward()
+        optimizer.step()
+        running_loss += loss
+
+        # Statistics
+        if int(torch.argmax(torch.nn.functional.softmax(outputs, dim=2))) == int(labels[0][0]):
+            pred_correct += 1
+        pred_all += 1
+
+    if scheduler:
+        scheduler.step(running_loss.item() / len(dataloader))
+
+    return running_loss, pred_correct, pred_all, (pred_correct / pred_all)
+
+
+def evaluate(model, dataloader, device, print_stats=False):
+
+    pred_correct, pred_all = 0, 0
+    stats = {i: [0, 0] for i in range(101)}
+
+    for i, data in enumerate(dataloader):
+        inputs, labels = data
+        inputs = inputs.squeeze(0).to(device)
+        labels = labels.to(device, dtype=torch.long)
+
+        outputs = model(inputs).expand(1, -1, -1)
+
+        # Statistics
+        if int(torch.argmax(torch.nn.functional.softmax(outputs, dim=2))) == int(labels[0][0]):
+            stats[int(labels[0][0])][0] += 1
+            pred_correct += 1
+
+        stats[int(labels[0][0])][1] += 1
+        pred_all += 1
+
+    if print_stats:
+        stats = {key: value[0] / value[1] for key, value in stats.items() if value[1] != 0}
+        print("Label accuracies statistics:")
+        print(str(stats) + "\n")
+        logging.info("Label accuracies statistics:")
+        logging.info(str(stats) + "\n")
+
+    return pred_correct, pred_all, (pred_correct / pred_all)
+
+
+def evaluate_top_k(model, dataloader, device, k=5):
+
+    pred_correct, pred_all = 0, 0
+
+    for i, data in enumerate(dataloader):
+        inputs, labels = data
+        inputs = inputs.squeeze(0).to(device)
+        labels = labels.to(device, dtype=torch.long)
+
+        outputs = model(inputs).expand(1, -1, -1)
+
+        if int(labels[0][0]) in torch.topk(outputs, k).indices.tolist():
+            pred_correct += 1
+
+        pred_all += 1
+
+    return pred_correct, pred_all, (pred_correct / pred_all)

spoter_mod/.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

spoter_mod/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,24 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="11">
+            <item index="0" class="java.lang.String" itemvalue="pandas" />
+            <item index="1" class="java.lang.String" itemvalue="feedparser" />
+            <item index="2" class="java.lang.String" itemvalue="sklearn" />
+            <item index="3" class="java.lang.String" itemvalue="numpy" />
+            <item index="4" class="java.lang.String" itemvalue="coremltools" />
+            <item index="5" class="java.lang.String" itemvalue="h5py" />
+            <item index="6" class="java.lang.String" itemvalue="torch" />
+            <item index="7" class="java.lang.String" itemvalue="einops" />
+            <item index="8" class="java.lang.String" itemvalue="firebase-admin" />
+            <item index="9" class="java.lang.String" itemvalue="pyemd" />
+            <item index="10" class="java.lang.String" itemvalue="matplotlib" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

spoter_mod/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

spoter_mod/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (spoter)" project-jdk-type="Python SDK" />
+</project>

spoter_mod/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/spoter.iml" filepath="$PROJECT_DIR$/.idea/spoter.iml" />
+    </modules>
+  </component>
+</project>

spoter_mod/.idea/spoter.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.7 (spoter)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

spoter_mod/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

spoter_mod/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2021-2022 Matyáš Boháček
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

spoter_mod/README.md

@@ -0,0 +1,61 @@
+![Alt Text](http://spoter.signlanguagerecognition.com/img/GitHub_banner.png)
+
+> by **[Matyáš Boháček](https://github.com/matyasbohacek)** and **[Marek Hrúz](https://github.com/mhruz)**, University of West Bohemia <br>
+> Should you have any questions or inquiries, feel free to contact us [here](mailto:matyas.bohacek@matsworld.io).
+
+[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/sign-pose-based-transformer-for-word-level/sign-language-recognition-on-lsa64)](https://paperswithcode.com/sota/sign-language-recognition-on-lsa64?p=sign-pose-based-transformer-for-word-level)
+
+Repository accompanying the [Sign Pose-based Transformer for Word-level Sign Language Recognition](https://openaccess.thecvf.com/content/WACV2022W/HADCV/html/Bohacek_Sign_Pose-Based_Transformer_for_Word-Level_Sign_Language_Recognition_WACVW_2022_paper.html) paper, where we present a novel architecture for word-level sign language recognition based on the Transformer model. We designed our solution with low computational cost in mind, since we see egreat potential in the usage of such recognition system on hand-held devices. We introduce multiple original augmentation techniques tailored for the task of sign language recognition and propose a unique normalization scheme based on sign language linguistics.
+
+![Alt Text](http://spoter.signlanguagerecognition.com/img/architecture_github.gif)
+
+## Get Started
+
+First, make sure to install all necessary dependencies using:
+
+```shell
+pip install -r requirements.txt
+```
+
+To train the model, simply specify the hyperparameters and run the following:
+
+```
+python -m train
+  --experiment_name [str; name of the experiment to name the output logs and plots]
+  
+  --epochs [int; number of epochs]
+  --lr [float; learning rate]
+  
+  --training_set_path [str; path to the csv file with training set's skeletal data]
+  --validation_set_path [str; path to the csv file with validation set's skeletal data]
+  --testing_set_path [str; path to the csv file with testing set's skeletal data]
+```
+
+If either the validation or testing sets' paths are left empty, these corresponding metrics will not be calculated. We also provide out-of-the box parameter to split the validation set as a desired split of the training set while preserving the label distribution for datasets without author-specified splits. These and many other specific hyperparameters with their descriptions can be found in the [train.py](https://github.com/matyasbohacek/spoter/blob/main/train.py) file. All of them are provided a default value we found to be working well in our experiments.
+
+## Data
+
+As SPOTER works on top of sequences of signers' skeletal data extracted from videos, we wanted to eliminate the computational demands of such annotation for each training run by pre-collecting this. For this reason and reproducibility, we are open-sourcing this data for WLASL100 and LSA64 datasets along with the repository. You can find the data [here](https://github.com/matyasbohacek/spoter/releases/tag/supplementary-data).
+
+![Alt Text](http://spoter.signlanguagerecognition.com/img/datasets_overview.gif)
+
+## License
+
+The **code** is published under the [Apache License 2.0](https://github.com/matyasbohacek/spoter/blob/main/LICENSE) which allows for both academic and commercial use if  relevant License and copyright notice is included, our work is cited and all changes are stated.
+
+The accompanying skeletal data of the [WLASL](https://arxiv.org/pdf/1910.11006.pdf) and [LSA64](https://core.ac.uk/download/pdf/76495887.pdf) datasets used for experiments are, however, shared under the [Attribution-NonCommercial 4.0 International (CC BY-NC 4.0)](https://creativecommons.org/licenses/by-nc/4.0/) license allowing only for non-commercial usage.
+
+## Citation
+
+If you find our work relevant, build upon it or compare your approaches with it, please cite our work as stated below:
+
+```
+@InProceedings{Bohacek_2022_WACV,
+    author    = {Boh\'a\v{c}ek, Maty\'a\v{s} and Hr\'uz, Marek},
+    title     = {Sign Pose-Based Transformer for Word-Level Sign Language Recognition},
+    booktitle = {Proceedings of the IEEE/CVF Winter Conference on Applications of Computer Vision (WACV) Workshops},
+    month     = {January},
+    year      = {2022},
+    pages     = {182-191}
+}
+```

spoter_mod/augmentations/__init__.py

@@ -0,0 +1,231 @@
+
+import math
+import logging
+import cv2
+import random
+
+import numpy as np
+
+from normalization.body_normalization import BODY_IDENTIFIERS
+from normalization.hand_normalization import HAND_IDENTIFIERS
+
+
+HAND_IDENTIFIERS = [id + "_0" for id in HAND_IDENTIFIERS] + [id + "_1" for id in HAND_IDENTIFIERS]
+ARM_IDENTIFIERS_ORDER = ["neck", "$side$Shoulder", "$side$Elbow", "$side$Wrist"]
+
+
+def __random_pass(prob):
+    return random.random() < prob
+
+
+def __numpy_to_dictionary(data_array: np.ndarray) -> dict:
+    """
+    Supplementary method converting a NumPy array of body landmark data into dictionaries. The array data must match the
+    order of the BODY_IDENTIFIERS list.
+    """
+
+    output = {}
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS):
+        output[identifier] = data_array[:, landmark_index].tolist()
+
+    return output
+
+
+def __dictionary_to_numpy(landmarks_dict: dict) -> np.ndarray:
+    """
+    Supplementary method converting dictionaries of body landmark data into respective NumPy arrays. The resulting array
+    will match the order of the BODY_IDENTIFIERS list.
+    """
+
+    output = np.empty(shape=(len(landmarks_dict["leftEar"]), len(BODY_IDENTIFIERS), 2))
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS):
+        output[:, landmark_index, 0] = np.array(landmarks_dict[identifier])[:, 0]
+        output[:, landmark_index, 1] = np.array(landmarks_dict[identifier])[:, 1]
+
+    return output
+
+
+def __rotate(origin: tuple, point: tuple, angle: float):
+    """
+    Rotates a point counterclockwise by a given angle around a given origin.
+
+    :param origin: Landmark in the (X, Y) format of the origin from which to count angle of rotation
+    :param point: Landmark in the (X, Y) format to be rotated
+    :param angle: Angle under which the point shall be rotated
+    :return: New landmarks (coordinates)
+    """
+
+    ox, oy = origin
+    px, py = point
+
+    qx = ox + math.cos(angle) * (px - ox) - math.sin(angle) * (py - oy)
+    qy = oy + math.sin(angle) * (px - ox) + math.cos(angle) * (py - oy)
+
+    return qx, qy
+
+
+def __preprocess_row_sign(sign: dict) -> (dict, dict):
+    """
+    Supplementary method splitting the single-dictionary skeletal data into two dictionaries of body and hand landmarks
+    respectively.
+    """
+
+    sign_eval = sign
+
+    if "nose_X" in sign_eval:
+        body_landmarks = {identifier: [(x, y) for x, y in zip(sign_eval[identifier + "_X"], sign_eval[identifier + "_Y"])]
+                          for identifier in BODY_IDENTIFIERS}
+        hand_landmarks = {identifier: [(x, y) for x, y in zip(sign_eval[identifier + "_X"], sign_eval[identifier + "_Y"])]
+                          for identifier in HAND_IDENTIFIERS}
+
+    else:
+        body_landmarks = {identifier: sign_eval[identifier] for identifier in BODY_IDENTIFIERS}
+        hand_landmarks = {identifier: sign_eval[identifier] for identifier in HAND_IDENTIFIERS}
+
+    return body_landmarks, hand_landmarks
+
+
+def __wrap_sign_into_row(body_identifiers: dict, hand_identifiers: dict) -> dict:
+    """
+    Supplementary method for merging body and hand data into a single dictionary.
+    """
+
+    return {**body_identifiers, **hand_identifiers}
+
+
+def augment_rotate(sign: dict, angle_range: tuple) -> dict:
+    """
+    AUGMENTATION TECHNIQUE. All the joint coordinates in each frame are rotated by a random angle up to 13 degrees with
+    the center of rotation lying in the center of the frame, which is equal to [0.5; 0.5].
+
+    :param sign: Dictionary with sequential skeletal data of the signing person
+    :param angle_range: Tuple containing the angle range (minimal and maximal angle in degrees) to randomly choose the
+                        angle by which the landmarks will be rotated from
+
+    :return: Dictionary with augmented (by rotation) sequential skeletal data of the signing person
+    """
+
+    body_landmarks, hand_landmarks = __preprocess_row_sign(sign)
+    angle = math.radians(random.uniform(*angle_range))
+
+    body_landmarks = {key: [__rotate((0.5, 0.5), frame, angle) for frame in value] for key, value in
+                      body_landmarks.items()}
+    hand_landmarks = {key: [__rotate((0.5, 0.5), frame, angle) for frame in value] for key, value in
+                      hand_landmarks.items()}
+
+    return __wrap_sign_into_row(body_landmarks, hand_landmarks)
+
+
+def augment_shear(sign: dict, type: str, squeeze_ratio: tuple) -> dict:
+    """
+    AUGMENTATION TECHNIQUE.
+
+        - Squeeze. All the frames are squeezed from both horizontal sides. Two different random proportions up to 15% of
+        the original frame's width for both left and right side are cut.
+
+        - Perspective transformation. The joint coordinates are projected onto a new plane with a spatially defined
+        center of projection, which simulates recording the sign video with a slight tilt. Each time, the right or left
+        side, as well as the proportion by which both the width and height will be reduced, are chosen randomly. This
+        proportion is selected from a uniform distribution on the [0; 1) interval. Subsequently, the new plane is
+        delineated by reducing the width at the desired side and the respective vertical edge (height) at both of its
+        adjacent corners.
+
+    :param sign: Dictionary with sequential skeletal data of the signing person
+    :param type: Type of shear augmentation to perform (either 'squeeze' or 'perspective')
+    :param squeeze_ratio: Tuple containing the relative range from what the proportion of the original width will be
+                          randomly chosen. These proportions will either be cut from both sides or used to construct the
+                          new projection
+
+    :return: Dictionary with augmented (by squeezing or perspective transformation) sequential skeletal data of the
+             signing person
+    """
+
+    body_landmarks, hand_landmarks = __preprocess_row_sign(sign)
+
+    if type == "squeeze":
+        move_left = random.uniform(*squeeze_ratio)
+        move_right = random.uniform(*squeeze_ratio)
+
+        src = np.array(((0, 1), (1, 1), (0, 0), (1, 0)), dtype=np.float32)
+        dest = np.array(((0 + move_left, 1), (1 - move_right, 1), (0 + move_left, 0), (1 - move_right, 0)),
+                        dtype=np.float32)
+        mtx = cv2.getPerspectiveTransform(src, dest)
+
+    elif type == "perspective":
+
+        move_ratio = random.uniform(*squeeze_ratio)
+        src = np.array(((0, 1), (1, 1), (0, 0), (1, 0)), dtype=np.float32)
+
+        if __random_pass(0.5):
+            dest = np.array(((0 + move_ratio, 1 - move_ratio), (1, 1), (0 + move_ratio, 0 + move_ratio), (1, 0)),
+                            dtype=np.float32)
+        else:
+            dest = np.array(((0, 1), (1 - move_ratio, 1 - move_ratio), (0, 0), (1 - move_ratio, 0 + move_ratio)),
+                            dtype=np.float32)
+
+        mtx = cv2.getPerspectiveTransform(src, dest)
+
+    else:
+
+        logging.error("Unsupported shear type provided.")
+        return {}
+
+    landmarks_array = __dictionary_to_numpy(body_landmarks)
+    augmented_landmarks = cv2.perspectiveTransform(np.array(landmarks_array, dtype=np.float32), mtx)
+
+    augmented_zero_landmark = cv2.perspectiveTransform(np.array([[[0, 0]]], dtype=np.float32), mtx)[0][0]
+    augmented_landmarks = np.stack([np.where(sub == augmented_zero_landmark, [0, 0], sub) for sub in augmented_landmarks])
+
+    body_landmarks = __numpy_to_dictionary(augmented_landmarks)
+
+    return __wrap_sign_into_row(body_landmarks, hand_landmarks)
+
+
+def augment_arm_joint_rotate(sign: dict, probability: float, angle_range: tuple) -> dict:
+    """
+    AUGMENTATION TECHNIQUE. The joint coordinates of both arms are passed successively, and the impending landmark is
+    slightly rotated with respect to the current one. The chance of each joint to be rotated is 3:10 and the angle of
+    alternation is a uniform random angle up to +-4 degrees. This simulates slight, negligible variances in each
+    execution of a sign, which do not change its semantic meaning.
+
+    :param sign: Dictionary with sequential skeletal data of the signing person
+    :param probability: Probability of each joint to be rotated (float from the range [0, 1])
+    :param angle_range: Tuple containing the angle range (minimal and maximal angle in degrees) to randomly choose the
+                        angle by which the landmarks will be rotated from
+
+    :return: Dictionary with augmented (by arm joint rotation) sequential skeletal data of the signing person
+    """
+
+    body_landmarks, hand_landmarks = __preprocess_row_sign(sign)
+
+    # Iterate over both directions (both hands)
+    for side in ["left", "right"]:
+        # Iterate gradually over the landmarks on arm
+        for landmark_index, landmark_origin in enumerate(ARM_IDENTIFIERS_ORDER):
+            landmark_origin = landmark_origin.replace("$side$", side)
+
+            # End the process on the current hand if the landmark is not present
+            if landmark_origin not in body_landmarks:
+                break
+
+            # Perform rotation by provided probability
+            if __random_pass(probability):
+                angle = math.radians(random.uniform(*angle_range))
+
+                for to_be_rotated in ARM_IDENTIFIERS_ORDER[landmark_index + 1:]:
+                    to_be_rotated = to_be_rotated.replace("$side$", side)
+
+                    # Skip if the landmark is not present
+                    if to_be_rotated not in body_landmarks:
+                        continue
+
+                    body_landmarks[to_be_rotated] = [__rotate(body_landmarks[landmark_origin][frame_index], frame,
+                        angle) for frame_index, frame in enumerate(body_landmarks[to_be_rotated])]
+
+    return __wrap_sign_into_row(body_landmarks, hand_landmarks)
+
+
+if __name__ == "__main__":
+    pass

spoter_mod/data_structurization/autsl.py

@@ -0,0 +1,22 @@
+
+import os
+import tqdm
+
+import pandas as pd
+from shutil import copyfile
+
+
+MAIN_PATH = "/Users/matyasbohacek/Documents/Academics/Projects/AUTSL"
+BATCH = "test"
+
+df = pd.read_csv(MAIN_PATH + "/" + BATCH + "_labels.csv", encoding="utf-8", sep=";")
+
+if not os.path.exists(MAIN_PATH + "/" + BATCH + "_preprocessed/"):
+    os.mkdir(MAIN_PATH + "/" + BATCH + "_preprocessed/")
+
+for index_row, row in tqdm.tqdm(df.iterrows()):
+    if not os.path.exists(MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(row["label"]) + "/"):
+        os.mkdir(MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(row["label"]) + "/")
+
+    copyfile(MAIN_PATH + "/" + BATCH + "/" + str(row["video"]) + "_color.mp4", MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(row["label"]) + "/" + str(row["video"]) + "_color.mp4")
+

spoter_mod/data_structurization/wlasl.py

@@ -0,0 +1,32 @@
+
+import os
+import json
+import tqdm
+
+from shutil import copyfile
+
+
+MAIN_PATH = "/Users/matyasbohacek/Documents/Academics/Projects/WLASL/start_kit"
+BATCH = "train"
+
+if not os.path.exists(MAIN_PATH + "/" + BATCH + "_preprocessed/"):
+    os.mkdir(MAIN_PATH + "/" + BATCH + "_preprocessed/")
+
+with open(MAIN_PATH + "/specs.json") as f:
+  data = json.load(f)
+
+for item_index, item in tqdm.tqdm(enumerate(data)):
+
+    for video in item["instances"]:
+
+        if video["split"] != BATCH:
+            continue
+
+        if not os.path.exists(MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(item_index) + "/"):
+            os.mkdir(MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(item_index) + "/")
+
+        original_path = MAIN_PATH + "/videos/" + str(video["video_id"]) + ".mp4"
+        new_path = MAIN_PATH + "/" + BATCH + "_preprocessed/" + str(item_index) + "/" + str(video["video_id"]) + ".mp4"
+
+        copyfile(original_path, new_path)
+

spoter_mod/datasets/czech_slr_dataset.py

@@ -0,0 +1,153 @@
+import ast
+import torch
+
+import pandas as pd
+import torch.utils.data as torch_data
+
+from random import randrange
+from augmentations import *
+from normalization.body_normalization import BODY_IDENTIFIERS
+from normalization.hand_normalization import HAND_IDENTIFIERS
+from normalization.body_normalization import normalize_single_dict as normalize_single_body_dict
+from normalization.hand_normalization import normalize_single_dict as normalize_single_hand_dict
+
+HAND_IDENTIFIERS = [id + "_0" for id in HAND_IDENTIFIERS] + [id + "_1" for id in HAND_IDENTIFIERS]
+
+DEFAULT_AUGMENTATIONS_CONFIG = {
+    "rotate-angle": 13,
+    "perspective-transform-ratio": 0.1,
+    "squeeze-ratio": 0.15,
+    "arm-joint-rotate-angle": 4,
+    "arm-joint-rotate-probability": 0.3
+}
+
+
+def load_dataset(file_location: str):
+
+    # Load the datset csv file
+    df = pd.read_csv(file_location, encoding="utf-8")
+
+    # TO BE DELETED
+    df.columns = [item.replace("_Left_", "_0_").replace("_Right_", "_1_") for item in list(df.columns)]
+    if "neck_X" not in df.columns:
+        df["neck_X"] = [0 for _ in range(df.shape[0])]
+        df["neck_Y"] = [0 for _ in range(df.shape[0])]
+
+    # TEMP
+    labels = df["labels"].to_list()
+    labels = [label + 1 for label in df["labels"].to_list()]
+    data = []
+
+    for row_index, row in df.iterrows():
+        current_row = np.empty(shape=(len(ast.literal_eval(row["leftEar_X"])), len(BODY_IDENTIFIERS + HAND_IDENTIFIERS), 2))
+        for index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+            current_row[:, index, 0] = ast.literal_eval(row[identifier + "_X"])
+            current_row[:, index, 1] = ast.literal_eval(row[identifier + "_Y"])
+
+        data.append(current_row)
+
+    return data, labels
+
+
+def tensor_to_dictionary(landmarks_tensor: torch.Tensor) -> dict:
+
+    data_array = landmarks_tensor.numpy()
+    output = {}
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+        output[identifier] = data_array[:, landmark_index]
+
+    return output
+
+
+def dictionary_to_tensor(landmarks_dict: dict) -> torch.Tensor:
+
+    output = np.empty(shape=(len(landmarks_dict["leftEar"]), len(BODY_IDENTIFIERS + HAND_IDENTIFIERS), 2))
+
+    for landmark_index, identifier in enumerate(BODY_IDENTIFIERS + HAND_IDENTIFIERS):
+        output[:, landmark_index, 0] = [frame[0] for frame in landmarks_dict[identifier]]
+        output[:, landmark_index, 1] = [frame[1] for frame in landmarks_dict[identifier]]
+
+    return torch.from_numpy(output)
+
+
+class CzechSLRDataset(torch_data.Dataset):
+    """Advanced object representation of the HPOES dataset for loading hand joints landmarks utilizing the Torch's
+    built-in Dataset properties"""
+
+    data: [np.ndarray]
+    labels: [np.ndarray]
+
+    def __init__(self, dataset_filename: str, num_labels=5, transform=None, augmentations=False,
+                 augmentations_prob=0.5, normalize=True, augmentations_config: dict = DEFAULT_AUGMENTATIONS_CONFIG):
+        """
+        Initiates the HPOESDataset with the pre-loaded data from the h5 file.
+
+        :param dataset_filename: Path to the h5 file
+        :param transform: Any data transformation to be applied (default: None)
+        """
+
+        loaded_data = load_dataset(dataset_filename)
+        data, labels = loaded_data[0], loaded_data[1]
+
+        self.data = data
+        self.labels = labels
+        self.targets = list(labels)
+        self.num_labels = num_labels
+        self.transform = transform
+
+        self.augmentations = augmentations
+        self.augmentations_prob = augmentations_prob
+        self.augmentations_config = augmentations_config
+        self.normalize = normalize
+
+    def __getitem__(self, idx):
+        """
+        Allocates, potentially transforms and returns the item at the desired index.
+
+        :param idx: Index of the item
+        :return: Tuple containing both the depth map and the label
+        """
+
+        depth_map = torch.from_numpy(np.copy(self.data[idx]))
+        label = torch.Tensor([self.labels[idx] - 1])
+
+        depth_map = tensor_to_dictionary(depth_map)
+
+        # Apply potential augmentations
+        if self.augmentations and random.random() < self.augmentations_prob:
+
+            selected_aug = randrange(4)
+
+            if selected_aug == 0:
+                depth_map = augment_rotate(depth_map, (-self.augmentations_config["rotate-angle"], self.augmentations_config["rotate-angle"]))
+
+            if selected_aug == 1:
+                depth_map = augment_shear(depth_map, "perspective", (0, self.augmentations_config["perspective-transform-ratio"]))
+
+            if selected_aug == 2:
+                depth_map = augment_shear(depth_map, "squeeze", (0, self.augmentations_config["squeeze-ratio"]))
+
+            if selected_aug == 3:
+                depth_map = augment_arm_joint_rotate(depth_map, self.augmentations_config["arm-joint-rotate-probability"], (-self.augmentations_config["arm-joint-rotate-angle"], self.augmentations_config["arm-joint-rotate-angle"]))
+
+        if self.normalize:
+            depth_map = normalize_single_body_dict(depth_map)
+            depth_map = normalize_single_hand_dict(depth_map)
+
+        depth_map = dictionary_to_tensor(depth_map)
+
+        # Move the landmark position interval to improve performance
+        depth_map = depth_map - 0.5
+
+        if self.transform:
+            depth_map = self.transform(depth_map)
+
+        return depth_map, label
+
+    def __len__(self):
+        return len(self.labels)
+
+
+if __name__ == "__main__":
+    pass

spoter_mod/normalization/body_normalization.py

@@ -0,0 +1,226 @@
+
+import logging
+import pandas as pd
+
+BODY_IDENTIFIERS = [
+    "nose",
+    "neck",
+    "rightEye",
+    "leftEye",
+    "rightEar",
+    "leftEar",
+    "rightShoulder",
+    "leftShoulder",
+    "rightElbow",
+    "leftElbow",
+    "rightWrist",
+    "leftWrist"
+]
+
+
+def normalize_body_full(df: pd.DataFrame) -> (pd.DataFrame, list):
+    """
+    Normalizes the body position data using the Bohacek-normalization algorithm.
+
+    :param df: pd.DataFrame to be normalized
+    :return: pd.DataFrame with normalized values for body pose
+    """
+
+    # TODO: Fix division by zero
+
+    normalized_df = pd.DataFrame(columns=df.columns)
+    invalid_row_indexes = []
+    body_landmarks = {"X": [], "Y": []}
+
+    # Construct the relevant identifiers
+    for identifier in BODY_IDENTIFIERS:
+        body_landmarks["X"].append(identifier + "_X")
+        body_landmarks["Y"].append(identifier + "_Y")
+
+    # Iterate over all of the records in the dataset
+    for index, row in df.iterrows():
+
+        sequence_size = len(row["leftEar_Y"])
+        valid_sequence = True
+        original_row = row
+
+        last_starting_point, last_ending_point = None, None
+
+        # Treat each element of the sequence (analyzed frame) individually
+        for sequence_index in range(sequence_size):
+
+            # Prevent from even starting the analysis if some necessary elements are not present
+            if (row["leftShoulder_X"][sequence_index] == 0 or row["rightShoulder_X"][sequence_index] == 0) and (row["neck_X"][sequence_index] == 0 or row["nose_X"][sequence_index] == 0):
+                if not last_starting_point:
+                    valid_sequence = False
+                    continue
+
+                else:
+                    starting_point, ending_point = last_starting_point, last_ending_point
+
+            else:
+
+                # NOTE:
+                #
+                # While in the paper, it is written that the head metric is calculated by halving the shoulder distance,
+                # this is meant for the distance between the very ends of one's shoulder, as literature studying body
+                # metrics and ratios generally states. The Vision Pose Estimation API, however, seems to be predicting
+                # rather the center of one's shoulder. Based on our experiments and manual reviews of the data, employing
+                # this as just the plain shoulder distance seems to be more corresponding to the desired metric.
+                #
+                # Please, review this if using other third-party pose estimation libraries.
+
+                if row["leftShoulder_X"][sequence_index] != 0 and row["rightShoulder_X"][sequence_index] != 0:
+                    left_shoulder = (row["leftShoulder_X"][sequence_index], row["leftShoulder_Y"][sequence_index])
+                    right_shoulder = (row["rightShoulder_X"][sequence_index], row["rightShoulder_Y"][sequence_index])
+                    shoulder_distance = ((((left_shoulder[0] - right_shoulder[0]) ** 2) + (
+                                (left_shoulder[1] - right_shoulder[1]) ** 2)) ** 0.5)
+                    head_metric = shoulder_distance
+                else:
+                    neck = (row["neck_X"][sequence_index], row["neck_Y"][sequence_index])
+                    nose = (row["nose_X"][sequence_index], row["nose_Y"][sequence_index])
+                    neck_nose_distance = ((((neck[0] - nose[0]) ** 2) + ((neck[1] - nose[1]) ** 2)) ** 0.5)
+                    head_metric = neck_nose_distance
+
+                # Set the starting and ending point of the normalization bounding box
+                starting_point = [row["neck_X"][sequence_index] - 3 * head_metric, row["leftEye_Y"][sequence_index] + (head_metric / 2)]
+                ending_point = [row["neck_X"][sequence_index] + 3 * head_metric, starting_point[1] - 6 * head_metric]
+
+                last_starting_point, last_ending_point = starting_point, ending_point
+
+            # Ensure that all of the bounding-box-defining coordinates are not out of the picture
+            if starting_point[0] < 0: starting_point[0] = 0
+            if starting_point[1] < 0: starting_point[1] = 0
+            if ending_point[0] < 0: ending_point[0] = 0
+            if ending_point[1] < 0: ending_point[1] = 0
+
+            # Normalize individual landmarks and save the results
+            for identifier in BODY_IDENTIFIERS:
+                key = identifier + "_"
+
+                # Prevent from trying to normalize incorrectly captured points
+                if row[key + "X"][sequence_index] == 0:
+                    continue
+
+                normalized_x = (row[key + "X"][sequence_index] - starting_point[0]) / (ending_point[0] -
+                                                                                       starting_point[0])
+                normalized_y = (row[key + "Y"][sequence_index] - ending_point[1]) / (starting_point[1] -
+                                                                                       ending_point[1])
+
+                row[key + "X"][sequence_index] = normalized_x
+                row[key + "Y"][sequence_index] = normalized_y
+
+        if valid_sequence:
+            normalized_df = normalized_df.append(row, ignore_index=True)
+        else:
+            logging.warning(" BODY LANDMARKS: One video instance could not be normalized.")
+            normalized_df = normalized_df.append(original_row, ignore_index=True)
+            invalid_row_indexes.append(index)
+
+    print("The normalization of body is finished.")
+    print("\t-> Original size:", df.shape[0])
+    print("\t-> Normalized size:", normalized_df.shape[0])
+    print("\t-> Problematic videos:", len(invalid_row_indexes))
+
+    return normalized_df, invalid_row_indexes
+
+
+def normalize_single_dict(row: dict):
+    """
+    Normalizes the skeletal data for a given sequence of frames with signer's body pose data. The normalization follows
+    the definition from our paper.
+
+    :param row: Dictionary containing key-value pairs with joint identifiers and corresponding lists (sequences) of
+                that particular joints coordinates
+    :return: Dictionary with normalized skeletal data (following the same schema as input data)
+    """
+
+    sequence_size = len(row["leftEar"])
+    valid_sequence = True
+    original_row = row
+
+    last_starting_point, last_ending_point = None, None
+
+    # Treat each element of the sequence (analyzed frame) individually
+    for sequence_index in range(sequence_size):
+
+        # Prevent from even starting the analysis if some necessary elements are not present
+        if (row["leftShoulder"][sequence_index][0] == 0 or row["rightShoulder"][sequence_index][0] == 0) and (
+                row["neck"][sequence_index][0] == 0 or row["nose"][sequence_index][0] == 0):
+            if not last_starting_point:
+                valid_sequence = False
+                continue
+
+            else:
+                starting_point, ending_point = last_starting_point, last_ending_point
+
+        else:
+
+            # NOTE:
+            #
+            # While in the paper, it is written that the head metric is calculated by halving the shoulder distance,
+            # this is meant for the distance between the very ends of one's shoulder, as literature studying body
+            # metrics and ratios generally states. The Vision Pose Estimation API, however, seems to be predicting
+            # rather the center of one's shoulder. Based on our experiments and manual reviews of the data, employing
+            # this as just the plain shoulder distance seems to be more corresponding to the desired metric.
+            #
+            # Please, review this if using other third-party pose estimation libraries.
+
+            if row["leftShoulder"][sequence_index][0] != 0 and row["rightShoulder"][sequence_index][0] != 0:
+                left_shoulder = (row["leftShoulder"][sequence_index][0], row["leftShoulder"][sequence_index][1])
+                right_shoulder = (row["rightShoulder"][sequence_index][0], row["rightShoulder"][sequence_index][1])
+                shoulder_distance = ((((left_shoulder[0] - right_shoulder[0]) ** 2) + (
+                        (left_shoulder[1] - right_shoulder[1]) ** 2)) ** 0.5)
+                head_metric = shoulder_distance
+            else:
+                neck = (row["neck"][sequence_index][0], row["neck"][sequence_index][1])
+                nose = (row["nose"][sequence_index][0], row["nose"][sequence_index][1])
+                neck_nose_distance = ((((neck[0] - nose[0]) ** 2) + ((neck[1] - nose[1]) ** 2)) ** 0.5)
+                head_metric = neck_nose_distance
+
+            # Set the starting and ending point of the normalization bounding box
+            #starting_point = [row["neck"][sequence_index][0] - 3 * head_metric,
+            #                 row["leftEye"][sequence_index][1] + (head_metric / 2)]
+            starting_point = [row["neck"][sequence_index][0] - 1 * head_metric,
+                            row["leftEye"][sequence_index][1] - head_metric/2]
+            ending_point = [row["neck"][sequence_index][0] + 1 * head_metric,
+                            starting_point[1] + 3 * head_metric]
+
+            last_starting_point, last_ending_point = starting_point, ending_point
+
+        # Ensure that all of the bounding-box-defining coordinates are not out of the picture
+        if starting_point[0] < 0: starting_point[0] = 0
+        if starting_point[1] > 1: starting_point[1] = 1
+        if ending_point[0] < 0: ending_point[0] = 0
+        if ending_point[1] > 1: ending_point[1] = 1
+
+        # Normalize individual landmarks and save the results
+        for identifier in BODY_IDENTIFIERS:
+            key = identifier
+
+            # Prevent from trying to normalize incorrectly captured points
+            if row[key][sequence_index][0] == 0:
+                continue
+
+            if (ending_point[0] - starting_point[0]) == 0 or (starting_point[1] - ending_point[1]) == 0:
+                logging.info("Problematic normalization")
+                valid_sequence = False
+                break
+
+            normalized_x = (row[key][sequence_index][0] - starting_point[0]) / (ending_point[0] - starting_point[0])
+            normalized_y = (row[key][sequence_index][1] - starting_point[1]) / (ending_point[1] - starting_point[1])
+
+            row[key][sequence_index] = list(row[key][sequence_index])
+
+            row[key][sequence_index][0] = normalized_x
+            row[key][sequence_index][1] = normalized_y
+
+    if valid_sequence:
+        return row
+
+    else:
+        return original_row
+
+
+if __name__ == "__main__":
+    pass

spoter_mod/normalization/hand_normalization.py

@@ -0,0 +1,192 @@
+
+import logging
+import pandas as pd
+
+HAND_IDENTIFIERS = [
+    "wrist",
+    "indexTip",
+    "indexDIP",
+    "indexPIP",
+    "indexMCP",
+    "middleTip",
+    "middleDIP",
+    "middlePIP",
+    "middleMCP",
+    "ringTip",
+    "ringDIP",
+    "ringPIP",
+    "ringMCP",
+    "littleTip",
+    "littleDIP",
+    "littlePIP",
+    "littleMCP",
+    "thumbTip",
+    "thumbIP",
+    "thumbMP",
+    "thumbCMC"
+]
+
+
+def normalize_hands_full(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Normalizes the hands position data using the Bohacek-normalization algorithm.
+
+    :param df: pd.DataFrame to be normalized
+    :return: pd.DataFrame with normalized values for hand pose
+    """
+
+    # TODO: Fix division by zero
+    df.columns = [item.replace("_left_", "_0_").replace("_right_", "_1_") for item in list(df.columns)]
+
+    normalized_df = pd.DataFrame(columns=df.columns)
+
+    hand_landmarks = {"X": {0: [], 1: []}, "Y": {0: [], 1: []}}
+
+    # Determine how many hands are present in the dataset
+    range_hand_size = 1
+    if "wrist_1_X" in df.columns:
+        range_hand_size = 2
+
+    # Construct the relevant identifiers
+    for identifier in HAND_IDENTIFIERS:
+        for hand_index in range(range_hand_size):
+            hand_landmarks["X"][hand_index].append(identifier + "_" + str(hand_index) + "_X")
+            hand_landmarks["Y"][hand_index].append(identifier + "_" + str(hand_index) + "_Y")
+
+    # Iterate over all of the records in the dataset
+    for index, row in df.iterrows():
+        # Treat each hand individually
+        for hand_index in range(range_hand_size):
+
+            sequence_size = len(row["wrist_" + str(hand_index) + "_X"])
+
+            # Treat each element of the sequence (analyzed frame) individually
+            for sequence_index in range(sequence_size):
+
+                # Retrieve all of the X and Y values of the current frame
+                landmarks_x_values = [row[key][sequence_index] for key in hand_landmarks["X"][hand_index] if row[key][sequence_index] != 0]
+                landmarks_y_values = [row[key][sequence_index] for key in hand_landmarks["Y"][hand_index] if row[key][sequence_index] != 0]
+
+                # Prevent from even starting the analysis if some necessary elements are not present
+                if not landmarks_x_values or not landmarks_y_values:
+                    logging.warning(
+                        " HAND LANDMARKS: One frame could not be normalized as there is no data present. Record: " + str(index) +
+                        ", Frame: " + str(sequence_index))
+                    continue
+
+                # Calculate the deltas
+                width, height = max(landmarks_x_values) - min(landmarks_x_values), max(landmarks_y_values) - min(
+                    landmarks_y_values)
+                if width > height:
+                    delta_x = 0.1 * width
+                    delta_y = delta_x + ((width - height) / 2)
+                else:
+                    delta_y = 0.1 * height
+                    delta_x = delta_y + ((height - width) / 2)
+
+                # Set the starting and ending point of the normalization bounding box
+                starting_point = (min(landmarks_x_values) - delta_x, min(landmarks_y_values) - delta_y)
+                ending_point = (max(landmarks_x_values) + delta_x, max(landmarks_y_values) + delta_y)
+
+                # Normalize individual landmarks and save the results
+                for identifier in HAND_IDENTIFIERS:
+                    key = identifier + "_" + str(hand_index) + "_"
+
+                    # Prevent from trying to normalize incorrectly captured points
+                    if row[key + "X"][sequence_index] == 0 or (ending_point[0] - starting_point[0]) == 0 or (starting_point[1] - ending_point[1]) == 0:
+                        continue
+
+                    normalized_x = (row[key + "X"][sequence_index] - starting_point[0]) / (ending_point[0] -
+                                                                                           starting_point[0])
+                    normalized_y = (row[key + "Y"][sequence_index] - ending_point[1]) / (starting_point[1] -
+                                                                                         ending_point[1])
+
+                    row[key + "X"][sequence_index] = normalized_x
+                    row[key + "Y"][sequence_index] = normalized_y
+
+        normalized_df = normalized_df.append(row, ignore_index=True)
+
+    return normalized_df
+
+
+def normalize_single_dict(row: dict):
+    """
+    Normalizes the skeletal data for a given sequence of frames with signer's hand pose data. The normalization follows
+    the definition from our paper.
+
+    :param row: Dictionary containing key-value pairs with joint identifiers and corresponding lists (sequences) of
+                that particular joints coordinates
+    :return: Dictionary with normalized skeletal data (following the same schema as input data)
+    """
+
+    hand_landmarks = {0: [], 1: []}
+
+    # Determine how many hands are present in the dataset
+    range_hand_size = 1
+    if "wrist_1" in row.keys():
+        range_hand_size = 2
+
+    # Construct the relevant identifiers
+    for identifier in HAND_IDENTIFIERS:
+        for hand_index in range(range_hand_size):
+            hand_landmarks[hand_index].append(identifier + "_" + str(hand_index))
+
+    # Treat each hand individually
+    for hand_index in range(range_hand_size):
+
+        sequence_size = len(row["wrist_" + str(hand_index)])
+
+        # Treat each element of the sequence (analyzed frame) individually
+        for sequence_index in range(sequence_size):
+
+            # Retrieve all of the X and Y values of the current frame
+            landmarks_x_values = [row[key][sequence_index][0] for key in hand_landmarks[hand_index] if
+                                  row[key][sequence_index][0] != 0]
+            landmarks_y_values = [row[key][sequence_index][1] for key in hand_landmarks[hand_index] if
+                                  row[key][sequence_index][1] != 0]
+
+            # Prevent from even starting the analysis if some necessary elements are not present
+            if not landmarks_x_values or not landmarks_y_values:
+                continue
+
+            # Calculate the deltas
+            width, height = max(landmarks_x_values) - min(landmarks_x_values), max(landmarks_y_values) - min(
+                landmarks_y_values)
+            if width > height:
+                delta_x = 0.1 * width
+                delta_y = delta_x + ((width - height) / 2)
+            else:
+                delta_y = 0.1 * height
+                delta_x = delta_y + ((height - width) / 2)
+
+            # Set the starting and ending point of the normalization bounding box
+            starting_point = [min(landmarks_x_values) - delta_x, min(landmarks_y_values) - delta_y]
+            ending_point = [max(landmarks_x_values) + delta_x, max(landmarks_y_values) + delta_y]
+            # Ensure that all of the bounding-box-defining coordinates are not out of the picture
+            if starting_point[0] < 0: starting_point[0] = 0
+            if starting_point[1] > 1: starting_point[1] = 1
+            if ending_point[0] < 0: ending_point[0] = 0
+            if ending_point[1] > 1: ending_point[1] = 1
+
+            # Normalize individual landmarks and save the results
+            for identifier in HAND_IDENTIFIERS:
+                key = identifier + "_" + str(hand_index)
+
+                # Prevent from trying to normalize incorrectly captured points
+                if row[key][sequence_index][0] == 0 or (ending_point[0] - starting_point[0]) == 0 or (
+                        starting_point[1] - ending_point[1]) == 0:
+                    continue
+
+                normalized_x = (row[key][sequence_index][0] - starting_point[0]) / (ending_point[0] - starting_point[0])
+                normalized_y = (row[key][sequence_index][1] - starting_point[1]) / (ending_point[1] - starting_point[1])
+
+                row[key][sequence_index] = list(row[key][sequence_index])
+
+                row[key][sequence_index][0] = normalized_x
+                row[key][sequence_index][1] = normalized_y
+
+    return row
+
+
+if __name__ == "__main__":
+    pass

spoter_mod/normalization/main.py

@@ -0,0 +1,43 @@
+
+import ast
+import pandas as pd
+
+from normalization.hand_normalization import normalize_hands_full
+from normalization.body_normalization import normalize_body_full
+
+
+# Load the dataset
+df = pd.read_csv("/Users/matyasbohacek/Documents/WLASL_test_15fps.csv", encoding="utf-8")
+
+# Retrieve metadata
+video_size_heights = df["video_size_height"].to_list()
+video_size_widths = df["video_size_width"].to_list()
+
+# Delete redundant (non-related) properties
+del df["video_size_height"]
+del df["video_size_width"]
+
+# Temporarily remove other relevant metadata
+labels = df["labels"].to_list()
+video_fps = df["video_fps"].to_list()
+del df["labels"]
+del df["video_fps"]
+
+# Convert the strings into lists
+convert = lambda x: ast.literal_eval(str(x))
+for column in df.columns:
+    df[column] = df[column].apply(convert)
+
+# Perform the normalizations
+df = normalize_hands_full(df)
+df, invalid_row_indexes = normalize_body_full(df)
+
+# Clear lists of items from deleted rows
+# labels = [t for i, t in enumerate(labels) if i not in invalid_row_indexes]
+# video_fps = [t for i, t in enumerate(video_fps) if i not in invalid_row_indexes]
+
+# Return the metadata back to the dataset
+df["labels"] = labels
+df["video_fps"] = video_fps
+
+df.to_csv("/Users/matyasbohacek/Desktop/WLASL_test_15fps_normalized.csv", encoding="utf-8", index=False)

spoter_mod/pose_model_identifier.py

@@ -0,0 +1,103 @@
+import pandas as pd
+
+BODY_IDENTIFIERS = {
+    "nose": 0,
+    "neck": -1,
+    "rightEye": 5,
+    "leftEye": 2,
+    "rightEar": 8,
+    "leftEar": 7,
+    "rightShoulder": 12,
+    "leftShoulder": 11,
+    "rightElbow": 14,
+    "leftElbow": 13,
+    "rightWrist": 16,
+    "leftWrist": 15
+}
+HAND_IDENTIFIERS = {
+    "wrist": 0,
+    "indexTip": 8,
+    "indexDIP": 7,
+    "indexPIP": 6,
+    "indexMCP": 5,
+    "middleTip": 12,
+    "middleDIP": 11,
+    "middlePIP": 10,
+    "middleMCP": 9,
+    "ringTip": 16,
+    "ringDIP": 15,
+    "ringPIP": 14,
+    "ringMCP": 13,
+    "littleTip": 20,
+    "littleDIP": 19,
+    "littlePIP": 18,
+    "littleMCP": 17,
+    "thumbTip": 4,
+    "thumbIP": 3,
+    "thumbMP": 2,
+    "thumbCMC": 1
+}
+
+
+class mp_holistic_data:
+    def __init__(self, column_names):
+        self.data_hub = {}
+        for n in column_names[1:-1]:
+            self.data_hub[n] = []
+
+    def hand_append_zero(self, handedness):
+        for k in self.data_hub.keys():
+            if "_" + handedness + "_" in k:
+                self.data_hub[k].append(0)
+
+    def hand_append_value(self, handedness, hand_landmarks):
+        for name, lm_idx in HAND_IDENTIFIERS.items():
+            lm = hand_landmarks.landmark[lm_idx]
+            for xy, xy_value in zip(['_X', '_Y'], [lm.x, lm.y]):
+                k = name + '_' + handedness + xy
+                self.data_hub[k].append(xy_value)
+
+    def get_series(self):
+        return pd.Series(self.data_hub)
+
+    def extract_data(self, holistic_results):
+        def neck(pose_results):
+            ls = pose_results.pose_landmarks.landmark[11]
+            rs = pose_results.pose_landmarks.landmark[12]
+            no = pose_results.pose_landmarks.landmark[0]
+            if (ls.visibility > 0.5) & (rs.visibility > 0.5) & (no.visibility > 0.5):
+                # This indicates the neck better. But it does not affect the result.
+                cx = (ls.x + rs.x) / 2
+                cy = (ls.y + rs.y) / 2
+                dx = no.x - cx
+                dy = no.y - cy
+                x = cx + 0.3 * dx
+                y = cy + 0.3 * dy
+                # x = (ls.x+rs.x)/2
+                # y = (ls.y+rs.y)/2
+            else:
+                x = 0
+                y = 0
+            return [x, y]
+
+        # for the frame that can not extract skeleton from
+        if not holistic_results.pose_landmarks:
+            return
+        for name, lm_idx in BODY_IDENTIFIERS.items():
+            if name == "neck":
+                xy_value = neck(holistic_results)
+            else:
+                lm = holistic_results.pose_landmarks.landmark[lm_idx]
+                visible = float(lm.visibility >= 0.5)
+                xy_value = [lm.x * visible, lm.y * visible]
+            for xy_id, xy in zip(['_X', '_Y'], xy_value):
+                s_name = name + xy_id
+                self.data_hub[s_name].append(xy)
+
+        for handedness, lm in zip(['Right', 'Left'],
+                                  [holistic_results.right_hand_landmarks, holistic_results.left_hand_landmarks]):
+            if lm:
+                self.hand_append_value(handedness, lm)
+            else:
+                self.hand_append_zero(handedness)
+        return

spoter_mod/requirements.txt

@@ -0,0 +1,7 @@
+pandas==1.1.5
+tqdm==4.54.1
+matplotlib
+torch==1.8.1
+torchvision
+scikit-learn
+opencv-python

spoter_mod/skeleton_extractor.py

@@ -0,0 +1,60 @@
+import warnings
+
+import pandas as pd
+from os import path
+import cv2
+import mediapipe as mp
+import json
+from spoter_mod.pose_model_identifier import BODY_IDENTIFIERS, HAND_IDENTIFIERS, mp_holistic_data
+
+mp_drawing = mp.solutions.drawing_utils
+mp_holistic = mp.solutions.holistic
+mp_drawing_styles = mp.solutions.drawing_styles
+
+holistic = mp_holistic.Holistic()
+
+column_names = []
+column_names.append('video_id')
+for id_name in BODY_IDENTIFIERS.keys():
+    for xy in ["_X", "_Y"]:
+        column_names.append(id_name + xy)
+
+for lr in ["_Right", "_Left"]:
+    for id_name in HAND_IDENTIFIERS.keys():
+        for xy in ["_X", "_Y"]:
+            column_names.append(id_name + lr + xy)
+
+column_names.append('labels')
+
+
+def create_df(flnm, column_names):
+    df = pd.DataFrame(columns=column_names)
+    return df
+
+
+def save_data(df, data, flnm):
+    df = df.append(data.get_series(), ignore_index=True)
+    df.to_pickle(flnm)
+
+
+def obtain_pose_data(path):
+    cap = cv2.VideoCapture(path)
+    data = mp_holistic_data(column_names)
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+        # Recolor image to RGB
+        image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        # Make detection
+        holistic_results = holistic.process(image)
+        # Extract feature and save to mp_pose_data class
+        data.extract_data(holistic_results)
+    cap.release()
+
+    return data
+
+
+if __name__ == '__main__':
+    pass

spoter_mod/sweep-agent.sh

@@ -0,0 +1,18 @@
+
+#PBS -N spoter-zhuo-sweep
+#PBS -q gpu
+#PBS -l walltime=24:00:00
+
+#PBS -l select=1:ncpus=1:ngpus=1:cluster=adan:mem=10gb
+#PBS -j oe
+#PBS -m ae
+
+echo "Experiment starting..."
+
+cd /storage/plzen4-ntis/home/mbohacek/spoter-zhuo
+
+module add conda-modules
+conda activate cslr-transformers
+
+wandb agent matyasbohacek/Zhuo-collab-SPOTER-Sweep/bh6fc056
+

spoter_mod/sweep.yaml

@@ -0,0 +1,50 @@
+program: train.py
+method: bayes
+project: Zhuo-collab-SPOTER-Sweep
+metric:
+  name: best-accuracy
+  goal: maximize
+parameters:
+  augmentations_probability:
+    min: 0.20
+    max: 0.75
+  rotate_angle:
+    min: 5
+    max: 20
+  perspective_transform_ratio:
+    min: 0.05
+    max: 0.2
+  squeeze_ratio:
+    min: 0.05
+    max: 0.4
+  arm_joint_rotate_angle:
+    min: 1
+    max: 10
+  arm_joint_rotate_probability:
+    min: 0.2
+    max: 0.5
+command:
+  - python3
+  - "-m"
+  - train
+  - "--epochs"
+  - 130
+  -  "--num_classes"
+  - 100
+  - "--lr"
+  - 0.001
+  - "--experiment_name"
+  - "zhuo-repro"
+  - "--training_set_path"
+  - "/storage/plzen4-ntis/home/mbohacek/spoter-zhuo/WLASL100_zhuo_train.csv"
+  - "--testing_set_path"
+  - "/storage/plzen4-ntis/home/mbohacek/spoter-zhuo/WLASL100_zhuo_test.csv"
+  - "--validation_set_path"
+  - "/storage/plzen4-ntis/home/mbohacek/spoter-zhuo/WLASL100_zhuo_val.csv"
+  - "--validation_set"
+  - "from-file"
+  - "--wandb_key"
+  - "beb8925bb5b17aaecd40139da4c299f76753291e"
+  - "--wandb_entity"
+  - "matyasbohacek"
+  - ${args}

spoter_mod/train.py

@@ -0,0 +1,312 @@
+
+import os
+import argparse
+import random
+import logging
+import torch
+import wandb
+
+import numpy as np
+import torch.nn as nn
+import torch.optim as optim
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+from torchvision import transforms
+from torch.utils.data import DataLoader
+from pathlib import Path
+
+from utils import __balance_val_split, __split_of_train_sequence
+from datasets.czech_slr_dataset import CzechSLRDataset
+from spoter.spoter_model import SPOTER
+from spoter.utils import train_epoch, evaluate
+from spoter.gaussian_noise import GaussianNoise
+
+
+def get_default_args():
+    parser = argparse.ArgumentParser(add_help=False)
+
+    parser.add_argument("--experiment_name", type=str, default="lsa_64_spoter",
+                        help="Name of the experiment after which the logs and plots will be named")
+    parser.add_argument("--num_classes", type=int, default=64, help="Number of classes to be recognized by the model")
+    parser.add_argument("--hidden_dim", type=int, default=108,
+                        help="Hidden dimension of the underlying Transformer model")
+    parser.add_argument("--seed", type=int, default=379,
+                        help="Seed with which to initialize all the random components of the training")
+
+    # Data
+    parser.add_argument("--training_set_path", type=str, default="", help="Path to the training dataset CSV file")
+    parser.add_argument("--testing_set_path", type=str, default="", help="Path to the testing dataset CSV file")
+    parser.add_argument("--experimental_train_split", type=float, default=None,
+                        help="Determines how big a portion of the training set should be employed (intended for the "
+                             "gradually enlarging training set experiment from the paper)")
+
+    parser.add_argument("--validation_set", type=str, choices=["from-file", "split-from-train", "none"],
+                        default="from-file", help="Type of validation set construction. See README for further rederence")
+    parser.add_argument("--validation_set_size", type=float,
+                        help="Proportion of the training set to be split as validation set, if 'validation_size' is set"
+                             " to 'split-from-train'")
+    parser.add_argument("--validation_set_path", type=str, default="", help="Path to the validation dataset CSV file")
+
+    # Training hyperparameters
+    parser.add_argument("--epochs", type=int, default=100, help="Number of epochs to train the model for")
+    parser.add_argument("--lr", type=float, default=0.001, help="Learning rate for the model training")
+    parser.add_argument("--log_freq", type=int, default=1,
+                        help="Log frequency (frequency of printing all the training info)")
+
+    # Checkpointing
+    parser.add_argument("--save_checkpoints", type=bool, default=True,
+                        help="Determines whether to save weights checkpoints")
+
+    # Scheduler
+    parser.add_argument("--scheduler_factor", type=int, default=0.1, help="Factor for the ReduceLROnPlateau scheduler")
+    parser.add_argument("--scheduler_patience", type=int, default=5,
+                        help="Patience for the ReduceLROnPlateau scheduler")
+
+    # Gaussian noise normalization
+    parser.add_argument("--gaussian_mean", type=int, default=0, help="Mean parameter for Gaussian noise layer")
+    parser.add_argument("--gaussian_std", type=int, default=0.001,
+                        help="Standard deviation parameter for Gaussian noise layer")
+
+    parser.add_argument("--augmentations_probability", type=float, default=0.5, help="")  # 0.462
+    parser.add_argument("--rotate_angle", type=int, default=17, help="")  # 17
+    parser.add_argument("--perspective_transform_ratio", type=float, default=0.2, help="")  # 0.1682
+    parser.add_argument("--squeeze_ratio", type=float, default=0.4, help="")  # 0.3971
+    parser.add_argument("--arm_joint_rotate_angle", type=int, default=4, help="")  # 3
+    parser.add_argument("--arm_joint_rotate_probability", type=float, default=0.4, help="")  # 0.3596
+
+    # Visualization
+    parser.add_argument("--plot_stats", type=bool, default=True,
+                        help="Determines whether continuous statistics should be plotted at the end")
+    parser.add_argument("--plot_lr", type=bool, default=True,
+                        help="Determines whether the LR should be plotted at the end")
+
+    # WANDB
+    parser.add_argument("--wandb_key", type=str, default="", help="")
+    parser.add_argument("--wandb_entity", type=str, default="", help="")
+
+    return parser
+
+
+def train(args):
+
+    if args.wandb_key:
+        wandb.login(key=args.wandb_key)
+        wandb.init(project=args.experiment_name, entity=args.wandb_entity)
+        wandb.config.update(args)
+
+    # MARK: TRAINING PREPARATION AND MODULES
+    args.experiment_name = args.experiment_name + "_lr" + wandb.run.id
+
+    # Initialize all the random seeds
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    os.environ["PYTHONHASHSEED"] = str(args.seed)
+    torch.manual_seed(args.seed)
+    torch.cuda.manual_seed(args.seed)
+    torch.cuda.manual_seed_all(args.seed)
+    torch.backends.cudnn.deterministic = True
+    g = torch.Generator()
+    g.manual_seed(args.seed)
+
+    # Set the output format to print into the console and save into LOG file
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        handlers=[
+            logging.FileHandler(args.experiment_name + "_" + str(args.experimental_train_split).replace(".", "") + ".log")
+        ]
+    )
+
+    # Set device to CUDA only if applicable
+    device = torch.device("cpu")
+    if torch.cuda.is_available():
+        device = torch.device("cuda")
+
+    # Construct the model
+    slrt_model = SPOTER(num_classes=args.num_classes, hidden_dim=args.hidden_dim)
+    slrt_model.train(True)
+    slrt_model.to(device)
+
+    # Construct the other modules
+    cel_criterion = nn.CrossEntropyLoss()
+    sgd_optimizer = optim.SGD(slrt_model.parameters(), lr=args.lr)
+    scheduler = optim.lr_scheduler.ReduceLROnPlateau(sgd_optimizer, factor=args.scheduler_factor, patience=args.scheduler_patience)
+
+    # Ensure that the path for checkpointing and for images both exist
+    Path("out-checkpoints/" + args.experiment_name + "/").mkdir(parents=True, exist_ok=True)
+    Path("out-img/").mkdir(parents=True, exist_ok=True)
+
+
+    # MARK: DATA
+
+    # Training set
+    transform = transforms.Compose([GaussianNoise(args.gaussian_mean, args.gaussian_std)])
+    augmentations_config = {
+        "rotate-angle": args.rotate_angle,
+        "perspective-transform-ratio": args.perspective_transform_ratio,
+        "squeeze-ratio": args.squeeze_ratio,
+        "arm-joint-rotate-angle": args.arm_joint_rotate_angle,
+        "arm-joint-rotate-probability": args.arm_joint_rotate_probability
+    }
+
+    train_set = CzechSLRDataset(args.training_set_path, transform=transform, augmentations=True,
+                                augmentations_prob=args.augmentations_probability, augmentations_config=augmentations_config)
+
+    # Validation set
+    if args.validation_set == "from-file":
+        val_set = CzechSLRDataset(args.validation_set_path)
+        val_loader = DataLoader(val_set, shuffle=True, generator=g)
+
+    elif args.validation_set == "split-from-train":
+        train_set, val_set = __balance_val_split(train_set, 0.2)
+
+        val_set.transform = None
+        val_set.augmentations = False
+        val_loader = DataLoader(val_set, shuffle=True, generator=g)
+
+    else:
+        val_loader = None
+
+    # Testing set
+    if args.testing_set_path:
+        eval_set = CzechSLRDataset(args.testing_set_path)
+        eval_loader = DataLoader(eval_set, shuffle=True, generator=g)
+
+    else:
+        eval_loader = None
+
+    # Final training set refinements
+    if args.experimental_train_split:
+        train_set = __split_of_train_sequence(train_set, args.experimental_train_split)
+
+    train_loader = DataLoader(train_set, shuffle=True, generator=g)
+
+
+    # MARK: TRAINING
+    train_acc, val_acc = 0, 0
+    losses, train_accs, val_accs = [], [], []
+    lr_progress = []
+    top_train_acc, top_val_acc = 0, 0
+    checkpoint_index = 0
+
+    if args.experimental_train_split:
+        print("Starting " + args.experiment_name + "_" + str(args.experimental_train_split).replace(".", "") + "...\n\n")
+        logging.info("Starting " + args.experiment_name + "_" + str(args.experimental_train_split).replace(".", "") + "...\n\n")
+
+    else:
+        print("Starting " + args.experiment_name + "...\n\n")
+        logging.info("Starting " + args.experiment_name + "...\n\n")
+
+    for epoch in range(args.epochs):
+        train_loss, _, _, train_acc = train_epoch(slrt_model, train_loader, cel_criterion, sgd_optimizer, device)
+        losses.append(train_loss.item() / len(train_loader))
+        train_accs.append(train_acc)
+
+        if val_loader:
+            slrt_model.train(False)
+            _, _, val_acc = evaluate(slrt_model, val_loader, device)
+            slrt_model.train(True)
+            val_accs.append(val_acc)
+
+        # Save checkpoints if they are best in the current subset
+        if args.save_checkpoints:
+            if train_acc > top_train_acc:
+                top_train_acc = train_acc
+                torch.save(slrt_model, "out-checkpoints/" + args.experiment_name + "/checkpoint_t_" + str(checkpoint_index) + ".pth")
+
+            if val_acc > top_val_acc:
+                top_val_acc = val_acc
+                torch.save(slrt_model, "out-checkpoints/" + args.experiment_name + "/checkpoint_v_" + str(checkpoint_index) + ".pth")
+
+        if epoch % args.log_freq == 0:
+            print("[" + str(epoch + 1) + "] TRAIN  loss: " + str(train_loss.item() / len(train_loader)) + " acc: " + str(train_acc))
+            logging.info("[" + str(epoch + 1) + "] TRAIN  loss: " + str(train_loss.item() / len(train_loader)) + " acc: " + str(train_acc))
+
+            wandb.log({
+                "epoch": int(epoch + 1),
+                "train-loss": float(train_loss.item() / len(train_loader)),
+                "train-accuracy": train_acc
+            })
+
+            if val_loader:
+                print("[" + str(epoch + 1) + "] VALIDATION  acc: " + str(val_acc))
+                logging.info("[" + str(epoch + 1) + "] VALIDATION  acc: " + str(val_acc))
+
+                if args.wandb_key:
+                    wandb.log({
+                        "validation-accuracy": val_acc
+                    })
+
+            print("")
+            logging.info("")
+
+        # Reset the top accuracies on static subsets
+        if epoch % 10 == 0:
+            top_train_acc, top_val_acc = 0, 0
+            checkpoint_index += 1
+
+        lr_progress.append(sgd_optimizer.param_groups[0]["lr"])
+
+    # MARK: TESTING
+
+    print("\nTesting checkpointed models starting...\n")
+    logging.info("\nTesting checkpointed models starting...\n")
+
+    top_result, top_result_name = 0, ""
+
+    if eval_loader:
+        for i in range(checkpoint_index):
+            for checkpoint_id in ["t", "v"]:
+                # tested_model = VisionTransformer(dim=2, mlp_dim=108, num_classes=100, depth=12, heads=8)
+                tested_model = torch.load("out-checkpoints/" + args.experiment_name + "/checkpoint_" + checkpoint_id + "_" + str(i) + ".pth")
+                tested_model.train(False)
+                _, _, eval_acc = evaluate(tested_model, eval_loader, device, print_stats=True)
+
+                if eval_acc > top_result:
+                    top_result = eval_acc
+                    top_result_name = args.experiment_name + "/checkpoint_" + checkpoint_id + "_" + str(i)
+
+                print("checkpoint_" + checkpoint_id + "_" + str(i) + "  ->  " + str(eval_acc))
+                logging.info("checkpoint_" + checkpoint_id + "_" + str(i) + "  ->  " + str(eval_acc))
+
+        print("\nThe top result was recorded at " + str(top_result) + " testing accuracy. The best checkpoint is " + top_result_name + ".")
+        logging.info("\nThe top result was recorded at " + str(top_result) + " testing accuracy. The best checkpoint is " + top_result_name + ".")
+
+    if args.wandb_key:
+        wandb.run.summary["best-accuracy"] = top_result
+        wandb.run.summary["best-checkpoint"] = top_result_name
+
+    # PLOT 0: Performance (loss, accuracies) chart plotting
+    if args.plot_stats:
+        fig, ax = plt.subplots()
+        ax.plot(range(1, len(losses) + 1), losses, c="#D64436", label="Training loss")
+        ax.plot(range(1, len(train_accs) + 1), train_accs, c="#00B09B", label="Training accuracy")
+
+        if val_loader:
+            ax.plot(range(1, len(val_accs) + 1), val_accs, c="#E0A938", label="Validation accuracy")
+
+        ax.xaxis.set_major_locator(ticker.MaxNLocator(integer=True))
+
+        ax.set(xlabel="Epoch", ylabel="Accuracy / Loss", title="")
+        plt.legend(loc="upper center", bbox_to_anchor=(0.5, 1.05), ncol=4, fancybox=True, shadow=True, fontsize="xx-small")
+        ax.grid()
+
+        fig.savefig("out-img/" + args.experiment_name + "_loss.png")
+
+    # PLOT 1: Learning rate progress
+    if args.plot_lr:
+        fig1, ax1 = plt.subplots()
+        ax1.plot(range(1, len(lr_progress) + 1), lr_progress, label="LR")
+        ax1.set(xlabel="Epoch", ylabel="LR", title="")
+        ax1.grid()
+
+        fig1.savefig("out-img/" + args.experiment_name + "_lr.png")
+
+    print("\nAny desired statistics have been plotted.\nThe experiment is finished.")
+    logging.info("\nAny desired statistics have been plotted.\nThe experiment is finished.")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("", parents=[get_default_args()], add_help=False)
+    args = parser.parse_args()
+    train(args)

spoter_mod/utils.py

@@ -0,0 +1,41 @@
+
+import numpy as np
+
+from collections import Counter
+from torch.utils.data import Subset
+from sklearn.model_selection import train_test_split
+
+
+def __balance_val_split(dataset, val_split=0.):
+    targets = np.array(dataset.targets)
+    train_indices, val_indices = train_test_split(
+        np.arange(targets.shape[0]),
+        test_size=val_split,
+        stratify=targets
+    )
+
+    train_dataset = Subset(dataset, indices=train_indices)
+    val_dataset = Subset(dataset, indices=val_indices)
+
+    return train_dataset, val_dataset
+
+
+def __split_of_train_sequence(subset: Subset, train_split=1.0):
+    if train_split == 1:
+        return subset
+
+    targets = np.array([subset.dataset.targets[i] for i in subset.indices])
+    train_indices, _ = train_test_split(
+        np.arange(targets.shape[0]),
+        test_size=1 - train_split,
+        stratify=targets
+    )
+
+    train_dataset = Subset(subset.dataset, indices=[subset.indices[i] for i in train_indices])
+
+    return train_dataset
+
+
+def __log_class_statistics(subset: Subset):
+    train_classes = [subset.dataset.targets[i] for i in subset.indices]
+    print(dict(Counter(train_classes)))