Implement and test custom pipeline

notebooks/01_Demo_Pipeline_Usage_In_Code.ipynb

@@ -0,0 +1,162 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "- Alternative maybe https://huggingface.co/docs/transformers/custom_models"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "outputs": [],
+   "source": [
+    "from transformers.pipelines import PIPELINE_REGISTRY\n",
+    "from transformers import pipeline\n",
+    "from pipeline_wrapper import MyPipeline"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-05T11:23:02.669143300Z",
+     "start_time": "2024-02-05T11:23:02.652092400Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ecg2hrv is already registered. Overwriting pipeline for task ecg2hrv...\n"
+     ]
+    }
+   ],
+   "source": [
+    "PIPELINE_REGISTRY.register_pipeline(\n",
+    "    \"ecg2hrv\",\n",
+    "    pipeline_class=MyPipeline,\n",
+    "    # model_class=MyModel\n",
+    ")\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-05T11:41:30.464821400Z",
+     "start_time": "2024-02-05T11:41:30.444149300Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "outputs": [
+    {
+     "ename": "KeyError",
+     "evalue": "'default'",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
+      "\u001B[1;31mKeyError\u001B[0m                                  Traceback (most recent call last)",
+      "Cell \u001B[1;32mIn[18], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m feature_extractor \u001B[38;5;241m=\u001B[39m \u001B[43mpipeline\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mecg2hrv\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\pipelines\\__init__.py:833\u001B[0m, in \u001B[0;36mpipeline\u001B[1;34m(task, model, config, tokenizer, feature_extractor, image_processor, framework, revision, use_fast, token, device, device_map, torch_dtype, trust_remote_code, model_kwargs, pipeline_class, **kwargs)\u001B[0m\n\u001B[0;32m    830\u001B[0m \u001B[38;5;66;03m# Use default model/config/tokenizer for the task if no model is provided\u001B[39;00m\n\u001B[0;32m    831\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m model \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m    832\u001B[0m     \u001B[38;5;66;03m# At that point framework might still be undetermined\u001B[39;00m\n\u001B[1;32m--> 833\u001B[0m     model, default_revision \u001B[38;5;241m=\u001B[39m \u001B[43mget_default_model_and_revision\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtargeted_task\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mframework\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtask_options\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    834\u001B[0m     revision \u001B[38;5;241m=\u001B[39m revision \u001B[38;5;28;01mif\u001B[39;00m revision \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;28;01melse\u001B[39;00m default_revision\n\u001B[0;32m    835\u001B[0m     logger\u001B[38;5;241m.\u001B[39mwarning(\n\u001B[0;32m    836\u001B[0m         \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mNo model was supplied, defaulted to \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mmodel\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m and revision\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    837\u001B[0m         \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mrevision\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m (\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mHUGGINGFACE_CO_RESOLVE_ENDPOINT\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m/\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mmodel\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m).\u001B[39m\u001B[38;5;130;01m\\n\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    838\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUsing a pipeline without specifying a model name and revision in production is not recommended.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    839\u001B[0m     )\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\pipelines\\base.py:399\u001B[0m, in \u001B[0;36mget_default_model_and_revision\u001B[1;34m(targeted_task, framework, task_options)\u001B[0m\n\u001B[0;32m    396\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m is_tf_available() \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m is_torch_available():\n\u001B[0;32m    397\u001B[0m     framework \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtf\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m--> 399\u001B[0m defaults \u001B[38;5;241m=\u001B[39m \u001B[43mtargeted_task\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mdefault\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m]\u001B[49m\n\u001B[0;32m    400\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m task_options:\n\u001B[0;32m    401\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m task_options \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m defaults:\n",
+      "\u001B[1;31mKeyError\u001B[0m: 'default'"
+     ]
+    }
+   ],
+   "source": [
+    "feature_extractor = pipeline(\"ecg2hrv\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-05T11:41:40.766340400Z",
+     "start_time": "2024-02-05T11:41:40.656799200Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "outputs": [
+    {
+     "ename": "OSError",
+     "evalue": "It looks like the config file at 'C:\\Users\\merti\\.cache\\huggingface\\hub\\models--HUBII-Platform--ECG2HRV\\snapshots\\7742915f18da533bfd88c1a49a9e9727635cd1d7\\config.json' is not a valid JSON file.",
+     "output_type": "error",
+     "traceback": [
+      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
+      "\u001B[1;31mJSONDecodeError\u001B[0m                           Traceback (most recent call last)",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\configuration_utils.py:719\u001B[0m, in \u001B[0;36mPretrainedConfig._get_config_dict\u001B[1;34m(cls, pretrained_model_name_or_path, **kwargs)\u001B[0m\n\u001B[0;32m    717\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m    718\u001B[0m     \u001B[38;5;66;03m# Load config dict\u001B[39;00m\n\u001B[1;32m--> 719\u001B[0m     config_dict \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mcls\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_dict_from_json_file\u001B[49m\u001B[43m(\u001B[49m\u001B[43mresolved_config_file\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    720\u001B[0m     config_dict[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m commit_hash\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\configuration_utils.py:818\u001B[0m, in \u001B[0;36mPretrainedConfig._dict_from_json_file\u001B[1;34m(cls, json_file)\u001B[0m\n\u001B[0;32m    817\u001B[0m     text \u001B[38;5;241m=\u001B[39m reader\u001B[38;5;241m.\u001B[39mread()\n\u001B[1;32m--> 818\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mjson\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mloads\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtext\u001B[49m\u001B[43m)\u001B[49m\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\json\\__init__.py:346\u001B[0m, in \u001B[0;36mloads\u001B[1;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001B[0m\n\u001B[0;32m    343\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\u001B[38;5;28mcls\u001B[39m \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m object_hook \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m\n\u001B[0;32m    344\u001B[0m         parse_int \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m parse_float \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m\n\u001B[0;32m    345\u001B[0m         parse_constant \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m object_pairs_hook \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m kw):\n\u001B[1;32m--> 346\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43m_default_decoder\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mdecode\u001B[49m\u001B[43m(\u001B[49m\u001B[43ms\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    347\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mcls\u001B[39m \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\json\\decoder.py:337\u001B[0m, in \u001B[0;36mJSONDecoder.decode\u001B[1;34m(self, s, _w)\u001B[0m\n\u001B[0;32m    333\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001B[39;00m\n\u001B[0;32m    334\u001B[0m \u001B[38;5;124;03mcontaining a JSON document).\u001B[39;00m\n\u001B[0;32m    335\u001B[0m \n\u001B[0;32m    336\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m--> 337\u001B[0m obj, end \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mraw_decode\u001B[49m\u001B[43m(\u001B[49m\u001B[43ms\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43midx\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m_w\u001B[49m\u001B[43m(\u001B[49m\u001B[43ms\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m0\u001B[39;49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mend\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    338\u001B[0m end \u001B[38;5;241m=\u001B[39m _w(s, end)\u001B[38;5;241m.\u001B[39mend()\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\json\\decoder.py:355\u001B[0m, in \u001B[0;36mJSONDecoder.raw_decode\u001B[1;34m(self, s, idx)\u001B[0m\n\u001B[0;32m    354\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mStopIteration\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m err:\n\u001B[1;32m--> 355\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m JSONDecodeError(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mExpecting value\u001B[39m\u001B[38;5;124m\"\u001B[39m, s, err\u001B[38;5;241m.\u001B[39mvalue) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m    356\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m obj, end\n",
+      "\u001B[1;31mJSONDecodeError\u001B[0m: Expecting value: line 1 column 1 (char 0)",
+      "\nDuring handling of the above exception, another exception occurred:\n",
+      "\u001B[1;31mOSError\u001B[0m                                   Traceback (most recent call last)",
+      "Cell \u001B[1;32mIn[17], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m feature_extractor \u001B[38;5;241m=\u001B[39m \u001B[43mpipeline\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mecg2hrv\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmodel\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43mHUBII-Platform/ECG2HRV\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m)\u001B[49m\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\pipelines\\__init__.py:782\u001B[0m, in \u001B[0;36mpipeline\u001B[1;34m(task, model, config, tokenizer, feature_extractor, image_processor, framework, revision, use_fast, token, device, device_map, torch_dtype, trust_remote_code, model_kwargs, pipeline_class, **kwargs)\u001B[0m\n\u001B[0;32m    779\u001B[0m                 adapter_config \u001B[38;5;241m=\u001B[39m json\u001B[38;5;241m.\u001B[39mload(f)\n\u001B[0;32m    780\u001B[0m                 model \u001B[38;5;241m=\u001B[39m adapter_config[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbase_model_name_or_path\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n\u001B[1;32m--> 782\u001B[0m     config \u001B[38;5;241m=\u001B[39m AutoConfig\u001B[38;5;241m.\u001B[39mfrom_pretrained(\n\u001B[0;32m    783\u001B[0m         model, _from_pipeline\u001B[38;5;241m=\u001B[39mtask, code_revision\u001B[38;5;241m=\u001B[39mcode_revision, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mhub_kwargs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mmodel_kwargs\n\u001B[0;32m    784\u001B[0m     )\n\u001B[0;32m    785\u001B[0m     hub_kwargs[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m config\u001B[38;5;241m.\u001B[39m_commit_hash\n\u001B[0;32m    787\u001B[0m custom_tasks \u001B[38;5;241m=\u001B[39m {}\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\models\\auto\\configuration_auto.py:1100\u001B[0m, in \u001B[0;36mAutoConfig.from_pretrained\u001B[1;34m(cls, pretrained_model_name_or_path, **kwargs)\u001B[0m\n\u001B[0;32m   1097\u001B[0m trust_remote_code \u001B[38;5;241m=\u001B[39m kwargs\u001B[38;5;241m.\u001B[39mpop(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mtrust_remote_code\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m)\n\u001B[0;32m   1098\u001B[0m code_revision \u001B[38;5;241m=\u001B[39m kwargs\u001B[38;5;241m.\u001B[39mpop(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcode_revision\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m)\n\u001B[1;32m-> 1100\u001B[0m config_dict, unused_kwargs \u001B[38;5;241m=\u001B[39m PretrainedConfig\u001B[38;5;241m.\u001B[39mget_config_dict(pretrained_model_name_or_path, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m   1101\u001B[0m has_remote_code \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mauto_map\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m config_dict \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mAutoConfig\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m config_dict[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mauto_map\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n\u001B[0;32m   1102\u001B[0m has_local_code \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmodel_type\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m config_dict \u001B[38;5;129;01mand\u001B[39;00m config_dict[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmodel_type\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;129;01min\u001B[39;00m CONFIG_MAPPING\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\configuration_utils.py:634\u001B[0m, in \u001B[0;36mPretrainedConfig.get_config_dict\u001B[1;34m(cls, pretrained_model_name_or_path, **kwargs)\u001B[0m\n\u001B[0;32m    632\u001B[0m original_kwargs \u001B[38;5;241m=\u001B[39m copy\u001B[38;5;241m.\u001B[39mdeepcopy(kwargs)\n\u001B[0;32m    633\u001B[0m \u001B[38;5;66;03m# Get config dict associated with the base config file\u001B[39;00m\n\u001B[1;32m--> 634\u001B[0m config_dict, kwargs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mcls\u001B[39m\u001B[38;5;241m.\u001B[39m_get_config_dict(pretrained_model_name_or_path, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m    635\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01min\u001B[39;00m config_dict:\n\u001B[0;32m    636\u001B[0m     original_kwargs[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m config_dict[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m]\n",
+      "File \u001B[1;32m~\\anaconda3\\envs\\py310\\lib\\site-packages\\transformers\\configuration_utils.py:722\u001B[0m, in \u001B[0;36mPretrainedConfig._get_config_dict\u001B[1;34m(cls, pretrained_model_name_or_path, **kwargs)\u001B[0m\n\u001B[0;32m    720\u001B[0m     config_dict[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m_commit_hash\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m commit_hash\n\u001B[0;32m    721\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m (json\u001B[38;5;241m.\u001B[39mJSONDecodeError, \u001B[38;5;167;01mUnicodeDecodeError\u001B[39;00m):\n\u001B[1;32m--> 722\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mEnvironmentError\u001B[39;00m(\n\u001B[0;32m    723\u001B[0m         \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mIt looks like the config file at \u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mresolved_config_file\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m is not a valid JSON file.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m    724\u001B[0m     )\n\u001B[0;32m    726\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m is_local:\n\u001B[0;32m    727\u001B[0m     logger\u001B[38;5;241m.\u001B[39minfo(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mloading configuration file \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mresolved_config_file\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m)\n",
+      "\u001B[1;31mOSError\u001B[0m: It looks like the config file at 'C:\\Users\\merti\\.cache\\huggingface\\hub\\models--HUBII-Platform--ECG2HRV\\snapshots\\7742915f18da533bfd88c1a49a9e9727635cd1d7\\config.json' is not a valid JSON file."
+     ]
+    }
+   ],
+   "source": [
+    "feature_extractor = pipeline(\"ecg2hrv\", model=\"HUBII-Platform/ECG2HRV\")"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-05T11:41:36.288140800Z",
+     "start_time": "2024-02-05T11:41:35.591183Z"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

pipeline_wrapper.py

@@ -1,5 +1,9 @@
 from transformers import Pipeline
 
+from src.conversion import csv_to_pandas
+from src.pydantic_models import ECGConfig, ECGSample
+from src.ecg_processing import process_batch
+
 
 class MyPipeline(Pipeline):
     def _sanitize_parameters(self, **kwargs):
@@ -8,16 +12,39 @@ class MyPipeline(Pipeline):
             preprocess_kwargs["maybe_arg"] = kwargs["maybe_arg"]
         return preprocess_kwargs, {}, {}
 
-    def preprocess(self, inputs, maybe_arg=2):
-        model_input = Tensor(inputs["input_ids"])
+    def preprocess(self, inputs: str) -> dict:
+        # inputs are csv files
+        df = csv_to_pandas(inputs)
+        # Implode
+        cols_to_implode = ['timestamp_idx', 'ecg', 'label']
+        df_imploded = df.groupby(list(set(df.columns) - set(cols_to_implode))) \
+            .agg({'timestamp_idx': list,
+                  'ecg': list,
+                  'label': list}) \
+            .reset_index()
+        # Get metadata
+        config_cols = [col for col in df.columns if col.startswith('configs.')]
+        configs = df_imploded[config_cols].iloc[0].to_dict()
+        configs = {key.removeprefix('configs.'): value for key, value in configs.items()}
+        configs = ECGConfig(**configs)
+        batch_cols = [col for col in df.columns if col.startswith('batch.')]
+        batch = df_imploded[batch_cols].iloc[0].to_dict()
+        batch = {key.removeprefix('batch.'): value for key, value in batch.items()}
+        # Get samples
+        samples = df_imploded.to_dict(orient='records')
+        samples = [ECGSample(**sample) for sample in samples]
+
+        model_input = {"samples": samples, "configs": configs, "batch": batch}
         return {"model_input": model_input}
 
     def _forward(self, model_inputs):
         # model_inputs == {"model_input": model_input}
-        outputs = self.model(**model_inputs)
-        # Maybe {"logits": Tensor(...)}
-        return outputs
+        samples = model_inputs["model_input"]["samples"]
+        configs = model_inputs["model_input"]["configs"]
+        batch = model_inputs["model_input"]["batch"]
+
+        features_df = process_batch(samples, configs)
+        return features_df
 
     def postprocess(self, model_outputs):
-        best_class = model_outputs["logits"].softmax(-1)
-        return best_class
+        return model_outputs