Create app.py

app.py

@@ -0,0 +1,131 @@
+
+Hugging Face's logo Hugging Face
+
+Models
+Datasets
+Spaces
+Docs
+Pricing
+Spaces:
+huggingface-projects
+/
+sd-multiplayer-bot
+private
+App
+Files and versions
+Community
+Settings
+sd-multiplayer-bot
+/ app.py
+radames's picture
+radames
+HF staff
+add uuid from files
+f6a107f
+unverified
+about 2 months ago
+raw
+history
+blame
+contribute
+delete
+3.65 kB
+import boto3
+import os
+import re
+import json
+from pathlib import Path
+import sqlite3
+from huggingface_hub import Repository, HfFolder
+import tqdm
+import subprocess
+
+from fastapi import FastAPI
+from fastapi_utils.tasks import repeat_every
+
+
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_KEY = os.getenv('AWS_SECRET_KEY')
+AWS_S3_BUCKET_NAME = os.getenv('AWS_S3_BUCKET_NAME')
+
+s3 = boto3.client(service_name='s3',
+                  aws_access_key_id=AWS_ACCESS_KEY_ID,
+                  aws_secret_access_key=AWS_SECRET_KEY)
+
+
+paginator = s3.get_paginator('list_objects_v2')
+
+
+S3_DATA_FOLDER = Path("sd-multiplayer-data")
+ROOMS_DATA_DB = S3_DATA_FOLDER / "rooms_data.db"
+
+
+repo = Repository(
+    local_dir=S3_DATA_FOLDER,
+    repo_type="dataset",
+    clone_from="huggingface-projects/sd-multiplayer-data",
+    use_auth_token=True,
+)
+repo.git_pull()
+
+
+if not ROOMS_DATA_DB.exists():
+    print("Creating database")
+    print("ROOMS_DATA_DB", ROOMS_DATA_DB)
+    db = sqlite3.connect(ROOMS_DATA_DB)
+    with open(Path("schema.sql"), "r") as f:
+        db.executescript(f.read())
+    db.commit()
+    db.close()
+
+
+def get_db(db_path):
+    db = sqlite3.connect(db_path, check_same_thread=False)
+    db.row_factory = sqlite3.Row
+    try:
+        yield db
+    except Exception:
+        db.rollback()
+    finally:
+        db.close()
+
+
+def sync_rooms_to_dataset():
+    for room_data_db in get_db(ROOMS_DATA_DB):
+
+                    except Exception as e:
+                        print(e)
+                        continue
+
+                objects += results
+            room_data_db.commit()
+
+            all_rows = [dict(row) for row in room_data_db.execute(
+                "SELECT * FROM rooms_data WHERE room_id = ?", (room_id,)).fetchall()]
+            data_path = S3_DATA_FOLDER / f"{room_id}.json"
+            with open(data_path, 'w') as f:
+                json.dump(all_rows, f, separators=(',', ':'))
+    print("Updating repository")
+    subprocess.Popen(
+        "git add . && git commit --amend -m 'update' && git push --force", cwd=S3_DATA_FOLDER, shell=True)
+
+
+app = FastAPI()
+
+
+@app.post("/")
+def read_root():
+    return "Just a bot to sync data to huggingface datasets and tweet tha latest data"
+
+
+@app.post("/sync")
+def sync():
+    sync_rooms_to_dataset()
+    return "Synced data to huggingface datasets"
+
+
+@app.on_event("startup")
+@repeat_every(seconds=1800)
+def repeat_sync():
+    sync_rooms_to_dataset()
+    return "Synced data to huggingface datasets"