feat: implement codebase and data management system with HF integration

- Created /manage interface for switching code versions and restoring data backups.
- Integrated Hugging Face Dataset API to list backup snapshots and restore specific revisions.
- Enhanced hf_sync.py with a default repo (Jaimodiji/Report-Generator-Data) and simulation mode via RPT_DEBUG.
- Added data protection: automated backups before any code switch or data restoration.
- Updated entrypoint.sh to support default repo ID and debug simulation.
- Added 'Manage Codebase' link to navbar dropdown.

entrypoint.sh

@@ -7,13 +7,11 @@ cd /app
 python3 hf_sync.py init
 
 # 2. Try to download existing data
-if [ -n "$DATASET_REPO_ID" ]; then
-    python3 hf_sync.py download
-else
-    echo "DATASET_REPO_ID not set, skipping initial download."
-fi
+# REPO_ID defaults to Jaimodiji/Report-Generator-Data in hf_sync.py
+python3 hf_sync.py download
 
 # 3. Setup symlinks
+# ... (rest of symlink setup)
 # Remove existing if any
 rm -rf database.db output processed uploads
 
@@ -27,16 +25,22 @@ ln -sf data_repo/processed processed
 ln -sf data_repo/uploads uploads
 
 # 4. Start periodic background upload
-if [ -n "$DATASET_REPO_ID" ] && [ -n "$HF_TOKEN" ]; then
+if [ -n "$HF_TOKEN" ]; then
     (
         while true; do
             sleep 120 # Every 2 minutes
-            echo "Performing scheduled backup to HF Datasets..."
+            if [[ "${RPT_DEBUG,,}" == "true" || "${RPT_DEBUG}" == "1" || "${RPT_DEBUG,,}" == "t" ]]; then
+                echo "[DEBUG] Simulating scheduled backup to HF Datasets..."
+            else
+                echo "Performing scheduled backup to HF Datasets..."
+            fi
+            # REPO_ID defaults to Jaimodiji/Report-Generator-Data in hf_sync.py
+            # hf_sync.py also respects RPT_DEBUG internally
             python3 hf_sync.py upload
         done
     ) &
 else
-    echo "DATASET_REPO_ID or HF_TOKEN not set, periodic backup disabled."
+    echo "HF_TOKEN not set, periodic backup disabled."
 fi
 
 # 5. Start the application

hf_sync.py

@@ -9,8 +9,9 @@ from datetime import datetime
 from huggingface_hub import snapshot_download, HfApi
 
 # Configuration
-REPO_ID = os.environ.get("DATASET_REPO_ID")
+REPO_ID = os.environ.get("DATASET_REPO_ID", "Jaimodiji/Report-Generator-Data")
 HF_TOKEN = os.environ.get("HF_TOKEN")
+DEBUG_MODE = os.environ.get("RPT_DEBUG", "false").lower() in ("true", "1", "t")
 DATA_DIR = "data_repo"
 DB_FILE = os.path.join(DATA_DIR, "database.db")
 STATE_FILE = os.path.join(DATA_DIR, "sync_state.json")
@@ -66,9 +67,12 @@ def upload():
         if backup_path:
             db_hash = get_file_hash(backup_path)
             if db_hash != state.get("last_db_hash"):
-                print("Syncing Database...")
-                # Upload the backup file directly without replacing the active database
-                api.upload_file(path_or_fileobj=backup_path, path_in_repo="database.db", repo_id=REPO_ID, repo_type="dataset")
+                print(f"{'[DEBUG] ' if DEBUG_MODE else ''}Syncing Database...")
+                if not DEBUG_MODE:
+                    # Upload the backup file directly without replacing the active database
+                    api.upload_file(path_or_fileobj=backup_path, path_in_repo="database.db", repo_id=REPO_ID, repo_type="dataset")
+                else:
+                    print(f"[DEBUG] Simulated upload of {backup_path} to {REPO_ID}/database.db")
                 state["last_db_hash"] = db_hash
                 changes_made = True
             # Clean up the backup file regardless
@@ -91,14 +95,17 @@ def upload():
                     file_id = f"{rel_path}_{size}_{mtime}"
                     
                     if state["uploaded_files"].get(rel_path) != file_id:
-                        print(f"Syncing new file: {rel_path}")
+                        print(f"{'[DEBUG] ' if DEBUG_MODE else ''}Syncing new file: {rel_path}")
                         try:
-                            api.upload_file(
-                                path_or_fileobj=full_path,
-                                path_in_repo=rel_path,
-                                repo_id=REPO_ID,
-                                repo_type="dataset"
-                            )
+                            if not DEBUG_MODE:
+                                api.upload_file(
+                                    path_or_fileobj=full_path,
+                                    path_in_repo=rel_path,
+                                    repo_id=REPO_ID,
+                                    repo_type="dataset"
+                                )
+                            else:
+                                print(f"[DEBUG] Simulated upload of {rel_path} to {REPO_ID}/{rel_path}")
                             state["uploaded_files"][rel_path] = file_id
                             changes_made = True
                         except Exception as e:
@@ -108,20 +115,31 @@ def upload():
             state["version"] += 1
             save_state(state)
             # Sync state file too
-            api.upload_file(path_or_fileobj=STATE_FILE, path_in_repo="sync_state.json", repo_id=REPO_ID, repo_type="dataset")
-            print(f"Sync complete. Version {state['version']} saved.")
+            if not DEBUG_MODE:
+                api.upload_file(path_or_fileobj=STATE_FILE, path_in_repo="sync_state.json", repo_id=REPO_ID, repo_type="dataset")
+            else:
+                print(f"[DEBUG] Simulated upload of {STATE_FILE} to {REPO_ID}/sync_state.json")
+            print(f"{'[DEBUG] ' if DEBUG_MODE else ''}Sync complete. Version {state['version']} saved.")
         else:
-            print("Everything up to date.")
+            print(f"{'[DEBUG] ' if DEBUG_MODE else ''}Everything up to date.")
 
     except Exception as e: print(f"Upload process failed: {e}")
     finally:
         if os.path.exists(LOCK_FILE): os.remove(LOCK_FILE)
 
-def download():
+def download(revision=None):
     if not REPO_ID: return
-    print(f"Downloading data from {REPO_ID}...")
+    rev_str = f" (revision: {revision})" if revision else ""
+    print(f"Downloading data from {REPO_ID}{rev_str}...")
     try:
-        snapshot_download(repo_id=REPO_ID, repo_type="dataset", local_dir=DATA_DIR, token=HF_TOKEN, max_workers=8)
+        snapshot_download(
+            repo_id=REPO_ID, 
+            repo_type="dataset", 
+            local_dir=DATA_DIR, 
+            token=HF_TOKEN, 
+            revision=revision,
+            max_workers=8
+        )
         print("Download successful.")
     except Exception as e: print(f"Download failed: {e}")
 
@@ -130,7 +148,9 @@ def init_local():
 
 if __name__ == "__main__":
     action = sys.argv[1] if len(sys.argv) > 1 else "help"
-    if action == "download": download()
+    if action == "download":
+        revision = sys.argv[2] if len(sys.argv) > 2 else None
+        download(revision)
     elif action == "upload": upload()
     elif action == "init": init_local()
-    else: print("Usage: python hf_sync.py [download|upload|init]")
+    else: print("Usage: python hf_sync.py [download [revision]|upload|init]")

requirements.txt

@@ -23,3 +23,4 @@ jsonschema
 gdown
 google-api-python-client
 google-auth-oauthlib
+huggingface_hub

routes/general.py

@@ -1,8 +1,134 @@
 import os
-from flask import jsonify, render_template, redirect, url_for, current_app, request
+import subprocess
+from flask import jsonify, render_template, redirect, url_for, current_app, request, flash
 from .common import main_bp, get_db_connection, login_required, current_user, upload_progress
 from strings import ROUTE_INDEX, METHOD_DELETE, METHOD_POST
+from huggingface_hub import HfApi
 
+def get_hf_backup_info():
+    try:
+        repo_id = os.environ.get("DATASET_REPO_ID", "Jaimodiji/Report-Generator-Data")
+        token = os.environ.get("HF_TOKEN")
+        if not token:
+            return [], repo_id
+            
+        api = HfApi(token=token)
+        commits = api.list_repo_commits(repo_id=repo_id, repo_type="dataset")
+        
+        hf_commits = []
+        for c in commits[:20]: # Last 20 backups
+            hf_commits.append({
+                'hash': c.commit_id,
+                'author': c.authors[0] if c.authors else "Unknown",
+                'date': c.created_at.strftime('%Y-%m-%d %H:%M:%S'),
+                'message': c.title
+            })
+        return hf_commits, repo_id
+    except Exception as e:
+        current_app.logger.error(f"HF backup info error: {e}")
+        return [], os.environ.get("DATASET_REPO_ID", "Jaimodiji/Report-Generator-Data")
+
+def get_git_info():
+    try:
+        # Get last 20 commits
+        log_output = subprocess.check_output(
+            ['git', 'log', '-n', '20', '--pretty=format:%h|%an|%ar|%s'],
+            stderr=subprocess.STDOUT, text=True
+        )
+        commits = []
+        for line in log_output.split('\n'):
+            if not line: continue
+            h, an, ar, s = line.split('|', 3)
+            commits.append({'hash': h, 'author': an, 'date': ar, 'message': s})
+        
+        # Get current hash and branch
+        current_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], text=True).strip()
+        try:
+            current_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], text=True).strip()
+        except:
+            current_branch = "Detached"
+            
+        # Check for uncommitted changes
+        status_output = subprocess.check_output(['git', 'status', '--porcelain'], text=True).strip()
+        uncommitted = bool(status_output)
+        
+        return commits, current_hash, current_branch, uncommitted
+    except Exception as e:
+        current_app.logger.error(f"Git info error: {e}")
+        return [], "Unknown", "Unknown", False
+
+@main_bp.route('/manage')
+@login_required
+def manage():
+    code_commits, current_hash, current_branch, uncommitted = get_git_info()
+    hf_commits, hf_repo_id = get_hf_backup_info()
+    return render_template('manage.html', 
+                         code_commits=code_commits, 
+                         hf_commits=hf_commits,
+                         current_hash=current_hash, 
+                         current_branch=current_branch, 
+                         uncommitted_changes=uncommitted,
+                         hf_repo_id=hf_repo_id)
+
+@main_bp.route('/manage/checkout', methods=['POST'])
+@login_required
+def manage_checkout():
+    target_hash = request.form.get('hash')
+    if not target_hash:
+        flash("No commit hash provided", "danger")
+        return redirect(url_for('main.manage'))
+
+    try:
+        # 1. Protection: Trigger immediate data backup before code change
+        current_app.logger.info(f"Triggering backup before switch to {target_hash}")
+        subprocess.run(['python3', 'hf_sync.py', 'upload'], check=True)
+
+        # 2. Protection: Stash uncommitted code changes
+        subprocess.run(['git', 'stash'], check=False)
+
+        # 3. Checkout target code commit
+        current_app.logger.info(f"Checking out code commit {target_hash}")
+        subprocess.run(['git', 'checkout', target_hash], check=True)
+
+        flash(f"Successfully switched code to {target_hash}. Application will restart.", "success")
+    except Exception as e:
+        current_app.logger.error(f"Checkout error: {e}")
+        flash(f"Error during code switch: {str(e)}", "danger")
+
+    return redirect(url_for('main.manage'))
+
+@main_bp.route('/manage/restore', methods=['POST'])
+@login_required
+def manage_restore():
+    target_hash = request.form.get('hash')
+    if not target_hash:
+        flash("No backup hash provided", "danger")
+        return redirect(url_for('main.manage'))
+
+    try:
+        # 1. Protection: Backup current state before restoring old one
+        current_app.logger.info(f"Triggering safety backup before restoring {target_hash}")
+        subprocess.run(['python3', 'hf_sync.py', 'upload'], check=True)
+
+        # 2. Restore data from HF at specific revision
+        current_app.logger.info(f"Restoring data from HF revision {target_hash}")
+        subprocess.run(['python3', 'hf_sync.py', 'download', target_hash], check=True)
+
+        flash(f"Successfully restored data from backup {target_hash[:8]}. Application will restart with old data.", "success")
+    except Exception as e:
+        current_app.logger.error(f"Restore error: {e}")
+        flash(f"Error during data restoration: {str(e)}", "danger")
+
+    return redirect(url_for('main.manage'))
+
+@main_bp.route('/manage/backup', methods=['POST'])
+@login_required
+def manage_backup():
+    try:
+        subprocess.run(['python3', 'hf_sync.py', 'upload'], check=True)
+        return jsonify({'success': True})
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
 @main_bp.route(ROUTE_INDEX)
 def index():
     if current_user.is_authenticated: return redirect(url_for('dashboard.dashboard'))

templates/_nav_links.html

@@ -33,6 +33,7 @@
             </a>
             <ul class="dropdown-menu dropdown-menu-end" aria-labelledby="navbarDropdown">
                 <li><a class="dropdown-item" href="{{ url_for('settings.settings') }}">Settings</a></li>
+                <li><a class="dropdown-item" href="{{ url_for('main.manage') }}"><i class="bi bi-git me-1"></i> Manage Codebase</a></li>
                 <li><hr class="dropdown-divider"></li>
                 <li><a class="dropdown-item" href="{{ url_for('auth.logout') }}">Logout</a></li>
             </ul>

templates/manage.html

@@ -0,0 +1,252 @@
+{% extends "base.html" %}
+
+{% block title %}Manage Codebase & Data{% endblock %}
+
+{% block head %}
+<style>
+    .commit-row { cursor: pointer; transition: background 0.2s; }
+    .commit-row:hover { background: rgba(255,255,255,0.05); }
+    .commit-row.current { border-left: 4px solid var(--accent-success); background: rgba(25, 135, 84, 0.1); }
+    .hash { font-family: monospace; color: var(--accent-info); font-weight: bold; }
+    .author { font-size: 0.85rem; color: var(--text-muted); }
+    .date { font-size: 0.85rem; color: var(--text-muted); }
+    .status-card { border-radius: 12px; border: 1px solid var(--border-subtle); margin-bottom: 20px; }
+    .nav-pills .nav-link { color: var(--text-muted); }
+    .nav-pills .nav-link.active { background-color: var(--accent-primary); color: white; }
+</style>
+{% endblock %}
+
+{% block content %}
+<div class="container py-4">
+    <div class="d-flex justify-content-between align-items-center mb-4">
+        <h2><i class="bi bi-gear-wide-connected me-2"></i>System Management</h2>
+        <div class="d-flex gap-2">
+            <button id="btn-backup-now" class="btn btn-outline-info">
+                <i class="bi bi-cloud-upload me-1"></i> Backup Data Now
+            </button>
+            <a href="{{ url_for('dashboard.dashboard') }}" class="btn btn-secondary">
+                <i class="bi bi-arrow-left me-1"></i> Back to Dashboard
+            </a>
+        </div>
+    </div>
+
+    <!-- Status Section -->
+    <div class="card status-card bg-dark text-white shadow-sm">
+        <div class="card-body">
+            <div class="row">
+                <div class="col-md-6 border-end border-secondary">
+                    <h6 class="text-info mb-3"><i class="bi bi-code-slash me-2"></i>Code Status</h6>
+                    <p class="mb-1 small">Branch: <span class="badge bg-primary">{{ current_branch }}</span></p>
+                    <p class="mb-0 small">Active Commit: <span class="hash">{{ current_hash }}</span></p>
+                    
+                    {% if uncommitted_changes %}
+                    <div class="text-warning mt-2 small">
+                        <i class="bi bi-exclamation-triangle-fill me-1"></i>
+                        Uncommitted code changes detected.
+                    </div>
+                    {% endif %}
+                </div>
+                <div class="col-md-6 ps-md-4">
+                    <h6 class="text-success mb-3"><i class="bi bi-database me-2"></i>Data Status (Hugging Face)</h6>
+                    <p class="mb-1 small">Repo: <code class="text-light">{{ hf_repo_id }}</code></p>
+                    <div id="backup-info" class="small text-muted">
+                        Auto-backup syncs every 2 minutes.
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+
+    <!-- Navigation Tabs -->
+    <ul class="nav nav-pills mb-3" id="manageTabs" role="tablist">
+        <li class="nav-item" role="presentation">
+            <button class="nav-link active" id="code-tab" data-bs-toggle="pill" data-bs-target="#code-pane" type="button" role="tab">
+                <i class="bi bi-git me-1"></i> Code Updates
+            </button>
+        </li>
+        <li class="nav-item" role="presentation">
+            <button class="nav-link" id="data-tab" data-bs-toggle="pill" data-bs-target="#data-pane" type="button" role="tab">
+                <i class="bi bi-cloud-check me-1"></i> Data Backups
+            </button>
+        </li>
+    </ul>
+
+    <div class="tab-content" id="manageTabsContent">
+        <!-- Code Tab -->
+        <div class="tab-pane fade show active" id="code-pane" role="tabpanel">
+            <div class="card bg-dark text-white border-secondary">
+                <div class="table-responsive">
+                    <table class="table table-dark table-hover mb-0">
+                        <thead>
+                            <tr>
+                                <th>Hash</th>
+                                <th>Message</th>
+                                <th>Author</th>
+                                <th>When</th>
+                                <th>Action</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for commit in code_commits %}
+                            <tr class="commit-row {{ 'current' if commit.hash == current_hash else '' }}">
+                                <td><span class="hash">{{ commit.hash }}</span></td>
+                                <td>{{ commit.message }}</td>
+                                <td><span class="author">{{ commit.author }}</span></td>
+                                <td><span class="date">{{ commit.date }}</span></td>
+                                <td>
+                                    {% if commit.hash != current_hash %}
+                                    <button class="btn btn-sm btn-outline-warning btn-switch-code" data-hash="{{ commit.hash }}">
+                                        Switch
+                                    </button>
+                                    {% else %}
+                                    <span class="badge bg-success">Active</span>
+                                    {% endif %}
+                                </td>
+                            </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+
+        <!-- Data Tab -->
+        <div class="tab-pane fade" id="data-pane" role="tabpanel">
+            <div class="card bg-dark text-white border-secondary">
+                <div class="table-responsive">
+                    <table class="table table-dark table-hover mb-0">
+                        <thead>
+                            <tr>
+                                <th>HF Hash</th>
+                                <th>Description</th>
+                                <th>Timestamp</th>
+                                <th>Action</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for commit in hf_commits %}
+                            <tr class="commit-row">
+                                <td><span class="hash text-success">{{ commit.hash[:8] }}</span></td>
+                                <td>{{ commit.message }}</td>
+                                <td><span class="date">{{ commit.date }}</span></td>
+                                <td>
+                                    <button class="btn btn-sm btn-outline-danger btn-restore-data" data-hash="{{ commit.hash }}">
+                                        Restore
+                                    </button>
+                                </td>
+                            </tr>
+                            {% endfor %}
+                            {% if not hf_commits %}
+                            <tr>
+                                <td colspan="4" class="text-center py-4 text-muted">
+                                    No Hugging Face backups found or HF_TOKEN not set.
+                                </td>
+                            </tr>
+                            {% endif %}
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<!-- Modal: Switch Code -->
+<div class="modal fade" id="switchCodeModal" tabindex="-1">
+    <div class="modal-dialog modal-dialog-centered">
+        <div class="modal-content bg-dark text-white border-warning">
+            <div class="modal-header border-secondary">
+                <h5 class="modal-title text-warning">Switch Application Version</h5>
+                <button type="button" class="btn-close btn-close-white" data-bs-dismiss="modal"></button>
+            </div>
+            <div class="modal-body">
+                <p>Switching code to commit <span id="target-code-hash" class="hash"></span>.</p>
+                <div class="alert alert-info py-2 small">
+                    <i class="bi bi-shield-check me-2"></i>
+                    <strong>Safety:</strong> We'll backup your data to HF and stash any uncommitted code before switching.
+                </div>
+            </div>
+            <div class="modal-footer border-secondary">
+                <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                <form action="{{ url_for('main.manage_checkout') }}" method="POST">
+                    <input type="hidden" name="hash" id="form-code-hash">
+                    <button type="submit" class="btn btn-warning">Backup & Switch Code</button>
+                </form>
+            </div>
+        </div>
+    </div>
+</div>
+
+<!-- Modal: Restore Data -->
+<div class="modal fade" id="restoreDataModal" tabindex="-1">
+    <div class="modal-dialog modal-dialog-centered">
+        <div class="modal-content bg-dark text-white border-danger">
+            <div class="modal-header border-secondary">
+                <h5 class="modal-title text-danger">Restore Database & Images</h5>
+                <button type="button" class="btn-close btn-close-white" data-bs-dismiss="modal"></button>
+            </div>
+            <div class="modal-body">
+                <p>Restoring data from HF snapshot <span id="target-data-hash" class="hash"></span>.</p>
+                <div class="alert alert-danger py-2 small">
+                    <i class="bi bi-exclamation-octagon-fill me-2"></i>
+                    <strong>Warning:</strong> This will replace your current database and images with this older version.
+                </div>
+                <div class="alert alert-success py-2 small">
+                    <i class="bi bi-shield-lock me-2"></i>
+                    <strong>Protection:</strong> We'll create a safety backup of your CURRENT state before restoring.
+                </div>
+            </div>
+            <div class="modal-footer border-secondary">
+                <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                <form action="{{ url_for('main.manage_restore') }}" method="POST">
+                    <input type="hidden" name="hash" id="form-data-hash">
+                    <button type="submit" class="btn btn-danger">Backup Current & Restore Old</button>
+                </form>
+            </div>
+        </div>
+    </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script>
+$(document).ready(function() {
+    const switchCodeModal = new bootstrap.Modal($('#switchCodeModal'));
+    const restoreDataModal = new bootstrap.Modal($('#restoreDataModal'));
+    
+    $('.btn-switch-code').on('click', function() {
+        const hash = $(this).data('hash');
+        $('#target-code-hash').text(hash);
+        $('#form-code-hash').val(hash);
+        switchCodeModal.show();
+    });
+
+    $('.btn-restore-data').on('click', function() {
+        const hash = $(this).data('hash');
+        $('#target-data-hash').text(hash.substring(0, 8));
+        $('#form-data-hash').val(hash);
+        restoreDataModal.show();
+    });
+
+    $('#btn-backup-now').on('click', function() {
+        const btn = $(this);
+        const originalHtml = btn.html();
+        btn.prop('disabled', true).html('<span class="spinner-border spinner-border-sm me-1"></span> Uploading...');
+        
+        $.ajax({
+            url: '/manage/backup',
+            type: 'POST',
+            success: function() {
+                btn.html('<i class="bi bi-check-lg me-1"></i> Success');
+                setTimeout(() => { btn.prop('disabled', false).html(originalHtml); }, 3000);
+            },
+            error: function() {
+                btn.html('<i class="bi bi-x-lg me-1"></i> Failed');
+                btn.addClass('btn-outline-danger').removeClass('btn-outline-info');
+                setTimeout(() => { btn.prop('disabled', false).html(originalHtml).removeClass('btn-outline-danger').addClass('btn-outline-info'); }, 3000);
+            }
+        });
+    });
+});
+</script>
+{% endblock %}