use api instead of duckdb on dataset

package.json

@@ -9,7 +9,6 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.28.1-dev106.0",
     "@emotion/react": "^11.13.0",
     "@emotion/styled": "^11.13.0",
     "@mui/material": "^5.16.6",

src/pages/index.tsx

@@ -2,145 +2,107 @@
 
 import { Inter } from "next/font/google";
 import ActivityCalendar from "react-activity-calendar";
-import * as duckdb from "@duckdb/duckdb-wasm"
 import { useState, useEffect } from "react";
 import { Tooltip as MuiTooltip } from '@mui/material';
 
 const inter = Inter({ subsets: ["latin"] });
 
-interface CustomActivity {
+interface ModelData {
+  createdAt: string;
+  id: string;
+}
+
+interface Activity {
   date: string;
   count: number;
   level: number;
-  details: Array<{ provider: string; count: number }>;
 }
 
-type ProviderKey = "mistralai" | "meta-llama" | "openai" | "anthropic" | "google";
-
 export default function Home() {
-  const [calendarData, setCalendarData] = useState<Record<ProviderKey, CustomActivity[]>>({} as Record<ProviderKey, CustomActivity[]>);
+  const [calendarData, setCalendarData] = useState<Record<string, Activity[]>>({});
   const [isLoading, setIsLoading] = useState(true);
 
-  const PROVIDERS_MAP: Record<ProviderKey, { name: string; color: string }> = {
-    "mistralai": {"name": "Mistral AI", "color": "#ff7000"},
-    "meta-llama": {"name": "Meta", "color": "#0668E1"},
-    "openai": {"name": "OpenAI", "color": "#10A37F"},
-    "anthropic": {"name": "Anthropic", "color": "#cc785c"},
-    "google": {"name": "Google", "color": "#4285F4"},
+  const PROVIDERS_MAP: Record<string, { color: string; authors: string[] }> = {
+    "Mistral AI": { color: "#ff7000", authors: ["mistralai"] },
+    "Meta": { color: "#0668E1", authors: ["facebook", "meta-llama"] },
+    "OpenAI": { color: "#10A37F", authors: ["openai"] },
+    "Anthropic": { color: "#cc785c", authors: ["anthropic"] },
+    "Google": { color: "#4285F4", authors: ["google"] },
   }
 
-  const getModelData = async (conn: duckdb.AsyncDuckDBConnection) => {
-    const result = await conn.query(`
-      SELECT 
-        STRFTIME(DATE_TRUNC('day', CAST(createdAt AS DATE)), '%Y-%m-%d') AS date,
-        CASE 
-          WHEN provider IN ('meta-llama', 'facebook') THEN 'meta-llama'
-          ELSE provider
-        END AS provider,
-        COUNT(*) AS count
-      FROM (
-        SELECT *, SPLIT_PART(id, '/', 1) AS provider
-        FROM models
-        WHERE CAST(createdAt AS DATE) >= DATE_TRUNC('year', CURRENT_DATE)
-      ) subquery
-      WHERE provider IN (${Object.keys(PROVIDERS_MAP).map(p => `'${p}'`).join(', ')}, 'facebook')
-      GROUP BY 1, 2
-      ORDER BY date
-    `);
-    return result.toArray().map((row: any) => ({
-      date: row.date,
-      provider: row.provider,
-      count: Number(row.count)
-    }));
-  }
-
-  const generateCalendarData = (modelData: any[]) => {
-    const data: Record<ProviderKey, CustomActivity[]> = Object.keys(PROVIDERS_MAP).reduce((acc, provider) => {
-      acc[provider as ProviderKey] = [];
-      return acc;
-    }, {} as Record<ProviderKey, CustomActivity[]>);
+  const generateCalendarData = (modelData: ModelData[]) => {
+    const data: Record<string, Activity[]> = Object.fromEntries(
+      Object.keys(PROVIDERS_MAP).map(provider => [provider, []])
+    );
 
     const today = new Date();
     const startOfYear = new Date(today.getFullYear(), 0, 1);
 
+    // generate daily data for each provider
     for (let d = new Date(startOfYear); d <= today; d.setDate(d.getDate() + 1)) {
       const dateString = d.toISOString().split('T')[0];
       
-      Object.keys(PROVIDERS_MAP).forEach((provider) => {
-        const dayData = modelData.filter(item => item.date === dateString && item.provider === provider);
-        const count = dayData.reduce((sum, item) => sum + item.count, 0);
+      Object.entries(PROVIDERS_MAP).forEach(([provider, { authors }]) => {
+        const count = modelData.filter(item => 
+          item.createdAt.startsWith(dateString) && 
+          authors.some(author => item.id.startsWith(author))
+        ).length;
         
-        data[provider as ProviderKey].push({
-          date: dateString,
-          count,
-          level: 0,   
-          details: dayData,
-        });
+        data[provider].push({ date: dateString, count, level: 0 });
       });
     }
 
-    const avgCounts: Record<ProviderKey, number> = Object.fromEntries(
-      Object.keys(PROVIDERS_MAP).map(provider => [
+    // calculate average counts for each provider
+    const avgCounts = Object.fromEntries(
+      Object.entries(data).map(([provider, days]) => [
         provider,
-        data[provider as ProviderKey].reduce((sum, day) => sum + day.count, 0) / data[provider as ProviderKey].length || 0
+        days.reduce((sum, day) => sum + day.count, 0) / days.length || 0
       ])
-    ) as Record<ProviderKey, number>;
+    );
 
+    // assign levels based on count relative to average
     Object.entries(data).forEach(([provider, days]) => {
-      const avgCount = avgCounts[provider as ProviderKey];
+      const avgCount = avgCounts[provider];
       days.forEach(day => {
-        day.level = day.count === 0 ? 0 :
-                    day.count <= avgCount * 0.5 ? 1 :
-                    day.count <= avgCount ? 2 :
-                    day.count <= avgCount * 1.5 ? 3 : 4;
+        day.level = 
+          day.count === 0 ? 0 :
+          day.count <= avgCount * 0.5 ? 1 :
+          day.count <= avgCount ? 2 :
+          day.count <= avgCount * 1.5 ? 3 : 4;
       });
     });
 
     return data;
   }
 
-  const initDB = async () => {
-    const CDN_BASE = `https://cdn.jsdelivr.net/npm/@duckdb/duckdb-wasm@next`
-
-    const JSDELIVR_BUNDLES = {
-      mvp: {
-        mainModule: `${CDN_BASE}/dist/duckdb-mvp.wasm`,
-        mainWorker: `${CDN_BASE}/dist/duckdb-browser-mvp.worker.js`,
-      },
-      eh: {
-        mainModule: `${CDN_BASE}/dist/duckdb-eh.wasm`,
-        mainWorker: `${CDN_BASE}/dist/duckdb-browser-eh.worker.js`,
-      },
+  const initData = async () => {
+    try {
+      const allAuthors = Object.values(PROVIDERS_MAP).flatMap(({ authors }) => authors);
+      const uniqueAuthors = Array.from(new Set(allAuthors));
+
+      const allModelData = await Promise.all(
+        uniqueAuthors.map(async (author) => {
+          const response = await fetch(`https://huggingface.co/api/models?author=${author}&sort=createdAt&direction=-1`);
+          const data = await response.json();
+          return data.map((item: any) => ({
+            createdAt: item.createdAt,
+            id: item.id,
+          }));
+        })
+      );
+
+      const flatModelData = allModelData.flat();
+      const calendarData = generateCalendarData(flatModelData);
+      setCalendarData(calendarData);
+    } catch (error) {
+      console.error("Error fetching data:", error);
+    } finally {
+      setIsLoading(false);
     }
-
-    const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES)
-    const worker_url = URL.createObjectURL(
-      new Blob([`importScripts("${bundle.mainWorker}");`], {
-        type: "text/javascript",
-      })
-    )
-
-    const worker = new Worker(worker_url)
-    const logger = new duckdb.ConsoleLogger()
-    const db = new duckdb.AsyncDuckDB(logger, worker)
-    await db.instantiate(bundle.mainModule)
-
-    const connection = await db.connect()
-
-    await connection.query(`
-        CREATE VIEW models AS SELECT * FROM read_parquet('https://huggingface.co/datasets/cfahlgren1/hub-stats/resolve/refs%2Fconvert%2Fparquet/models/train/0000.parquet?download=true');
-    `);
-
-    const modelData = await getModelData(connection);
-    const calendarData = generateCalendarData(modelData);
-    setCalendarData(calendarData);
-    setIsLoading(false);
-
-    await connection.close();
   }
 
   useEffect(() => {
-    initDB();
+    initData();
   }, []);
 
   return (
@@ -154,21 +116,22 @@ export default function Home() {
           <>
             {Object.entries(PROVIDERS_MAP)
               .sort(([keyA], [keyB]) => 
-                calendarData[keyB as ProviderKey].reduce((sum, day) => sum + day.count, 0) -
-                calendarData[keyA as ProviderKey].reduce((sum, day) => sum + day.count, 0)
+                calendarData[keyB].reduce((sum, day) => sum + day.count, 0) -
+                calendarData[keyA].reduce((sum, day) => sum + day.count, 0)
               )
-              .map(([key, value]) => (
-                <div key={key} className="mb-8">
-                  <h2 className="text-2xl font-bold mb-2">{value.name}</h2>
+              .map(([providerName, { color }]) => (
+                <div key={providerName} className="mb-8">
+                  <h2 className="text-2xl font-bold mb-2">{providerName}</h2>
                   <ActivityCalendar 
-                    data={calendarData[key as ProviderKey]}
+                    data={calendarData[providerName]}
                     theme={{
-                      dark: ['#161b22', value.color],
+                      dark: ['#161b22', color],
                     }}
+                    hideTotalCount
                     colorScheme="dark"
                     renderBlock={(block, activity) => (
                       <MuiTooltip
-                        title={`${activity.count} activities on ${activity.date}`}
+                        title={`${activity.count} models created on ${activity.date}`}
                       >
                         {block}
                       </MuiTooltip>