work in progress

README.md

@@ -1,5 +1,5 @@
 ---
-title: AI Bedtime Story
+title: AI Bedtime Story 🛏️
 emoji: 🌙
 colorFrom: yellow
 colorTo: gray
@@ -8,6 +8,6 @@ pinned: true
 app_port: 3000
 ---
 
-# AI Bedtime Story
+# 🌜 AI Bedtime Story 🛏️
 
 (To be continued)

package-lock.json

@@ -50,6 +50,7 @@
         "react-virtualized-auto-sizer": "^1.0.20",
         "replicate": "^0.17.0",
         "sbd": "^1.0.19",
+        "sentence-splitter": "^4.3.0",
         "sharp": "^0.32.5",
         "styled-components": "^6.0.7",
         "tailwind-merge": "^1.13.2",
@@ -1691,6 +1692,11 @@
         "tslib": "^2.4.0"
       }
     },
+    "node_modules/@textlint/ast-node-types": {
+      "version": "13.4.0",
+      "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-13.4.0.tgz",
+      "integrity": "sha512-roVeLjnf8UPntFICb1uEwE2dccC8V/T5N1x7eBxkT3VDmSQkyfIAuGtlpwyH0wNKEwJmjO/2gSm2fCjW5K/rbA=="
+    },
     "node_modules/@tsconfig/node10": {
       "version": "1.0.9",
       "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz",
@@ -2248,6 +2254,11 @@
         "readable-stream": "^3.4.0"
       }
     },
+    "node_modules/boundary": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz",
+      "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA=="
+    },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -5815,6 +5826,15 @@
         "node": ">=10"
       }
     },
+    "node_modules/sentence-splitter": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/sentence-splitter/-/sentence-splitter-4.3.0.tgz",
+      "integrity": "sha512-srJOMqv7JeEmsbVa/N64ULey2N6/OuZzeKWn2Zrj0DiTBlU930JGr/rKKlKQRigzXtLMOtl32/Gm5G3HW8/ULA==",
+      "dependencies": {
+        "@textlint/ast-node-types": "^13.2.0",
+        "structured-source": "^4.0.0"
+      }
+    },
     "node_modules/set-function-length": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz",
@@ -6084,6 +6104,14 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/structured-source": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz",
+      "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==",
+      "dependencies": {
+        "boundary": "^2.0.0"
+      }
+    },
     "node_modules/styled-components": {
       "version": "6.1.1",
       "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-6.1.1.tgz",

package.json

@@ -51,6 +51,7 @@
     "react-virtualized-auto-sizer": "^1.0.20",
     "replicate": "^0.17.0",
     "sbd": "^1.0.19",
+    "sentence-splitter": "^4.3.0",
     "sharp": "^0.32.5",
     "styled-components": "^6.0.7",
     "tailwind-merge": "^1.13.2",

src/app/interface/generate/index.tsx

@@ -3,21 +3,19 @@
 import { useEffect, useRef, useState, useTransition } from "react"
 import { useSpring, animated } from "@react-spring/web"
 import { usePathname, useRouter, useSearchParams } from "next/navigation"
+import { split } from "sentence-splitter"
 
 import { useToast } from "@/components/ui/use-toast"
 import { cn } from "@/lib/utils"
 import { headingFont } from "@/app/interface/fonts"
 import { useCharacterLimit } from "@/lib/useCharacterLimit"
-import { generateStory } from "@/app/server/actions/generateStory"
-import { getLatestPosts, getPost, postToCommunity } from "@/app/server/actions/community"
-import { HotshotImageInferenceSize, Post, SDXLModel, Story, TTSVoice } from "@/types"
-import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
+import { generateStoryLines } from "@/app/server/actions/generateStoryLines"
+import { Story, StoryLine, TTSVoice } from "@/types"
 import { TooltipProvider } from "@radix-ui/react-tooltip"
-
 import { useCountdown } from "@/lib/useCountdown"
+import { useAudio } from "@/lib/useAudio"
 
 import { Countdown } from "../countdown"
-import { useAudio } from "@/lib/useAudio"
 
 type Stage = "generate" | "finished"
 
@@ -38,32 +36,54 @@ export function Generate() {
   const [runs, setRuns] = useState(0)
   const runsRef = useRef(0)
 
-  const [story, setStory] = useState<Story>({ text: "", audio: "" })
-  const storyText = story?.text || ""
-  const audioData = story?.audio || ""
+  const currentLineIndexRef = useRef(0)
+  const [currentLineIndex, setCurrentLineIndex] = useState(0)
+
+  useEffect(() => {
+    currentLineIndexRef.current = currentLineIndex
+  }, [currentLineIndex])
+  
+  const [storyLines, setStoryLines] = useState<StoryLine[]>([])
+
+  // computing those is cheap
+  const wholeStory = storyLines.map(line => line.text).join("\n")
+  const currentLine = storyLines.at(currentLineIndex)
+  const currentLineText = currentLine?.text || ""
+  const currentLineAudio = currentLine?.audio || ""
+
+  // reset the whole player when story changes
+  useEffect(() => {
+    setCurrentLineIndex(0)
+  }, [wholeStory])
 
   const [stage, setStage] = useState<Stage>("generate")
   
   const { toast } = useToast()
 
+  const audio = useAudio()
+
+  /*
+  // to simulate a "typing" effect
+  however.. we don't need this as we already have an audio player!
+
   const [typedStoryText, setTypedStoryText] = useState("")
   const [typedStoryCharacterIndex, setTypedStoryCharacterIndex] = useState(0)
 
-  const audio = useAudio()
-
   useEffect(() => {
     if (storyText && typedStoryCharacterIndex < storyText.length) {
       setTimeout(() => {
         setTypedStoryText(typedStoryText + story.text[typedStoryCharacterIndex])
         setTypedStoryCharacterIndex(typedStoryCharacterIndex + 1)
+        console.log("boom")
       }, 40)
     }
   }, [storyText, typedStoryCharacterIndex])
+  */
   
   const { progressPercent, remainingTimeInSec } = useCountdown({
     isActive: isLocked,
     timerId: runs, // everytime we change this, the timer will reset
-    durationInSec: /*stage === "interpolate" ? 30 :*/ 25, // it usually takes 40 seconds, but there might be lag
+    durationInSec: /*stage === "interpolate" ? 30 :*/ 35, // it usually takes 40 seconds, but there might be lag
     onEnd: () => {}
   })
   
@@ -108,28 +128,17 @@ export function Generate() {
       const search = current.toString()
       router.push(`${pathname}${search ? `?${search}` : ""}`)
 
-      let story: Story = {
-        text: "",
-        audio: ""
-      }
-
       const voice: TTSVoice = "Cloée"
       
       setRuns(runsRef.current + 1)
 
       try {
         // console.log("starting transition, calling generateAnimation")
-        story = await generateStory(promptDraft, voice)
-
-        console.log("generated story:", story)
-
-        if (!story) {
-          throw new Error("invalid story")
-        }
+        const newStoryLines = await generateStoryLines(promptDraft, voice)
 
-        (window as any)["debugJuju"] = story
+        console.log(`generated ${newStoryLines.length} story lines`)
 
-        setStory(story)
+        setStoryLines(newStoryLines)
 
       } catch (err) {
 
@@ -171,21 +180,28 @@ export function Generate() {
 
 
   useEffect(() => {
-    if (!audioData) {
-      return
-    }
-    console.log("story audio changed!", audioData)
+    const fn = async () => {
+      if (!currentLineAudio) {
+        return
+      }
+      console.log("story audio changed!")
 
-    try {
-      audio(audioData) // play
-    } catch (err) {
-      console.error(err)
+      try {
+        console.log("playing audio!")
+        await audio(currentLineAudio) // play
+        console.log("audio has ended, I think? let's go next!")
+        setCurrentLineIndex(currentLineIndexRef.current += 1)
+        // TODO change the line
+      } catch (err) {
+        console.error(err)
+      }
     }
+    fn()
 
     return () => {
       audio() // stop
     }
-  }, [audioData])
+  }, [currentLineAudio])
 
   return (
     <div
@@ -212,9 +228,10 @@ export function Generate() {
         <div
           className={cn(
             `flex flex-col`,
-            `flex-grow rounded-2xl md:rounded-3xl`,
-            `backdrop-blur-md bg-gray-800/30`,
-            `border-2 border-white/10`,
+            `flex-grow`,
+            // `rounded-2xl md:rounded-3xl`,
+            // `backdrop-blur-md bg-gray-800/30`,
+            // `border-2 border-white/10`,
             `items-center`,
             `space-y-6 md:space-y-8 lg:space-y-12 xl:space-y-14`,
             `px-3 py-6 md:px-6 md:py-12 xl:px-8 xl:py-14`,
@@ -252,14 +269,14 @@ export function Generate() {
                     `w-full`,
                     `input input-bordered rounded-full`,
                     `transition-all duration-300 ease-in-out`,
+                     `backdrop-blur-md `,
                     `placeholder:text-gray-400`,
                     `disabled:bg-gray-500 disabled:text-yellow-300 disabled:border-transparent`,
                     isLocked
-                      ? `bg-gray-600 text-yellow-300 border-transparent`
-                      : `bg-white/10 text-yellow-400 selection:bg-yellow-200`,
+                      ? `bg-white/10 text-yellow-400/60 selection:bg-yellow-200/60  selection:text-yellow-200/60 border-transparent`
+                      : `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
                     `text-left`,
-                    `text-xl leading-10 px-6 h-16 pt-1`,
-                    `selection:bg-yellow-200 selection:text-yellow-200`
+                    `text-2xl leading-10 px-6 h-16 pt-1`,
                   )}
                   value={promptDraft}
                   onChange={e => setPromptDraft(e.target.value)}
@@ -276,7 +293,7 @@ export function Generate() {
                   `flex flew-row ml-[-64px] items-center`,
                   `transition-all duration-300 ease-in-out`,
                   `text-base`,
-                  `bg-yellow-200`,
+                  // `bg-yellow-200`,
                   `rounded-full`,
                   `text-right`,
                   `p-1`,
@@ -289,7 +306,7 @@ export function Generate() {
                   <span>{nbCharsLimits}</span>
                 </div>
               </div>
-              <div className="flex flex-row w-52">
+              <div className="flex flex-row w-44">
                 <animated.button
                   style={{
                     textShadow: "0px 0px 1px #000000ab",
@@ -298,15 +315,16 @@ export function Generate() {
                   onMouseEnter={() => setOverSubmitButton(true)}
                   onMouseLeave={() => setOverSubmitButton(false)}
                   className={cn(
-                    `px-6 py-3`,
+                    `px-4 h-16`,
                     `rounded-full`,
                     `transition-all duration-300 ease-in-out`,
+                    `backdrop-blur-sm`,
                     isLocked
-                      ? `bg-orange-500/20  border-orange-800/10`
-                      : `bg-yellow-500/80 hover:bg-yellow-400/100  border-yellow-800/20`,
+                      ? `bg-orange-200/50 text-sky-50/80 border-yellow-600/10`
+                      : `bg-yellow-400/70 text-sky-50  border-yellow-800/20 hover:bg-yellow-400/80`,
                     `text-center`,
                     `w-full`,
-                    `text-2xl text-sky-50`,
+                    `text-2xl `,
                     `border`,
                     headingFont.className,
                     // `transition-all duration-300`,
@@ -342,7 +360,9 @@ export function Generate() {
             `items-center`,
             `space-y-6 md:space-y-8 lg:space-y-12 xl:space-y-14`,
             `px-3 py-6 md:px-6 md:py-12 xl:px-8 xl:py-14`,
-            story.text ? 'scale-1' : 'scale-0'
+             storyLines.length
+               ? 'scale-100'
+               : 'scale-0'
           )}>
             {assetUrl ? <div
               className={cn(
@@ -366,16 +386,23 @@ export function Generate() {
               `items-center justify-between`
             )}>
               <div className={cn(
-                `flex flex-row flex-grow w-full`
+                `flex flex-col flex-grow w-full space-y-2 text-2xl text-blue-200/90`
               )}>
-              <p>{storyText}</p>
+                {storyLines.map((line, i) =>
+                  <div
+                    key={`${line.text}_${i}`}
+
+                    // TODO change a color if we have progressed at the current index (i)
+                    className={cn()}
+                    >{
+                      line.text
+                    }</div>)}
               </div>
             </div>
           </div>
 
         </div>
 
-      
       </TooltipProvider>
     </div>
   )

src/app/layout.tsx

@@ -8,8 +8,8 @@ import './globals.css'
 const inter = Inter({ subsets: ['latin'] })
 
 export const metadata: Metadata = {
-  title: 'AI Bedtime Story 🌜',
-  description: 'AI Bedtime Story 🌜',
+  title: '🌜 AI Bedtime Story 🛏️',
+  description: '🌜 AI Bedtime Story 🛏️',
 }
 
 export default function RootLayout({

src/app/server/actions/{generateStory.ts → generateStoryLines.ts}

@@ -1,11 +1,11 @@
 "use server"
 
-import { Story, TTSVoice } from "@/types"
+import { Story, StoryLine, TTSVoice } from "@/types"
 
 const instance = `${process.env.AI_BEDTIME_STORY_API_GRADIO_URL || ""}`
 const secretToken = `${process.env.AI_BEDTIME_STORY_API_SECRET_TOKEN || ""}`
 
-export async function generateStory(prompt: string, voice: TTSVoice): Promise<Story> {
+export async function generateStoryLines(prompt: string, voice: TTSVoice): Promise<StoryLine[]> {
   if (!prompt?.length) {
     throw new Error(`prompt is too short!`)
   }
@@ -34,22 +34,18 @@ export async function generateStory(prompt: string, voice: TTSVoice): Promise<St
     // next: { revalidate: 1 }
   })
 
-  console.log("res:", res)
-  const rawJson = await res.json()
-  console.log("rawJson:", rawJson)
-  const data = rawJson.data as Story[]
-  console.log("data:", data)
-
-  const story = data?.[0] || { text: "", audio: "" }
 
-  // console.log("story:", story)
+  const rawJson = await res.json()
+  const data = rawJson.data as StoryLine[][]
 
-  // Recommendation: handle errors
-  if (res.status !== 200 || !story?.text || !story?.audio) {
+  const stories = data?.[0] || []
 
-    // This will activate the closest `error.js` Error Boundary
+  if (res.status !== 200) {
     throw new Error('Failed to fetch data')
   }
 
-  return story
+  return stories.map(line => ({
+    text: line.text.replaceAll(" .", ".").replaceAll(" ?", "?").replaceAll(" !", "!").trim(),
+    audio: line.audio
+  }))
 }

src/lib/useAudio.ts

@@ -1,54 +1,80 @@
-import { useCallback, useEffect, useRef } from "react"
+import { useCallback, useEffect, useRef } from 'react';
 
-/**
- * Custom React hook to play a Base64 WAV file.
- */
 export function useAudio() {
-  // Reference to keep track of the current Audio object
-  const audioRef = useRef<HTMLAudioElement | null>(null);
-
-  // This cleanup function will stop the audio and clean up the audio element
-  const stopAndCleanupAudio = useCallback(() => {
-    if (audioRef.current) {
-      audioRef.current.pause();
-      audioRef.current.src = ''; // Release the object URL to avoid memory leaks
-      audioRef.current = null;
-    }
+  const audioContextRef = useRef<AudioContext | null>(null);
+
+  const stopAudio = useCallback(() => {
+    audioContextRef.current?.close();
+    audioContextRef.current = null;
   }, []);
 
-  // Function to load and play the audio, or stop it if called without parameters
+  // Helper function to handle conversion from Base64 to an ArrayBuffer
+  async function base64ToArrayBuffer(base64: string): Promise<ArrayBuffer> {
+    const response = await fetch(base64);
+    return response.arrayBuffer();
+  }
+
   const playAudio = useCallback(
-    (base64Data?: string) => {
-      const base64str = `${base64Data || ""}`
-      if (base64str) {
-        const base64wav = base64str.startsWith("data:audio/wav")
-          ? base64str
-          : `data:audio/wav;base64,${base64str}`
-      
-        // Clean up any existing audio first
-        stopAndCleanupAudio();
-
-        // Create a new Audio object and start playing
-        audioRef.current = new Audio(base64wav);
-        audioRef.current.play().catch((e) => {
-          console.error('Failed to play the audio:', e);
-        });
-      } else {
-        // If no base64 data provided, stop the audio
-        stopAndCleanupAudio();
+    async (base64Data?: string) => {
+      stopAudio(); // Stop any playing audio first
+
+      // If no base64 data provided, we don't attempt to play any audio
+      if (!base64Data) {
+        return;
       }
+
+      // Initialize AudioContext
+      const audioContext = new AudioContext();
+      audioContextRef.current = audioContext;
+
+      // Format Base64 string if necessary and get ArrayBuffer
+      const formattedBase64 =
+        base64Data.startsWith('data:audio/wav') || base64Data.startsWith('data:audio/wav;base64,')
+          ? base64Data
+          : `data:audio/wav;base64,${base64Data}`;
+
+      console.log(`formattedBase64: ${formattedBase64.slice(0, 50)} (len: ${formattedBase64.length})`);
+
+      const arrayBuffer = await base64ToArrayBuffer(formattedBase64);
+
+      return new Promise((resolve, reject) => {
+        // Decode the audio data and play
+        audioContext.decodeAudioData(arrayBuffer, (audioBuffer) => {
+          // Create a source node and gain node
+          const source = audioContext.createBufferSource();
+          const gainNode = audioContext.createGain();
+          
+          // Set buffer and gain
+          source.buffer = audioBuffer;
+          gainNode.gain.value = 1.0; 
+
+          // Connect nodes
+          source.connect(gainNode);
+          gainNode.connect(audioContext.destination);
+
+          // Start playback and handle finishing
+          source.start();
+          
+          source.onended = () => {
+            stopAudio();
+            resolve(true);
+          };
+        }, (error) => {
+          console.error('Error decoding audio data:', error);
+          reject(error);
+        });
+      })
     },
-    [stopAndCleanupAudio]
+    [stopAudio]
   );
 
   // Effect to handle cleanup on component unmount
   useEffect(() => {
     return () => {
-      stopAndCleanupAudio();
+      stopAudio();
     };
-  }, [stopAndCleanupAudio]);
+  }, [stopAudio]);
 
   // Return the playAudio function from the hook
   return playAudio;
-}
-
+}

src/types.ts

@@ -197,7 +197,11 @@ export type QualityOption = {
   label: string
 }
 
-export type Story = {
+export type StoryLine = {
   text: string
   audio: string // in base64
+}
+
+export type Story = {
+  lines: StoryLine[]
 }