work in progress on the AI Stories Factory

src/core/exporters/clapWithStoryboardsToVideoFile.mts

@@ -0,0 +1,106 @@
+import { join } from "node:path"
+
+import { ClapProject } from "../clap/types.mts"
+import { concatenateVideosWithAudio } from "../ffmpeg/concatenateVideosWithAudio.mts"
+import { writeBase64ToFile } from "../files/writeBase64ToFile.mts"
+import { getRandomDirectory } from "../files/getRandomDirectory.mts"
+import { addTextToVideo } from "../ffmpeg/addTextToVideo.mts"
+import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2.mts"
+import { deleteFile } from "../files/deleteFile.mts"
+import { extractBase64 } from "../base64/extractBase64.mts"
+import { ClapSegment } from "../clap/types.mts"
+import { imageToVideoBase64 } from "../ffmpeg/imageToVideoBase64.mts"
+
+export async function clapWithStoryboardsToVideoFile({
+  clap,
+  storyboardSegments = [],
+  outputDir = "",
+}: {
+  clap: ClapProject
+  storyboardSegments: ClapSegment[]
+  outputDir?: string
+}): Promise<{
+  outputDir: string
+  videoFilePaths: string[]
+}> {
+
+  outputDir = outputDir || (await getRandomDirectory())
+
+  const videoFilePaths: string[] = []
+
+  for (const segment of storyboardSegments) {
+
+    let storyboardSegmentVideoFilePath = join(outputDir, `tmp_asset_${segment.id}_as_video.mp4`)
+
+    await imageToVideoBase64({
+      inputImageInBase64: segment.assetUrl,
+      outputFilePath: storyboardSegmentVideoFilePath,
+      outputDir,
+      clearOutputDirAtTheEnd: false, // <- must stay false or else we lose everything!
+      outputVideoFormat: "mp4",
+    })
+
+    const interfaceSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:text/") &&
+      s.category === "interface" &&
+      startOfSegment1IsWithinSegment2(s, segment)
+    )
+    const interfaceSegment = interfaceSegments.at(0)
+    if (interfaceSegment) {
+      // here we are free to use mp4, since this is an internal intermediary format
+      const videoSegmentWithOverlayFilePath = join(outputDir, `tmp_asset_${segment.id}_with_interface.mp4`)
+
+      await addTextToVideo({
+        inputVideoPath: storyboardSegmentVideoFilePath,
+        outputVideoPath: videoSegmentWithOverlayFilePath,
+        text: atob(extractBase64(interfaceSegment.assetUrl).data),
+        width: clap.meta.width,
+        height: clap.meta.height,
+      })
+
+      // we overwrite
+      await deleteFile(storyboardSegmentVideoFilePath)
+      storyboardSegmentVideoFilePath = videoSegmentWithOverlayFilePath
+    }
+
+    const dialogueSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:audio/") &&
+      s.category === "dialogue" &&
+      startOfSegment1IsWithinSegment2(s, segment)
+    )
+    const dialogueSegment = dialogueSegments.at(0)
+    if (dialogueSegment) {
+      extractBase64(dialogueSegment.assetUrl)
+      const base64Info = extractBase64(segment.assetUrl)
+
+      const dialogueSegmentFilePath = await writeBase64ToFile(
+        dialogueSegment.assetUrl,
+        join(outputDir, `tmp_asset_${segment.id}_dialogue.${base64Info.extension}`)
+      )
+  
+      const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
+        output: join(outputDir, `${segment.id}_video_with_audio.mp4`),
+        audioFilePath: dialogueSegmentFilePath,
+        videoFilePaths: [storyboardSegmentVideoFilePath],
+        // videos are silent, so they can stay at 0
+        videoTracksVolume: 0.0,
+        audioTrackVolume: 1.0,
+      })
+
+      // we delete the temporary dialogue file
+      await deleteFile(dialogueSegmentFilePath)
+
+      // we overwrite the video segment
+      await deleteFile(storyboardSegmentVideoFilePath)
+
+      storyboardSegmentVideoFilePath = finalFilePathOfVideoWithSound
+    }
+
+    videoFilePaths.push(storyboardSegmentVideoFilePath)
+  }
+
+  return {
+    outputDir,
+    videoFilePaths,
+  }
+}

src/core/exporters/clapWithVideosToVideoFile.mts

@@ -0,0 +1,103 @@
+import { join } from "node:path"
+
+import { ClapProject } from "../clap/types.mts"
+import { concatenateVideosWithAudio } from "../ffmpeg/concatenateVideosWithAudio.mts"
+import { writeBase64ToFile } from "../files/writeBase64ToFile.mts"
+import { getRandomDirectory } from "../files/getRandomDirectory.mts"
+import { addTextToVideo } from "../ffmpeg/addTextToVideo.mts"
+import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2.mts"
+import { deleteFile } from "../files/deleteFile.mts"
+import { extractBase64 } from "../base64/extractBase64.mts"
+import { ClapSegment } from "../clap/types.mts"
+
+export async function clapWithVideosToVideoFile({
+  clap,
+  videoSegments = [],
+  outputDir = "",
+}: {
+  clap: ClapProject
+  videoSegments: ClapSegment[]
+  outputDir?: string
+}): Promise<{
+  outputDir: string
+  videoFilePaths: string[]
+}> {
+
+  outputDir = outputDir || (await getRandomDirectory())
+
+  const videoFilePaths: string[] = []
+
+  for (const segment of videoSegments) {
+
+    const base64Info = extractBase64(segment.assetUrl)
+ 
+    // we write it to the disk *unconverted* (it might be a mp4, a webm or something else)
+    let videoSegmentFilePath = await writeBase64ToFile(
+      segment.assetUrl,
+      join(outputDir, `tmp_asset_${segment.id}.${base64Info.extension}`)
+    )
+
+    const interfaceSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:text/") &&
+      s.category === "interface" &&
+      startOfSegment1IsWithinSegment2(s, segment)
+    )
+    const interfaceSegment = interfaceSegments.at(0)
+    if (interfaceSegment) {
+      // here we are free to use mp4, since this is an internal intermediary format
+      const videoSegmentWithOverlayFilePath = join(outputDir, `tmp_asset_${segment.id}_with_interface.mp4`)
+
+      await addTextToVideo({
+        inputVideoPath: videoSegmentFilePath,
+        outputVideoPath: videoSegmentWithOverlayFilePath,
+        text: atob(extractBase64(interfaceSegment.assetUrl).data),
+        width: clap.meta.width,
+        height: clap.meta.height,
+      })
+
+      // we overwrite
+      await deleteFile(videoSegmentFilePath)
+      videoSegmentFilePath = videoSegmentWithOverlayFilePath
+    }
+
+    const dialogueSegments = clap.segments.filter(s =>
+      s.assetUrl.startsWith("data:audio/") &&
+      s.category === "dialogue" &&
+      startOfSegment1IsWithinSegment2(s, segment)
+    )
+    const dialogueSegment = dialogueSegments.at(0)
+    if (dialogueSegment) {
+      extractBase64(dialogueSegment.assetUrl)
+      const base64Info = extractBase64(segment.assetUrl)
+
+      const dialogueSegmentFilePath = await writeBase64ToFile(
+        dialogueSegment.assetUrl,
+        join(outputDir, `tmp_asset_${segment.id}_dialogue.${base64Info.extension}`)
+      )
+  
+      const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
+        output: join(outputDir, `${segment.id}_video_with_audio.mp4`),
+        audioFilePath: dialogueSegmentFilePath,
+        videoFilePaths: [videoSegmentFilePath],
+        // videos are silent, so they can stay at 0
+        videoTracksVolume: 0.0,
+        audioTrackVolume: 1.0,
+      })
+
+      // we delete the temporary dialogue file
+      await deleteFile(dialogueSegmentFilePath)
+
+      // we overwrite the video segment
+      await deleteFile(videoSegmentFilePath)
+
+      videoSegmentFilePath = finalFilePathOfVideoWithSound
+    }
+
+    videoFilePaths.push(videoSegmentFilePath)
+  }
+
+  return {
+    outputDir,
+    videoFilePaths,
+  }
+}

src/core/ffmpeg/createVideoFromFrames.mts

@@ -1,5 +1,4 @@
-import { promises as fs } from "node:fs"
-import { writeFile, readFile } from "node:fs/promises"
+import { access, rm, mkdir, writeFile, readFile } from "node:fs/promises"
 import os from "node:os"
 import path from "node:path"
 
@@ -39,7 +38,7 @@ export async function createVideoFromFrames({
   asBase64?: boolean;
 }): Promise<string> {
   // Ensure the input directory exists
-  await fs.access(inputFramesDirectory);
+  await access(inputFramesDirectory);
 
 
   // Construct the input frame pattern
@@ -48,7 +47,7 @@ export async function createVideoFromFrames({
 
   // Create a temporary working directory
   const tempDir = path.join(os.tmpdir(), uuidv4());
-  await fs.mkdir(tempDir);
+  await mkdir(tempDir);
 
    
   let inputVideoToUseAsAudioFilePath = "";
@@ -65,7 +64,7 @@ export async function createVideoFromFrames({
   }
 
   if (debug) {
-    console.log("      createVideoFromFraes(): inputVideoToUseAsAudioFilePath = ", inputVideoToUseAsAudioFilePath)
+    console.log("      createVideoFromFrames(): inputVideoToUseAsAudioFilePath = ", inputVideoToUseAsAudioFilePath)
   }
 
 
@@ -73,7 +72,7 @@ export async function createVideoFromFrames({
   // Also, if provided, check that the audio source file exists
   if (inputVideoToUseAsAudioFilePath) {
     try {
-      await fs.access(inputVideoToUseAsAudioFilePath)
+      await access(inputVideoToUseAsAudioFilePath)
       const info = await getMediaInfo(inputVideoToUseAsAudioFilePath)
       if (info.hasAudio) {
         canUseInputVideoForAudio = true
@@ -108,7 +107,7 @@ export async function createVideoFromFrames({
       .input(inputFramePattern)
       .inputFPS(framesPerSecond)
       .outputOptions([
-        // by default ffmpeg doesn't tell us why it fails to convet
+        // by default ffmpeg doesn't tell us why it fails to convert
         // so we need to force it to spit everything out
         "-loglevel", "debug",
 
@@ -165,7 +164,7 @@ export async function createVideoFromFrames({
           reject(new Error(`Error loading the video file: ${error}`));
         } finally {
           // Clean up temporary files
-          await fs.rm(tempDir, { recursive: true });
+          await rm(tempDir, { recursive: true });
         }
       });
   });

src/core/ffmpeg/imageToVideoBase64.mts

@@ -0,0 +1,87 @@
+import { rm, mkdir, writeFile, readFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import ffmpeg from "fluent-ffmpeg";
+import { getRandomDirectory } from "../files/getRandomDirectory.mts";
+
+/**
+ * Converts an image in Base64 format to a video encoded in Base64.
+ * 
+ * @param inputImageInBase64 - The input image encoded in Base64.
+ * @param outputVideoFormat - Optional. Format of the output video (default is "mp4").
+ * @param outputVideoDurationInMs - Optional. Duration of the video in milliseconds (default is 1000ms).
+ * @param codec - Optional. Codec used for video coding. Defaults differ based on `outputVideoFormat`.
+ * @param width - Optional. Width of the output video.
+ * @param height - Optional. Height of the output video.
+ * @param fps - Optional. Frame rate of the output video.
+ * 
+ * @returns - A promise that resolves to the video as a Base64 encoded string.
+ */
+export async function imageToVideoBase64({
+  inputImageInBase64,
+  outputFilePath,
+  outputDir,
+  clearOutputDirAtTheEnd = true,
+  outputVideoFormat = "mp4",
+  outputVideoDurationInMs = 1000,
+  codec = outputVideoFormat === "webm" ? "libvpx-vp9" : "libx264",
+  width = 1920,
+  height = 1080,
+  fps = 25
+}: {
+  inputImageInBase64: string
+  outputFilePath?: string
+  outputDir?: string
+  clearOutputDirAtTheEnd?: boolean
+  outputVideoFormat?: string
+  outputVideoDurationInMs?: number
+  codec?: string
+  width?: number
+  height?: number
+  fps?: number
+}): Promise<string> {
+
+  outputDir = outputDir || (await getRandomDirectory())
+
+  // Decode the Base64 image and write it to a temporary file.
+  const base64Data = inputImageInBase64.substring(inputImageInBase64.indexOf(',') + 1);
+  const buffer = Buffer.from(base64Data, 'base64');
+  const inputImagePath = path.join(outputDir, 'inputImage.png');
+  await writeFile(inputImagePath, buffer);
+
+  // Set the path for the output video.
+  outputFilePath = outputFilePath || path.join(outputDir, `output.${outputVideoFormat}`);
+  const durationInSeconds = outputVideoDurationInMs / 1000;
+
+  // Process the image to video conversion using ffmpeg.
+  await new Promise<void>((resolve, reject) => {
+    ffmpeg(inputImagePath)
+      .outputOptions([
+        `-t ${durationInSeconds}`,
+        `-r ${fps}`,
+        `-s ${width}x${height}`, // set frame size
+        `-c:v ${codec}`, // set the codec
+        '-tune stillimage',
+        '-pix_fmt yuv420p'
+      ])
+      .on('end', () => resolve())
+      .on('error', (err) => reject(err))
+      .save(outputFilePath);
+  });
+
+  // Read the video file, encode it to Base64, and format it as a data URI.
+  const videoBuffer = await readFile(outputFilePath);
+  const videoBase64 = videoBuffer.toString('base64');
+  const resultAsBase64DataUri = `data:video/${outputVideoFormat};base64,${videoBase64}`;
+
+  // Attempt to clean up temporary work files.
+  if (clearOutputDirAtTheEnd) {
+    try {
+      await rm(outputDir, { recursive: true, force: true });
+    } catch (error) {
+      console.error('Error removing temporary files:', error);
+    }
+  }
+  
+  return resultAsBase64DataUri;
+}

src/main.mts

@@ -7,13 +7,11 @@ import { writeBase64ToFile } from "./core/files/writeBase64ToFile.mts";
 import { concatenateVideos } from "./core/ffmpeg/concatenateVideos.mts"
 import { deleteFilesWithName } from "./core/files/deleteFileWithName.mts"
 import { getRandomDirectory } from "./core/files/getRandomDirectory.mts";
-import { addTextToVideo } from "./core/ffmpeg/addTextToVideo.mts";
-import { startOfSegment1IsWithinSegment2 } from "./core/utils/startOfSegment1IsWithinSegment2.mts";
-import { deleteFile } from "./core/files/deleteFile.mts";
-import { extractBase64 } from "./core/base64/extractBase64.mts";
+import { clapWithVideosToVideoFile } from "./core/exporters/clapWithVideosToVideoFile.mts";
+import { clapWithStoryboardsToVideoFile } from "./core/exporters/clapWithStoryboardsToVideoFile.mts";
 
 /**
- * Generate a .mp4 video inside a direcory (if none is provided, it will be created in /tmp)
+ * Generate a .mp4 video inside a directory (if none is provided, it will be created in /tmp)
  * 
  * @param clap 
  * @returns file path to the final .mp4
@@ -38,75 +36,30 @@ export async function clapToTmpVideoFilePath({
   outputDir = outputDir || (await getRandomDirectory())
 
   const videoFilePaths: string[] = []
-  const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
-
-  for (const segment of videoSegments) {
-
-    const base64Info = extractBase64(segment.assetUrl)
- 
-    // we write it to the disk *unconverted* (it might be a mp4, a webm or something else)
-    let videoSegmentFilePath = await writeBase64ToFile(
-      segment.assetUrl,
-      join(outputDir, `tmp_asset_${segment.id}.${base64Info.extension}`)
-    )
 
-    const interfaceSegments = clap.segments.filter(s =>
-      s.assetUrl.startsWith("data:text/") &&
-      s.category === "interface" &&
-      startOfSegment1IsWithinSegment2(s, segment)
-    )
-    const interfaceSegment = interfaceSegments.at(0)
-    if (interfaceSegment) {
-      // here we are free to use mp4, since this is an internal intermediary format
-      const videoSegmentWithOverlayFilePath = join(outputDir, `tmp_asset_${segment.id}_with_interface.mp4`)
-
-      await addTextToVideo({
-        inputVideoPath: videoSegmentFilePath,
-        outputVideoPath: videoSegmentWithOverlayFilePath,
-        text: atob(extractBase64(interfaceSegment.assetUrl).data),
-        width: clap.meta.width,
-        height: clap.meta.height,
-      })
-
-      // we overwrite
-      await deleteFile(videoSegmentFilePath)
-      videoSegmentFilePath = videoSegmentWithOverlayFilePath
-    }
-
-    const dialogueSegments = clap.segments.filter(s =>
-      s.assetUrl.startsWith("data:audio/") &&
-      s.category === "dialogue" &&
-      startOfSegment1IsWithinSegment2(s, segment)
-    )
-    const dialogueSegment = dialogueSegments.at(0)
-    if (dialogueSegment) {
-      extractBase64(dialogueSegment.assetUrl)
-      const base64Info = extractBase64(segment.assetUrl)
-
-      const dialogueSegmentFilePath = await writeBase64ToFile(
-        dialogueSegment.assetUrl,
-        join(outputDir, `tmp_asset_${segment.id}_dialogue.${base64Info.extension}`)
-      )
-  
-      const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
-        output: join(outputDir, `${segment.id}_video_with_audio.mp4`),
-        audioFilePath: dialogueSegmentFilePath,
-        videoFilePaths: [videoSegmentFilePath],
-        // videos are silent, so they can stay at 0
-        videoTracksVolume: 0.0,
-        audioTrackVolume: 1.0,
-      })
-
-      // we delete the temporary dialogue file
-      await deleteFile(dialogueSegmentFilePath)
-
-      // we overwrite the video segment
-      await deleteFile(videoSegmentFilePath)
-
-      videoSegmentFilePath = finalFilePathOfVideoWithSound
-    }
-
-    videoFilePaths.push(videoSegmentFilePath)
+  const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
+  const storyboardSegments = clap.segments.filter(s => s.category === "storyboard" && s.assetUrl.startsWith("data:image/"))
+
+  const canUseVideos = videoSegments.length > 0
+  const canUseStoryboards = !canUseVideos && storyboardSegments.length > 0
+
+  // two possibilities:
+  // we can either generate from the video files, or from the storyboards
+  // the storyboard video will be a bit more boring, but at least it should process faster
+  if (canUseVideos) {
+    await clapWithVideosToVideoFile({
+      clap,
+      videoSegments,
+      outputDir,
+    })
+  } else if (canUseStoryboards) {
+    await clapWithStoryboardsToVideoFile({
+      clap,
+      storyboardSegments,
+      outputDir,
+    })
+  } else {
+    throw new Error(`the provided Clap doesn't contain any video or storyboard`)
   }
 
   const concatenatedVideosNoMusic = await concatenateVideos({