upgrade dependencies

package-lock.json

@@ -9,7 +9,7 @@
       "version": "1.0.0",
       "license": "Apache License",
       "dependencies": {
-        "@aitube/clap": "0.0.10",
+        "@aitube/clap": "0.0.14",
         "@aitube/encoders": "0.0.0",
         "@aitube/io": "0.0.0",
         "dotenv": "^16.3.1",
@@ -29,9 +29,9 @@
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.10",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.10.tgz",
-      "integrity": "sha512-pj855yWhUJ0QTJIaxMJhg0PwC3JaHsScJ5rQ8iqog7zDBWNNJR9MPBtFDYRuIs3vmUsGVxHwSlIBz5I1VjMF8w==",
+      "version": "0.0.14",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.14.tgz",
+      "integrity": "sha512-i4mq3YFecWVOTS/p5QaSQ0VJfurKXlyRc8FJMqKI6P/7rpf4vE4IL+jBKa4HPsYeNt85/KOt3MJKEFVtgiWGfQ==",
       "dependencies": {
         "pure-uuid": "^1.8.1",
         "yaml": "^2.4.1"
@@ -1065,9 +1065,9 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "20.12.8",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.8.tgz",
-      "integrity": "sha512-NU0rJLJnshZWdE/097cdCBbyW1h4hEg0xpovcoAQYHl8dnEyp/NAOiE45pvc+Bd1Dt+2r94v2eGFpQJ4R7g+2w==",
+      "version": "20.12.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.10.tgz",
+      "integrity": "sha512-Eem5pH9pmWBHoGAT8Dr5fdc5rYA+4NAovdM4EktRPVAAiJhmWWfQrA0cFhAbOsQdSfIHjAud6YdkbL69+zSKjw==",
       "dependencies": {
         "undici-types": "~5.26.4"
       }
@@ -2049,9 +2049,9 @@
       }
     },
     "node_modules/get-tsconfig": {
-      "version": "4.7.3",
-      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.3.tgz",
-      "integrity": "sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==",
+      "version": "4.7.4",
+      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.4.tgz",
+      "integrity": "sha512-ofbkKj+0pjXjhejr007J/fLf+sW+8H7K5GCm+msC8q3IpvgjobpyPqSRFemNyIMxklC0zeJpi7VDFna19FacvQ==",
       "dev": true,
       "dependencies": {
         "resolve-pkg-maps": "^1.0.0"
@@ -2667,15 +2667,15 @@
       }
     },
     "node_modules/puppeteer": {
-      "version": "22.7.1",
-      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.7.1.tgz",
-      "integrity": "sha512-JBCBCwQ9+dyPp5haqeecgv0N0vgWFx44woUeKJaPeJT8CU3RXrd8F/tqJQbuAmcWlbMhYJSlTJkIFrwVAs6BNA==",
+      "version": "22.8.0",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.8.0.tgz",
+      "integrity": "sha512-Z616wyTr0d7KpxmfcBG22rAkzuo/xzHJ3ycpu4KiJ3dZNHn/C1CpqcCwPlpiIIsmPojTAfWjo6EMR7M+AaC0Ww==",
       "hasInstallScript": true,
       "dependencies": {
         "@puppeteer/browsers": "2.2.3",
         "cosmiconfig": "9.0.0",
         "devtools-protocol": "0.0.1273771",
-        "puppeteer-core": "22.7.1"
+        "puppeteer-core": "22.8.0"
       },
       "bin": {
         "puppeteer": "lib/esm/puppeteer/node/cli.js"
@@ -2685,15 +2685,15 @@
       }
     },
     "node_modules/puppeteer-core": {
-      "version": "22.7.1",
-      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.7.1.tgz",
-      "integrity": "sha512-jD7T7yN7PWGuJmNT0TAEboA26s0VVnvbgCxqgQIF+eNQW2u71ENaV2JwzSJiCHO+e72H4Ue6AgKD9USQ8xAcOQ==",
+      "version": "22.8.0",
+      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.8.0.tgz",
+      "integrity": "sha512-S5bWx3g/fNuyFxjZX9TkZMN07CEH47+9Zm6IiTl1QfqI9pnVaShbwrD9kRe5vmz/XPp/jLGhhxRUj1sY4wObnA==",
       "dependencies": {
         "@puppeteer/browsers": "2.2.3",
         "chromium-bidi": "0.5.19",
         "debug": "4.3.4",
         "devtools-protocol": "0.0.1273771",
-        "ws": "8.16.0"
+        "ws": "8.17.0"
       },
       "engines": {
         "node": ">=18"
@@ -3211,9 +3211,9 @@
       "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
     "node_modules/tsx": {
-      "version": "4.9.0",
-      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.9.0.tgz",
-      "integrity": "sha512-UY0UUhDPL6MkqkZU4xTEjEBOLfV+RIt4xeeJ1qwK73xai4/zveG+X6+tieILa7rjtegUW2LE4p7fw7gAoLuytA==",
+      "version": "4.9.3",
+      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.9.3.tgz",
+      "integrity": "sha512-czVbetlILiyJZI5zGlj2kw9vFiSeyra9liPD4nG+Thh4pKTi0AmMEQ8zdV/L2xbIVKrIqif4sUNrsMAOksx9Zg==",
       "dev": true,
       "dependencies": {
         "esbuild": "~0.20.2",
@@ -3385,9 +3385,9 @@
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
     "node_modules/ws": {
-      "version": "8.16.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.16.0.tgz",
-      "integrity": "sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==",
+      "version": "8.17.0",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
+      "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
       "engines": {
         "node": ">=10.0.0"
       },

package.json

@@ -13,7 +13,7 @@
   "author": "Julian Bilcke <julian.bilcke@huggingface.co>",
   "license": "Apache License",
   "dependencies": {
-    "@aitube/clap": "0.0.10",
+    "@aitube/clap": "0.0.14",
     "@aitube/encoders": "0.0.0",
     "@aitube/io": "0.0.0",
     "dotenv": "^16.3.1",

src/core/exporters/storyboardSegmentToVideoFile.ts

@@ -1,13 +1,11 @@
 import { join } from "node:path"
 
-import { ClapProject, ClapSegment } from "@aitube/clap"
+import { ClapProject, ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode, filterSegments } from "@aitube/clap"
 import { extractBase64 } from "@aitube/encoders"
 import { deleteFile, writeBase64ToFile } from "@aitube/io"
 //import { addTextToVideo, concatenateVideosWithAudio, imageToVideoBase64 } from  "@aitube/ffmpeg"
 import { addTextToVideo, concatenateVideosWithAudio, imageToVideoBase64 } from  "../../bug-in-bun/aitube_ffmpeg"
 
-import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2"
-
 export async function storyboardSegmentToVideoFile({
   clap,
   segment,
@@ -31,13 +29,13 @@ export async function storyboardSegmentToVideoFile({
     outputVideoFormat: "mp4",
   })
 
-  const interfaceSegments = clap.segments.filter(s =>
-    // nope, not all interfaces asset have the assetUrl
-    // although in the future.. we might want to
-    // s.assetUrl.startsWith("data:text/") &&
-    s.category === "interface" &&
-    startOfSegment1IsWithinSegment2(s, segment)
+  const interfaceSegments = filterSegments(
+    ClapSegmentFilteringMode.START,
+    segment,
+    clap.segments,
+    ClapSegmentCategory.INTERFACE
   )
+
   console.log(`clapWithStoryboardsToVideoFile: got ${interfaceSegments.length} interface segments for shot ${segment.id} [${segment.startTimeInMs}:${segment.endTimeInMs}]`)
 
   const interfaceSegment = interfaceSegments.at(0)
@@ -60,12 +58,12 @@ export async function storyboardSegmentToVideoFile({
     storyboardSegmentVideoFilePath = videoSegmentWithOverlayFilePath
   }
 
-
-  const dialogueSegments = clap.segments.filter(s =>
-    s.assetUrl.startsWith("data:audio/") &&
-    s.category === "dialogue" &&
-    startOfSegment1IsWithinSegment2(s, segment)
-  )
+  const dialogueSegments = filterSegments(
+    ClapSegmentFilteringMode.START,
+    segment,
+    clap.segments,
+    ClapSegmentCategory.DIALOGUE
+  ).filter(s => s.assetUrl.startsWith("data:audio/"))
 
   console.log(`clapWithStoryboardsToVideoFile: got ${dialogueSegments.length} dialogue segments for shot ${segment.id} [${segment.startTimeInMs}:${segment.endTimeInMs}]`)
   

src/core/exporters/videoSegmentToVideoFile.ts

@@ -1,13 +1,11 @@
 import { join } from "node:path"
 
-import { ClapProject, ClapSegment } from "@aitube/clap"
+import { ClapProject, ClapSegment, ClapSegmentCategory, ClapSegmentFilteringMode, filterSegments } from "@aitube/clap"
 import { extractBase64 } from "@aitube/encoders"
 import { deleteFile, writeBase64ToFile } from "@aitube/io"
 // import { addTextToVideo, concatenateVideosWithAudio } from "@aitube/ffmpeg"
 import { addTextToVideo, concatenateVideosWithAudio } from "../../bug-in-bun/aitube_ffmpeg"
 
-import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2"
-
 export async function videoSegmentToVideoFile({
   clap,
   segment,
@@ -25,13 +23,13 @@ export async function videoSegmentToVideoFile({
     segment.assetUrl,
     join(outputDir, `tmp_asset_${segment.id}.${base64Info.extension}`)
   )
-  const interfaceSegments = clap.segments.filter(s =>
-    // nope, not all interfaces asset have the assetUrl
-    // although in the future.. we might want to
-    // s.assetUrl.startsWith("data:text/") &&
-    s.category === "interface" &&
-    startOfSegment1IsWithinSegment2(s, segment)
+  const interfaceSegments = filterSegments(
+    ClapSegmentFilteringMode.START,
+    segment,
+    clap.segments,
+    ClapSegmentCategory.INTERFACE
   )
+  
   const interfaceSegment = interfaceSegments.at(0)
   if (interfaceSegment) {
     // here we are free to use mp4, since this is an internal intermediary format
@@ -49,11 +47,14 @@ export async function videoSegmentToVideoFile({
     await deleteFile(videoSegmentFilePath)
     videoSegmentFilePath = videoSegmentWithOverlayFilePath
   }
-  const dialogueSegments = clap.segments.filter(s =>
-    s.assetUrl.startsWith("data:audio/") &&
-    s.category === "dialogue" &&
-    startOfSegment1IsWithinSegment2(s, segment)
-  )
+
+  const dialogueSegments = filterSegments(
+    ClapSegmentFilteringMode.START,
+    segment,
+    clap.segments,
+    ClapSegmentCategory.DIALOGUE
+  ).map(s => s.assetUrl.startsWith("data:audio/"))
+
   const dialogueSegment = dialogueSegments.at(0)
   if (dialogueSegment) {
     extractBase64(dialogueSegment.assetUrl)

src/core/utils/startOfSegment1IsWithinSegment2.ts

@@ -1,6 +0,0 @@
-import { ClapSegment } from "@aitube/clap"
-
-export function startOfSegment1IsWithinSegment2(s1: ClapSegment, s2: ClapSegment) {
-  const startOfSegment1 = s1.startTimeInMs 
-  return s2.startTimeInMs <= startOfSegment1 && startOfSegment1 <= s2.endTimeInMs
-}

src/main.ts

@@ -1,6 +1,6 @@
 import { join } from "node:path"
 
-import { ClapProject } from "@aitube/clap"
+import { ClapProject, ClapSegmentCategory } from "@aitube/clap"
 import { deleteFilesWithName, getRandomDirectory, writeBase64ToFile } from "@aitube/io"
 import {
   concatenateAudio,
@@ -47,8 +47,8 @@ export async function clapToTmpVideoFilePath({
 
   outputDir = outputDir || (await getRandomDirectory())
 
-  const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
-  const storyboardSegments = clap.segments.filter(s => s.category === "storyboard" && s.assetUrl.startsWith("data:image/"))
+  const videoSegments = clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO && s.assetUrl.startsWith("data:video/"))
+  const storyboardSegments = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD && s.assetUrl.startsWith("data:image/"))
 
   const canUseVideos = videoSegments.length > 0
   const canUseStoryboards = !canUseVideos && storyboardSegments.length > 0
@@ -97,7 +97,7 @@ export async function clapToTmpVideoFilePath({
   const audioTracks: string[] = []
 
   const musicSegments = clap.segments.filter(s =>
-      s.category === "music" &&
+      s.category === ClapSegmentCategory.MUSIC &&
       s.assetUrl.startsWith("data:audio/")
   )