use @aitube/clap

src/core/clap/getClapAssetSourceType.mts

@@ -1,25 +0,0 @@
-import { ClapAssetSource } from "./types.mts"
-
-export function getClapAssetSourceType(input: string = ""): ClapAssetSource {
-  
-  const str = `${input || ""}`.trim()
-
-  if (!str || !str.length) {
-    return "EMPTY"
-  }
-
-  if (str.startsWith("https://") || str.startsWith("http://")) {
-    return "REMOTE"
-  }
-
-  // note that "path" assets are potentially a security risk, they need to be treated with care
-  if (str.startsWith("/") || str.startsWith("../") || str.startsWith("./")) {
-    return "PATH"
-  }
-
-  if (str.startsWith("data:")) {
-    return "DATA"
-  }
-
-  return "PROMPT"
-}

src/core/clap/parseClap.mts

@@ -1,320 +0,0 @@
-
-import { v4 as uuidv4 } from "uuid"
-import YAML from "yaml"
-
-import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types.mts"
-import { getValidNumber } from "../parsers/getValidNumber.mts"
-import { dataUriToBlob } from "../base64/dataUriToBlob.mts"
-
-type StringOrBlob = string | Blob
-
-/**
- * Import a clap file from various data sources into an ClapProject
- * 
- * Inputs can be:
- * - a Clap project (which is an object)
- * - an URL to a remote .clap file
- * - a string containing a YAML array
- * - a data uri containing a gzipped YAML array
- * - a Blob containing a gzipped YAML array
- * 
- * note: it is not really async, because for some reason YAML.parse is a blocking call like for JSON,
- * there is no async version although we are now in the 20s not 90s
- */
-export async function parseClap(src?: ClapProject | string | Blob, debug = false): Promise<ClapProject> {
-
-  try {
-    if (
-      typeof src === "object" &&
-      Array.isArray( (src as any)?.scenes) &&
-      Array.isArray((src as any)?.models)
-    ) {
-      if (debug) {
-        console.log("parseClap: input is already a Clap file, nothing to do:", src)
-      }
-      // we can skip verification
-      return src as ClapProject
-    }
-  } catch (err) {
-    // well, this is not a clap project
-  }
-
-  let stringOrBlob = (src || "") as StringOrBlob
-
-  // both should work
-  const dataUriHeader1 = "data:application/x-gzip;base64,"
-  const dataUriHeader2 = "data:application/octet-stream;base64,"
-
-  const inputIsString = typeof stringOrBlob === "string"
-  const inputIsDataUri = typeof stringOrBlob === "string" ? stringOrBlob.startsWith(dataUriHeader1) || stringOrBlob.startsWith(dataUriHeader2) : false
-  const inputIsRemoteFile = typeof stringOrBlob === "string" ? (stringOrBlob.startsWith("http://") || stringOrBlob.startsWith("https://")) : false
-
-  let inputIsBlob = typeof stringOrBlob !== "string"
-
-  let inputYamlArrayString = ""
-
-  if (debug) {
-    console.log(`parseClap: pre-analysis: ${JSON.stringify({
-      inputIsString,
-      inputIsBlob,
-      inputIsDataUri,
-      inputIsRemoteFile
-    }, null, 2)}`)
-  }
-
-  if (typeof stringOrBlob === "string") {
-    if (debug) {
-      console.log("parseClap: input is a string ", stringOrBlob.slice(0, 120))
-    }
-    if (inputIsDataUri) {
-      if (debug) {
-        console.log(`parseClap: input is a data uri archive`)
-      }
-      stringOrBlob = dataUriToBlob(stringOrBlob, "application/x-gzip")
-      if (debug) {
-        console.log(`parseClap: inputBlob = `, stringOrBlob)
-      }
-      inputIsBlob = true
-    } else if (inputIsRemoteFile) {
-      try {
-        if (debug) {
-          console.log(`parseClap: input is a remote .clap file`)
-        }
-        const res = await fetch(stringOrBlob)
-        stringOrBlob = await res.blob()
-        if (!stringOrBlob) { throw new Error("blob is empty") }
-        inputIsBlob = true
-      } catch (err) {
-        // url seems invalid
-        throw new Error(`failed to download the .clap file (${err})`)
-      }
-    } else {
-      if (debug) {
-        console.log("parseClap: input is a text string containing a YAML array")
-      }
-      inputYamlArrayString = stringOrBlob
-      inputIsBlob = false
-    }
-  }
-
-  if (typeof stringOrBlob !== "string" && stringOrBlob) {
-    if (debug) {
-      console.log("parseClap: decompressing the blob..")
-    }
-    // Decompress the input blob using gzip
-    const decompressedStream = stringOrBlob.stream().pipeThrough(new DecompressionStream('gzip'))
-
-    try {
-      // Convert the stream to text using a Response object
-      const decompressedOutput = new Response(decompressedStream)
-      // decompressedOutput.headers.set("Content-Type", "application/x-gzip")
-      if (debug) {
-        console.log("parseClap: decompressedOutput: ", decompressedOutput)
-      }
-      // const blobAgain = await decompressedOutput.blob()
-      inputYamlArrayString = await decompressedOutput.text()
-
-      if (debug && inputYamlArrayString) {
-        console.log("parseClap: successfully decompressed the blob!")
-      }
-    } catch (err) {
-      const message = `parseClap: failed to decompress (${err})`
-      console.error(message)
-      throw new Error(message)
-    }
-  }
-
-  // we don't need this anymore I think
-  // new Blob([inputStringOrBlob], { type: "application/x-yaml" })
-
-  let maybeArray: any = {}
-  try {
-    if (debug) {
-      console.log("parseClap: parsing the YAML array..")
-    }
-    // Parse YAML string to raw data
-    maybeArray = YAML.parse(inputYamlArrayString)
-  } catch (err) {
-    throw new Error("invalid clap file (input string is not YAML)")
-  }
-
-  if (!Array.isArray(maybeArray) || maybeArray.length < 2) {
-    throw new Error("invalid clap file (need a clap format header block and project metadata block)")
-  }
-
-  if (debug) {
-    console.log("parseClap: the YAML seems okay, continuing decoding..")
-  }
-
-  const maybeClapHeader = maybeArray[0] as ClapHeader
-
-  if (maybeClapHeader.format !== "clap-0") {
-    throw new Error("invalid clap file (sorry, but you can't make up version numbers like that)")
-  }
-
-
-  const maybeClapMeta = maybeArray[1] as ClapMeta
-
-  const clapMeta: ClapMeta = {
-    id: typeof maybeClapMeta.title === "string" ? maybeClapMeta.id : uuidv4(),
-    title: typeof maybeClapMeta.title === "string" ? maybeClapMeta.title : "",
-    description: typeof maybeClapMeta.description === "string" ? maybeClapMeta.description : "",
-    synopsis: typeof maybeClapMeta.synopsis === "string" ? maybeClapMeta.synopsis : "",
-    licence: typeof maybeClapMeta.licence === "string" ? maybeClapMeta.licence : "",
-    orientation: maybeClapMeta.orientation === "portrait" ? "portrait" : maybeClapMeta.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(maybeClapMeta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(maybeClapMeta.width, 128, 8192, 1024),
-    height: getValidNumber(maybeClapMeta.height, 128, 8192, 576),
-    defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
-    screenplay: typeof maybeClapMeta.screenplay === "string" ? maybeClapMeta.screenplay : "",
-    isLoop: typeof maybeClapMeta.isLoop === "boolean" ? maybeClapMeta.isLoop : false,
-    isInteractive: typeof maybeClapMeta.isInteractive === "boolean" ? maybeClapMeta.isInteractive : false,
-  }
-
-  /*
-  in case we want to support streaming (mix of models and segments etc), we could do it this way:
-
-  const maybeModelsOrSegments = rawData.slice(2)
-  maybeModelsOrSegments.forEach((unknownElement: any) => {
-    if (isValidNumber(unknownElement?.track)) {
-      maybeSegments.push(unknownElement as ClapSegment)
-    } else {
-      maybeModels.push(unknownElement as ClapModel)
-    }
-  })
-  */
-
-
-  const expectedNumberOfModels = maybeClapHeader.numberOfModels || 0
-  const expectedNumberOfScenes = maybeClapHeader.numberOfScenes || 0
-  const expectedNumberOfSegments = maybeClapHeader.numberOfSegments || 0
-
-  // note: we assume the order is strictly enforced!
-  // if you implement streaming (mix of models and segments) you will have to rewrite this!
-
-  const afterTheHeaders = 2
-  const afterTheModels = afterTheHeaders + expectedNumberOfModels
-
-  const afterTheScenes = afterTheModels + expectedNumberOfScenes
-
-  // note: if there are no expected models, maybeModels will be empty
-  const maybeModels = maybeArray.slice(afterTheHeaders, afterTheModels) as ClapModel[]
-
-  // note: if there are no expected scenes, maybeScenes will be empty
-  const maybeScenes = maybeArray.slice(afterTheModels, afterTheScenes) as ClapScene[]
-
-  const maybeSegments = maybeArray.slice(afterTheScenes) as ClapSegment[]
-
-  const clapModels: ClapModel[] = maybeModels.map(({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }))
-
-  const clapScenes: ClapScene[] = maybeScenes.map(({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events,
-  }) => ({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events: events.map(e => e)
-  }))
-  
-  const clapSegments: ClapSegment[] = maybeSegments.map(({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }))
-
-  if (debug) {
-    console.log(`parseClap: successfully parsed ${clapModels.length} models, ${clapScenes.length} scenes and ${clapSegments.length} segments`)
-  }
-  return {
-    meta: clapMeta,
-    models: clapModels,
-    scenes: clapScenes,
-    segments: clapSegments
-  }
-}

src/core/clap/types.mts

@@ -1,203 +0,0 @@
-
-export type ClapSegmentCategory =
-  | "splat"
-  | "mesh"
-  | "depth"
-  | "event"
-  | "interface"
-  | "phenomenon"
-  | "video"
-  | "storyboard"
-  | "transition"
-  | "characters"
-  | "location"
-  | "time"
-  | "era"
-  | "lighting"
-  | "weather"
-  | "action"
-  | "music"
-  | "sound"
-  | "dialogue"
-  | "style"
-  | "camera"
-  | "generic"
-
-export type ClapOutputType =
-  | "text"
-  | "animation"
-  | "interface"
-  | "event"
-  | "phenomenon"
-  | "transition"
-  | "image"
-  | "video"
-  | "audio"
-
-export type ClapSegmentStatus =
-  | "to_generate"
-  | "to_interpolate"
-  | "to_upscale"
-  | "completed"
-  | "error"
-
-export type ClapAuthor =
-  | "auto" // the element was edited automatically using basic if/else logical rules
-  | "ai" // the element was edited using a large language model
-  | "human" // the element was edited by a human
-
-export type ClapAssetSource =
-  | "REMOTE" // http:// or https://
-
-    // note that "path" assets are potentially a security risk, they need to be treated with care
-  | "PATH" // a file path eg. /path or ./path/to/ or ../path/to/
-
-  | "DATA" // a data URI, starting with data:
-
-  | "PROMPT" // by default, a plain text prompt
-
-  | "EMPTY"
-
-export type ClapModelGender =
-  | "male"
-  | "female"
-  | "person"
-  | "object"
-
-export type ClapModelAppearance = "serious" | "neutral" | "friendly" | "chill"
-
-// this is used for accent, style..
-export type ClapModelRegion =
-  | "american"
-  | "british"
-  | "australian"
-  | "canadian"
-  | "indian"
-  | "french"
-  | "italian"
-  | "german"
-  | "chinese"
-
-// note: this is all very subjective, so please use good judgment
-//
-// "deep" might indicate a deeper voice tone, thicker, rich in harmonics
-// in this context, it is used to indicate voices that could
-// be associated with African American (AADOS) characters
-//
-// "high" could be used for some other countries, eg. asia
-export type ClapModelTimbre = "high" | "neutral" | "deep"
-
-export type ClapVoiceVendor = "ElevenLabs" | "XTTS"
-
-export type ClapVoice = {
-  name: string
-  gender: ClapModelGender
-  age: number
-  region: ClapModelRegion
-  timbre: ClapModelTimbre
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapHeader = {
-  format: "clap-0"
-  numberOfModels: number
-  numberOfScenes: number
-  numberOfSegments: number
-}
-
-export type ClapMeta = {
-  id: string
-  title: string
-  description: string
-  synopsis: string
-  licence: string
-  orientation: string
-
-  // the default duration of the experience
-  // the real one might last longer if made interactive
-  durationInMs: number
-
-  width: number
-  height: number
-  defaultVideoModel: string
-  extraPositivePrompt: string[]
-  screenplay: string
-  isLoop: boolean
-  isInteractive: boolean
-}
-
-export type ClapSceneEvent = {
-  id: string
-  type: "description" | "dialogue" | "action"
-  character?: string
-  description: string
-  behavior: string
-  startAtLine: number
-  endAtLine: number
-}
-
-export type ClapScene = {
-  id: string
-  scene: string
-  line: string
-  rawLine: string
-  sequenceFullText: string
-  sequenceStartAtLine: number
-  sequenceEndAtLine: number
-  startAtLine: number
-  endAtLine: number
-  events: ClapSceneEvent[]
-}
-
-export type ClapSegment = {
-  id: string
-  track: number
-  startTimeInMs: number
-  endTimeInMs: number
-  category: ClapSegmentCategory
-  modelId: string
-  sceneId: string
-  prompt: string
-  label: string
-  outputType: ClapOutputType
-  renderId: string
-  status: ClapSegmentStatus
-  assetUrl: string
-  assetDurationInMs: number
-  createdBy: ClapAuthor
-  editedBy: ClapAuthor
-  outputGain: number
-  seed: number
-}
-
-export type ClapModel = {
-  id: string
-  category: ClapSegmentCategory
-  triggerName: string
-  label: string
-  description: string
-  author: string
-  thumbnailUrl: string
-  seed: number
-
-  assetSourceType: ClapAssetSource
-  assetUrl: string
-  
-  // those are only used by certain types of models
-  age: number
-  gender: ClapModelGender
-  region: ClapModelRegion
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapProject = {
-  meta: ClapMeta
-  models: ClapModel[]
-  scenes: ClapScene[]
-  segments: ClapSegment[]
-  // let's keep room for other stuff (screenplay etc)
-}

src/core/exporters/clapWithStoryboardsToVideoFile.mts

@@ -1,6 +1,6 @@
 import { join } from "node:path"
 
-import { ClapProject } from "../clap/types.mts"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { concatenateVideosWithAudio } from "../ffmpeg/concatenateVideosWithAudio.mts"
 import { writeBase64ToFile } from "../files/writeBase64ToFile.mts"
 import { getRandomDirectory } from "../files/getRandomDirectory.mts"
@@ -8,7 +8,6 @@ import { addTextToVideo } from "../ffmpeg/addTextToVideo.mts"
 import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2.mts"
 import { deleteFile } from "../files/deleteFile.mts"
 import { extractBase64 } from "../base64/extractBase64.mts"
-import { ClapSegment } from "../clap/types.mts"
 import { imageToVideoBase64 } from "../ffmpeg/imageToVideoBase64.mts"
 
 export async function clapWithStoryboardsToVideoFile({

src/core/exporters/clapWithVideosToVideoFile.mts

@@ -1,6 +1,6 @@
 import { join } from "node:path"
 
-import { ClapProject } from "../clap/types.mts"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { concatenateVideosWithAudio } from "../ffmpeg/concatenateVideosWithAudio.mts"
 import { writeBase64ToFile } from "../files/writeBase64ToFile.mts"
 import { getRandomDirectory } from "../files/getRandomDirectory.mts"
@@ -8,7 +8,6 @@ import { addTextToVideo } from "../ffmpeg/addTextToVideo.mts"
 import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2.mts"
 import { deleteFile } from "../files/deleteFile.mts"
 import { extractBase64 } from "../base64/extractBase64.mts"
-import { ClapSegment } from "../clap/types.mts"
 
 export async function clapWithVideosToVideoFile({
   clap,

src/index.mts

@@ -1,9 +1,9 @@
 
+import { Blob } from "node:buffer"
+
 import express from "express"
-import { Blob } from "buffer"
+import { parseClap, ClapProject } from "@aitube/clap"
 
-import { parseClap } from "./core/clap/parseClap.mts"
-import { ClapProject } from "./core/clap/types.mts"
 import { clapToTmpVideoFilePath } from "./main.mts"
 import { deleteFile } from "./core/files/deleteFile.mts"
 

src/main.mts

@@ -1,6 +1,7 @@
 import { join } from "node:path"
 
-import { ClapProject } from "./core/clap/types.mts";
+import { ClapProject } from "@aitube/clap";
+
 import { concatenateAudio } from "./core/ffmpeg/concatenateAudio.mts";
 import { concatenateVideosWithAudio } from "./core/ffmpeg/concatenateVideosWithAudio.mts";
 import { writeBase64ToFile } from "./core/files/writeBase64ToFile.mts";