Hugging Face's logo

Spaces:
jbilcke-hf
/
ai-tube-clap-exporter
Running on CPU Upgrade

App Files Community
ai-tube-clap-exporter
File size: 3,651 Bytes 
2ed47da
 
2f67628
62e8997
2ed47da
 
 
 
 
 
 
 
62e8997
2ed47da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
bb503b1
 
 
2ed47da
 
 
 
 
 
 
 
 
 
 
bb503b1
 
 
2ed47da
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
import { join } from "node:path"

import { ClapProject, ClapSegment } from "@aitube/clap"

import { concatenateVideosWithAudio } from "../ffmpeg/concatenateVideosWithAudio.mts"
import { writeBase64ToFile } from "../files/writeBase64ToFile.mts"
import { getRandomDirectory } from "../files/getRandomDirectory.mts"
import { addTextToVideo } from "../ffmpeg/addTextToVideo.mts"
import { startOfSegment1IsWithinSegment2 } from "../utils/startOfSegment1IsWithinSegment2.mts"
import { deleteFile } from "../files/deleteFile.mts"
import { extractBase64 } from "../base64/extractBase64.mts"


export async function clapWithVideosToVideoFile({
  clap,
  videoSegments = [],
  outputDir = "",
}: {
  clap: ClapProject
  videoSegments: ClapSegment[]
  outputDir?: string
}): Promise<{
  outputDir: string
  videoFilePaths: string[]
}> {

  outputDir = outputDir || (await getRandomDirectory())

  const videoFilePaths: string[] = []

  for (const segment of videoSegments) {

    const base64Info = extractBase64(segment.assetUrl)
 
    // we write it to the disk *unconverted* (it might be a mp4, a webm or something else)
    let videoSegmentFilePath = await writeBase64ToFile(
      segment.assetUrl,
      join(outputDir, `tmp_asset_${segment.id}.${base64Info.extension}`)
    )

    const interfaceSegments = clap.segments.filter(s =>
      // nope, not all interfaces asset have the assetUrl
      // although in the future.. we might want to
      // s.assetUrl.startsWith("data:text/") &&
      s.category === "interface" &&
      startOfSegment1IsWithinSegment2(s, segment)
    )
    const interfaceSegment = interfaceSegments.at(0)
    if (interfaceSegment) {
      // here we are free to use mp4, since this is an internal intermediary format
      const videoSegmentWithOverlayFilePath = join(outputDir, `tmp_asset_${segment.id}_with_interface.mp4`)

      await addTextToVideo({
        inputVideoPath: videoSegmentFilePath,
        outputVideoPath: videoSegmentWithOverlayFilePath,
        text: interfaceSegment.assetUrl.startsWith("data:text/")
          ? atob(extractBase64(interfaceSegment.assetUrl).data)
          : interfaceSegment.assetUrl,
        width: clap.meta.width,
        height: clap.meta.height,
      })

      // we overwrite
      await deleteFile(videoSegmentFilePath)
      videoSegmentFilePath = videoSegmentWithOverlayFilePath
    }

    const dialogueSegments = clap.segments.filter(s =>
      s.assetUrl.startsWith("data:audio/") &&
      s.category === "dialogue" &&
      startOfSegment1IsWithinSegment2(s, segment)
    )
    const dialogueSegment = dialogueSegments.at(0)
    if (dialogueSegment) {
      extractBase64(dialogueSegment.assetUrl)
      const base64Info = extractBase64(segment.assetUrl)

      const dialogueSegmentFilePath = await writeBase64ToFile(
        dialogueSegment.assetUrl,
        join(outputDir, `tmp_asset_${segment.id}_dialogue.${base64Info.extension}`)
      )
  
      const finalFilePathOfVideoWithSound = await concatenateVideosWithAudio({
        output: join(outputDir, `${segment.id}_video_with_audio.mp4`),
        audioFilePath: dialogueSegmentFilePath,
        videoFilePaths: [videoSegmentFilePath],
        // videos are silent, so they can stay at 0
        videoTracksVolume: 0.0,
        audioTrackVolume: 1.0,
      })

      // we delete the temporary dialogue file
      await deleteFile(dialogueSegmentFilePath)

      // we overwrite the video segment
      await deleteFile(videoSegmentFilePath)

      videoSegmentFilePath = finalFilePathOfVideoWithSound
    }

    videoFilePaths.push(videoSegmentFilePath)
  }

  return {
    outputDir,
    videoFilePaths,
  }
}