File size: 5,183 Bytes
2cae2a9
 
2f67628
 
fc91769
4cb7ad9
2cae2a9
 
 
46fcec6
2ed47da
 
2cae2a9
4cb7ad9
2cae2a9
2ed47da
2cae2a9
 
 
 
46fcec6
 
4cb7ad9
46fcec6
 
 
 
2cae2a9
4cb7ad9
46fcec6
 
 
 
 
 
 
 
 
 
 
2cae2a9
2ed47da
 
 
 
 
 
1f38dc1
 
2ed47da
 
 
 
1f38dc1
2ed47da
 
 
 
b80b9f7
 
 
1f38dc1
2ed47da
1f38dc1
2ed47da
 
 
 
b80b9f7
 
 
1f38dc1
2ed47da
 
2cae2a9
 
b80b9f7
6349b58
46fcec6
2cae2a9
46fcec6
2cae2a9
 
6349b58
 
2cae2a9
 
46fcec6
 
 
 
6349b58
 
 
2cae2a9
 
 
 
46fcec6
2cae2a9
 
 
 
fc91769
 
 
 
 
 
 
 
 
 
 
 
 
6349b58
46fcec6
4cb7ad9
 
fc91769
46fcec6
2cae2a9
fc91769
 
2cae2a9
6349b58
 
46fcec6
 
 
 
2cae2a9
6349b58
 
 
 
 
46fcec6
 
 
 
2cae2a9
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
import { join } from "node:path"

import { ClapProject } from "@aitube/clap";

import { concatenateAudio, ConcatenateAudioOutput } from "./core/ffmpeg/concatenateAudio.mts";
import { concatenateVideosWithAudio, defaultExportFormat, SupportedExportFormat } from "./core/ffmpeg/concatenateVideosWithAudio.mts";
import { writeBase64ToFile } from "./core/files/writeBase64ToFile.mts";
import { concatenateVideos } from "./core/ffmpeg/concatenateVideos.mts"
import { deleteFilesWithName } from "./core/files/deleteFileWithName.mts"
import { getRandomDirectory } from "./core/files/getRandomDirectory.mts";
import { clapWithVideosToVideoFile } from "./core/exporters/clapWithVideosToVideoFile.mts";
import { clapWithStoryboardsToVideoFile } from "./core/exporters/clapWithStoryboardsToVideoFile.mts";


/**
 * Generate a .mp4 video inside a directory (if none is provided, it will be created in /tmp)
 * 
 * @param clap 
 * @returns file path to the final .mp4
 */
export async function clapToTmpVideoFilePath({
  clap,
  format = defaultExportFormat,
  outputDir = "",
  clearTmpFilesAtEnd = false
}: {
  clap: ClapProject

  format?: SupportedExportFormat
  outputDir?: string

  // if you leave this to false, you will have to clear files yourself
  // (eg. after sending the final video file over)
  clearTmpFilesAtEnd?: boolean
}): Promise<{
  tmpWorkDir: string
  outputFilePath: string
}> {

  outputDir = outputDir || (await getRandomDirectory())

  const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
  const storyboardSegments = clap.segments.filter(s => s.category === "storyboard" && s.assetUrl.startsWith("data:image/"))

  const canUseVideos = videoSegments.length > 0
  const canUseStoryboards = !canUseVideos && storyboardSegments.length > 0

  let videoFilePaths: string[] = []

  // two possibilities:
  // we can either generate from the video files, or from the storyboards
  // the storyboard video will be a bit more boring, but at least it should process faster
  if (canUseVideos) {
    const concatenatedData = await clapWithVideosToVideoFile({
      clap,
      videoSegments,
      outputDir,
    })

    console.log(`clapToTmpVideoFilePath: called clapWithVideosToVideoFile, got concatenatedData = ${JSON.stringify(concatenatedData, null, 2)}`)
  
    videoFilePaths = concatenatedData.videoFilePaths
  } else if (canUseStoryboards) {
    const concatenatedData = await clapWithStoryboardsToVideoFile({
      clap,
      storyboardSegments,
      outputDir,
    })

    console.log(`clapToTmpVideoFilePath: called clapWithStoryboardsToVideoFile, got concatenatedData = ${JSON.stringify(concatenatedData, null, 2)}`)
  
    videoFilePaths = concatenatedData.videoFilePaths
  } else {
    throw new Error(`the provided Clap doesn't contain any video or storyboard`)
  }

  console.log(`clapToTmpVideoFilePath: calling concatenateVideos over ${videoFilePaths.length} video chunks: ${JSON.stringify(videoFilePaths, null, 2)}`)
  
  const concatenatedVideosNoMusic = await concatenateVideos({
    videoFilePaths,
    output: join(outputDir, `tmp_asset_concatenated_videos.mp4`)
  })

  console.log(`clapToTmpVideoFilePath: concatenatedVideosNoMusic`, concatenatedVideosNoMusic)
  
  const audioTracks: string[] = []

  const musicSegments = clap.segments.filter(s =>
      s.category === "music" &&
      s.assetUrl.startsWith("data:audio/")
  )

  console.log(`clapToTmpVideoFilePath: got ${musicSegments.length} music segments in total`)
  
  for (const segment of musicSegments) {
    audioTracks.push(
      await writeBase64ToFile(
        segment.assetUrl,
        join(outputDir, `tmp_asset_${segment.id}.wav`)
      )
    )
  }

  let concatenatedAudio: ConcatenateAudioOutput | undefined = undefined

  if (audioTracks.length > 0) {
    console.log(`clapToTmpVideoFilePath: calling concatenateAudio over ${audioTracks.length} audio tracks`)
    
    concatenatedAudio = await concatenateAudio({
      output: join(outputDir, `tmp_asset_concatenated_audio.wav`),
      audioTracks,
      crossfadeDurationInSec: 2 // 2 seconds
    })
    console.log(`clapToTmpVideoFilePath: concatenatedAudio = ${concatenatedAudio}`)
  }

  
  const finalFilePathOfVideoWithMusic = await concatenateVideosWithAudio({
    output: join(outputDir, `final_video.${format}`),
    format,
    audioFilePath: concatenatedAudio ? concatenatedAudio?.filepath : undefined,
    videoFilePaths: [concatenatedVideosNoMusic.filepath],
    // videos are silent, so they can stay at 0
    videoTracksVolume: concatenatedAudio ? 0.85 : 1.0,
    audioTrackVolume: concatenatedAudio ? 0.15 : 0.0, // let's keep the music volume low
  })
  console.log(`clapToTmpVideoFilePath: finalFilePathOfVideoWithMusic = ${finalFilePathOfVideoWithMusic}`)
  
  if (clearTmpFilesAtEnd) {
    // we delete all the temporary assets
    await deleteFilesWithName(outputDir, `tmp_asset_`)
  }

  console.log(`clapToTmpVideoFilePath: returning ${JSON.stringify( {
    tmpWorkDir: outputDir,
    outputFilePath: finalFilePathOfVideoWithMusic
  }, null, 2)}`)

  return {
    tmpWorkDir: outputDir,
    outputFilePath: finalFilePathOfVideoWithMusic
  }
}