Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 5,183 Bytes
2cae2a9 2f67628 fc91769 4cb7ad9 2cae2a9 46fcec6 2ed47da 2cae2a9 4cb7ad9 2cae2a9 2ed47da 2cae2a9 46fcec6 4cb7ad9 46fcec6 2cae2a9 4cb7ad9 46fcec6 2cae2a9 2ed47da 1f38dc1 2ed47da 1f38dc1 2ed47da b80b9f7 1f38dc1 2ed47da 1f38dc1 2ed47da b80b9f7 1f38dc1 2ed47da 2cae2a9 b80b9f7 6349b58 46fcec6 2cae2a9 46fcec6 2cae2a9 6349b58 2cae2a9 46fcec6 6349b58 2cae2a9 46fcec6 2cae2a9 fc91769 6349b58 46fcec6 4cb7ad9 fc91769 46fcec6 2cae2a9 fc91769 2cae2a9 6349b58 46fcec6 2cae2a9 6349b58 46fcec6 2cae2a9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
import { join } from "node:path"
import { ClapProject } from "@aitube/clap";
import { concatenateAudio, ConcatenateAudioOutput } from "./core/ffmpeg/concatenateAudio.mts";
import { concatenateVideosWithAudio, defaultExportFormat, SupportedExportFormat } from "./core/ffmpeg/concatenateVideosWithAudio.mts";
import { writeBase64ToFile } from "./core/files/writeBase64ToFile.mts";
import { concatenateVideos } from "./core/ffmpeg/concatenateVideos.mts"
import { deleteFilesWithName } from "./core/files/deleteFileWithName.mts"
import { getRandomDirectory } from "./core/files/getRandomDirectory.mts";
import { clapWithVideosToVideoFile } from "./core/exporters/clapWithVideosToVideoFile.mts";
import { clapWithStoryboardsToVideoFile } from "./core/exporters/clapWithStoryboardsToVideoFile.mts";
/**
* Generate a .mp4 video inside a directory (if none is provided, it will be created in /tmp)
*
* @param clap
* @returns file path to the final .mp4
*/
export async function clapToTmpVideoFilePath({
clap,
format = defaultExportFormat,
outputDir = "",
clearTmpFilesAtEnd = false
}: {
clap: ClapProject
format?: SupportedExportFormat
outputDir?: string
// if you leave this to false, you will have to clear files yourself
// (eg. after sending the final video file over)
clearTmpFilesAtEnd?: boolean
}): Promise<{
tmpWorkDir: string
outputFilePath: string
}> {
outputDir = outputDir || (await getRandomDirectory())
const videoSegments = clap.segments.filter(s => s.category === "video" && s.assetUrl.startsWith("data:video/"))
const storyboardSegments = clap.segments.filter(s => s.category === "storyboard" && s.assetUrl.startsWith("data:image/"))
const canUseVideos = videoSegments.length > 0
const canUseStoryboards = !canUseVideos && storyboardSegments.length > 0
let videoFilePaths: string[] = []
// two possibilities:
// we can either generate from the video files, or from the storyboards
// the storyboard video will be a bit more boring, but at least it should process faster
if (canUseVideos) {
const concatenatedData = await clapWithVideosToVideoFile({
clap,
videoSegments,
outputDir,
})
console.log(`clapToTmpVideoFilePath: called clapWithVideosToVideoFile, got concatenatedData = ${JSON.stringify(concatenatedData, null, 2)}`)
videoFilePaths = concatenatedData.videoFilePaths
} else if (canUseStoryboards) {
const concatenatedData = await clapWithStoryboardsToVideoFile({
clap,
storyboardSegments,
outputDir,
})
console.log(`clapToTmpVideoFilePath: called clapWithStoryboardsToVideoFile, got concatenatedData = ${JSON.stringify(concatenatedData, null, 2)}`)
videoFilePaths = concatenatedData.videoFilePaths
} else {
throw new Error(`the provided Clap doesn't contain any video or storyboard`)
}
console.log(`clapToTmpVideoFilePath: calling concatenateVideos over ${videoFilePaths.length} video chunks: ${JSON.stringify(videoFilePaths, null, 2)}`)
const concatenatedVideosNoMusic = await concatenateVideos({
videoFilePaths,
output: join(outputDir, `tmp_asset_concatenated_videos.mp4`)
})
console.log(`clapToTmpVideoFilePath: concatenatedVideosNoMusic`, concatenatedVideosNoMusic)
const audioTracks: string[] = []
const musicSegments = clap.segments.filter(s =>
s.category === "music" &&
s.assetUrl.startsWith("data:audio/")
)
console.log(`clapToTmpVideoFilePath: got ${musicSegments.length} music segments in total`)
for (const segment of musicSegments) {
audioTracks.push(
await writeBase64ToFile(
segment.assetUrl,
join(outputDir, `tmp_asset_${segment.id}.wav`)
)
)
}
let concatenatedAudio: ConcatenateAudioOutput | undefined = undefined
if (audioTracks.length > 0) {
console.log(`clapToTmpVideoFilePath: calling concatenateAudio over ${audioTracks.length} audio tracks`)
concatenatedAudio = await concatenateAudio({
output: join(outputDir, `tmp_asset_concatenated_audio.wav`),
audioTracks,
crossfadeDurationInSec: 2 // 2 seconds
})
console.log(`clapToTmpVideoFilePath: concatenatedAudio = ${concatenatedAudio}`)
}
const finalFilePathOfVideoWithMusic = await concatenateVideosWithAudio({
output: join(outputDir, `final_video.${format}`),
format,
audioFilePath: concatenatedAudio ? concatenatedAudio?.filepath : undefined,
videoFilePaths: [concatenatedVideosNoMusic.filepath],
// videos are silent, so they can stay at 0
videoTracksVolume: concatenatedAudio ? 0.85 : 1.0,
audioTrackVolume: concatenatedAudio ? 0.15 : 0.0, // let's keep the music volume low
})
console.log(`clapToTmpVideoFilePath: finalFilePathOfVideoWithMusic = ${finalFilePathOfVideoWithMusic}`)
if (clearTmpFilesAtEnd) {
// we delete all the temporary assets
await deleteFilesWithName(outputDir, `tmp_asset_`)
}
console.log(`clapToTmpVideoFilePath: returning ${JSON.stringify( {
tmpWorkDir: outputDir,
outputFilePath: finalFilePathOfVideoWithMusic
}, null, 2)}`)
return {
tmpWorkDir: outputDir,
outputFilePath: finalFilePathOfVideoWithMusic
}
} |