Commit
•
7448744
1
Parent(s):
c141ccf
update to use the new improved API
Browse files- .env +4 -6
- src/app/interface/generate/index.tsx +27 -3
- src/app/server/actions/animation.ts +39 -119
- src/app/server/actions/generateWithGradioApi.ts +52 -0
- src/app/server/actions/generateWithReplicateAPI.ts +114 -0
- src/types.ts +35 -8
.env
CHANGED
@@ -1,7 +1,5 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
AUTH_REPLICATE_API_TOKEN="<YOUR SECRET>"
|
5 |
|
6 |
# TODO: those could be
|
7 |
# VIDEO_HOTSHOT_XL_API_OFFICIAL
|
@@ -16,12 +14,12 @@ VIDEO_HOTSHOT_XL_API_OFFICIAL=""
|
|
16 |
|
17 |
# the Node API developed by @jbilcke-hf (me) to allow using any LoRA
|
18 |
# note: doesn't work yet
|
19 |
-
VIDEO_HOTSHOT_XL_API_NODE="https://jbilcke-hf-hotshot-xl-api.hf.space
|
20 |
|
21 |
# the Gradio Space (with the auto API) developed by @fffiloni
|
22 |
# forked to support multiple LoRAs too
|
23 |
# note: work in progress
|
24 |
-
VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server.hf.space
|
25 |
|
26 |
# If you decided to use Replicate for the RENDERING engine
|
27 |
VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
|
|
|
1 |
+
AUTH_REPLICATE_API_TOKEN="<USE YOUR OWN>"
|
2 |
+
AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN="<USE YOUR OWN>"
|
|
|
|
|
3 |
|
4 |
# TODO: those could be
|
5 |
# VIDEO_HOTSHOT_XL_API_OFFICIAL
|
|
|
14 |
|
15 |
# the Node API developed by @jbilcke-hf (me) to allow using any LoRA
|
16 |
# note: doesn't work yet
|
17 |
+
VIDEO_HOTSHOT_XL_API_NODE="https://jbilcke-hf-hotshot-xl-api.hf.space"
|
18 |
|
19 |
# the Gradio Space (with the auto API) developed by @fffiloni
|
20 |
# forked to support multiple LoRAs too
|
21 |
# note: work in progress
|
22 |
+
VIDEO_HOTSHOT_XL_API_GRADIO="https://jbilcke-hf-hotshot-xl-server-1.hf.space"
|
23 |
|
24 |
# If you decided to use Replicate for the RENDERING engine
|
25 |
VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL="cloneofsimo/hotshot-xl-lora-controlnet"
|
src/app/interface/generate/index.tsx
CHANGED
@@ -46,8 +46,31 @@ export function Generate() {
|
|
46 |
console.log("starting transition, calling generateAnimation")
|
47 |
const newAssetUrl = await generateAnimation({
|
48 |
prompt: promptDraft,
|
49 |
-
|
50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
})
|
52 |
setAssetUrl(newAssetUrl)
|
53 |
} catch (err) {
|
@@ -120,7 +143,8 @@ export function Generate() {
|
|
120 |
? `bg-sky-100 text-sky-500 border-transparent`
|
121 |
: `bg-sky-200 text-sky-600 selection:bg-sky-200`,
|
122 |
`text-left`,
|
123 |
-
`text-xl leading-10 px-6 h-16 pt-1
|
|
|
124 |
)}
|
125 |
value={promptDraft}
|
126 |
onChange={e => setPromptDraft(e.target.value)}
|
|
|
46 |
console.log("starting transition, calling generateAnimation")
|
47 |
const newAssetUrl = await generateAnimation({
|
48 |
prompt: promptDraft,
|
49 |
+
huggingFaceLora: "KappaNeuro/studio-ghibli-style",
|
50 |
+
// huggingFaceLora: "veryVANYA/ps1-graphics-sdxl-v2", //
|
51 |
+
// huggingFaceLora: "ostris/crayon_style_lora_sdxl", // "https://huggingface.co/ostris/crayon_style_lora_sdxl/resolve/main/crayons_v1_sdxl.safetensors",
|
52 |
+
// replicateLora: "https://replicate.com/jbilcke/sdxl-panorama",
|
53 |
+
|
54 |
+
// ---- replicate models -----
|
55 |
+
// use: "in the style of TOK" in the prompt!
|
56 |
+
// or this? "in the style of <s0><s1>"
|
57 |
+
// I don't see a lot of diff
|
58 |
+
//
|
59 |
+
// Zelda BOTW
|
60 |
+
// replicateLora: "https://pbxt.replicate.delivery/8UkalcGbGnrNHxGeqeCrhKcPbrRDlx4vLToRRlUWqzpnfieFB/trained_model.tar",
|
61 |
+
|
62 |
+
// Zelda64
|
63 |
+
// replicateLora: "https://pbxt.replicate.delivery/HPZlvCwDWtb5KpefUUcofwvZwTbrZAH9oLvzrn24hqUcQBfFB/trained_model.tar",
|
64 |
+
|
65 |
+
// panorama lora
|
66 |
+
// replicateLora: "https://pbxt.replicate.delivery/nuXez5QNfEmhPk1TLGtl8Q0TwyucZbzTsfUe1ibUfNV0JrMMC/trained_model.tar",
|
67 |
+
|
68 |
+
// foundation
|
69 |
+
// replicateLora: "https://pbxt.replicate.delivery/VHU109Irgh6EPJrZ7aVScvadYDqXhlL3izfEAjfhs8Cvz0hRA/trained_model.tar",
|
70 |
+
|
71 |
+
size: "768x320", // "1024x512", // "512x512" // "320x768"
|
72 |
+
|
73 |
+
steps: 40,
|
74 |
})
|
75 |
setAssetUrl(newAssetUrl)
|
76 |
} catch (err) {
|
|
|
143 |
? `bg-sky-100 text-sky-500 border-transparent`
|
144 |
: `bg-sky-200 text-sky-600 selection:bg-sky-200`,
|
145 |
`text-left`,
|
146 |
+
`text-xl leading-10 px-6 h-16 pt-1`,
|
147 |
+
`selection:bg-sky-800 selection:text-sky-200`
|
148 |
)}
|
149 |
value={promptDraft}
|
150 |
onChange={e => setPromptDraft(e.target.value)}
|
src/app/server/actions/animation.ts
CHANGED
@@ -1,42 +1,38 @@
|
|
1 |
"use server"
|
2 |
|
3 |
-
import
|
4 |
|
5 |
-
import {
|
6 |
-
import {
|
7 |
-
import { sleep } from "@/lib/sleep"
|
8 |
|
9 |
const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
|
10 |
|
11 |
-
const officialApi = `${process.env.VIDEO_HOTSHOT_XL_API_OFFICIAL || ""}`
|
12 |
const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
|
13 |
-
const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
|
14 |
-
|
15 |
-
const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
|
16 |
-
const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
|
17 |
-
const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
|
18 |
|
19 |
export async function generateAnimation({
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
|
|
29 |
throw new Error(`prompt is too short!`)
|
30 |
}
|
31 |
|
32 |
-
// pimp
|
33 |
-
|
|
|
34 |
"beautiful",
|
35 |
-
prompt,
|
36 |
"hd"
|
37 |
].join(", ")
|
38 |
|
39 |
-
|
|
|
40 |
"cropped",
|
41 |
"dark",
|
42 |
"underexposed",
|
@@ -45,76 +41,20 @@ export async function generateAnimation({
|
|
45 |
"watermarked",
|
46 |
].join(", ")
|
47 |
|
48 |
-
const [width, height] = size.split("x").map(x => Number(x))
|
49 |
-
|
50 |
try {
|
51 |
|
52 |
if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
}
|
63 |
-
const replicate = new Replicate({ auth: replicateToken })
|
64 |
-
|
65 |
-
|
66 |
-
// console.log("Calling replicate..")
|
67 |
-
const seed = generateSeed()
|
68 |
-
const prediction = await replicate.predictions.create({
|
69 |
-
version: replicateModelVersion,
|
70 |
-
input: {
|
71 |
-
prompt,
|
72 |
-
negative_prompt: negativePrompt,
|
73 |
-
hf_lora_url: lora,
|
74 |
-
width,
|
75 |
-
height,
|
76 |
-
|
77 |
-
// those are used to create an upsampling or downsampling
|
78 |
-
// original_width: width,
|
79 |
-
// original_height: height,
|
80 |
-
// target_width: width,
|
81 |
-
// target_height: height,
|
82 |
-
|
83 |
-
video_length: 8, // nb frames
|
84 |
-
video_duration: 1000, // video duration in ms
|
85 |
-
// seed
|
86 |
-
}
|
87 |
})
|
88 |
|
89 |
-
// console.log("prediction:", prediction)
|
90 |
-
|
91 |
-
// no need to reply straight away as images take time to generate, this isn't instantaneous
|
92 |
-
// also our friends at Replicate won't like it if we spam them with requests
|
93 |
-
await sleep(5000)
|
94 |
-
|
95 |
-
|
96 |
-
const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
|
97 |
-
method: "GET",
|
98 |
-
headers: {
|
99 |
-
Authorization: `Token ${replicateToken}`,
|
100 |
-
},
|
101 |
-
cache: 'no-store',
|
102 |
-
// we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
|
103 |
-
// next: { revalidate: 1 }
|
104 |
-
})
|
105 |
-
|
106 |
-
// Recommendation: handle errors
|
107 |
-
if (res.status !== 200) {
|
108 |
-
// This will activate the closest `error.js` Error Boundary
|
109 |
-
throw new Error('Failed to fetch data')
|
110 |
-
}
|
111 |
-
|
112 |
-
const response = (await res.json()) as any
|
113 |
-
const error = `${response?.error || ""}`
|
114 |
-
if (error) {
|
115 |
-
throw new Error(error)
|
116 |
-
}
|
117 |
-
return `${response?.output || ""}`
|
118 |
} else if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
|
119 |
// TODO: support other API to avoid duplicate work?
|
120 |
// (are the other API supporting custom LoRAs?)
|
@@ -126,8 +66,8 @@ export async function generateAnimation({
|
|
126 |
// Authorization: `Bearer ${token}`,
|
127 |
},
|
128 |
body: JSON.stringify({
|
129 |
-
prompt,
|
130 |
-
lora,
|
131 |
size,
|
132 |
}),
|
133 |
cache: "no-store",
|
@@ -146,36 +86,16 @@ export async function generateAnimation({
|
|
146 |
|
147 |
return content
|
148 |
} else if (videoEngine === "VIDEO_HOTSHOT_XL_API_GRADIO") {
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
158 |
-
data: [
|
159 |
-
prompt,
|
160 |
-
lora,
|
161 |
-
size,
|
162 |
-
],
|
163 |
-
}),
|
164 |
-
cache: "no-store",
|
165 |
-
// we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
|
166 |
-
// next: { revalidate: 1 }
|
167 |
})
|
168 |
-
|
169 |
-
const { data } = await res.json()
|
170 |
-
|
171 |
-
// Recommendation: handle errors
|
172 |
-
if (res.status !== 200) {
|
173 |
-
// This will activate the closest `error.js` Error Boundary
|
174 |
-
throw new Error('Failed to fetch data')
|
175 |
-
}
|
176 |
-
// console.log("data:", data.slice(0, 50))
|
177 |
-
|
178 |
-
return data
|
179 |
} else {
|
180 |
throw new Error(`not implemented yet!`)
|
181 |
}
|
|
|
1 |
"use server"
|
2 |
|
3 |
+
import { VideoOptions } from "@/types"
|
4 |
|
5 |
+
import { generateVideoWithGradioAPI } from "./generateWithGradioApi"
|
6 |
+
import { generateVideoWithReplicateAPI } from "./generateWithReplicateAPI"
|
|
|
7 |
|
8 |
const videoEngine = `${process.env.VIDEO_ENGINE || ""}`
|
9 |
|
10 |
+
// const officialApi = `${process.env.VIDEO_HOTSHOT_XL_API_OFFICIAL || ""}`
|
11 |
const nodeApi = `${process.env.VIDEO_HOTSHOT_XL_API_NODE || ""}`
|
|
|
|
|
|
|
|
|
|
|
12 |
|
13 |
export async function generateAnimation({
|
14 |
+
positivePrompt = "",
|
15 |
+
negativePrompt = "",
|
16 |
+
size = "512x512",
|
17 |
+
huggingFaceLora,
|
18 |
+
replicateLora,
|
19 |
+
nbFrames = 8,
|
20 |
+
duration = 1000,
|
21 |
+
steps = 30,
|
22 |
+
}: VideoOptions): Promise<string> {
|
23 |
+
if (!positivePrompt?.length) {
|
24 |
throw new Error(`prompt is too short!`)
|
25 |
}
|
26 |
|
27 |
+
// pimp the prompt
|
28 |
+
positivePrompt = [
|
29 |
+
positivePrompt,
|
30 |
"beautiful",
|
|
|
31 |
"hd"
|
32 |
].join(", ")
|
33 |
|
34 |
+
negativePrompt = [
|
35 |
+
negativePrompt,
|
36 |
"cropped",
|
37 |
"dark",
|
38 |
"underexposed",
|
|
|
41 |
"watermarked",
|
42 |
].join(", ")
|
43 |
|
|
|
|
|
44 |
try {
|
45 |
|
46 |
if (videoEngine === "VIDEO_HOTSHOT_XL_API_REPLICATE") {
|
47 |
+
return generateVideoWithReplicateAPI({
|
48 |
+
positivePrompt,
|
49 |
+
negativePrompt,
|
50 |
+
size,
|
51 |
+
huggingFaceLora,
|
52 |
+
replicateLora,
|
53 |
+
nbFrames,
|
54 |
+
duration,
|
55 |
+
steps,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
56 |
})
|
57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
58 |
} else if (videoEngine === "VIDEO_HOTSHOT_XL_API_NODE") {
|
59 |
// TODO: support other API to avoid duplicate work?
|
60 |
// (are the other API supporting custom LoRAs?)
|
|
|
66 |
// Authorization: `Bearer ${token}`,
|
67 |
},
|
68 |
body: JSON.stringify({
|
69 |
+
prompt: positivePrompt,
|
70 |
+
lora: huggingFaceLora,
|
71 |
size,
|
72 |
}),
|
73 |
cache: "no-store",
|
|
|
86 |
|
87 |
return content
|
88 |
} else if (videoEngine === "VIDEO_HOTSHOT_XL_API_GRADIO") {
|
89 |
+
return generateVideoWithGradioAPI({
|
90 |
+
positivePrompt,
|
91 |
+
negativePrompt,
|
92 |
+
size,
|
93 |
+
huggingFaceLora,
|
94 |
+
replicateLora,
|
95 |
+
nbFrames,
|
96 |
+
duration,
|
97 |
+
steps,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
99 |
} else {
|
100 |
throw new Error(`not implemented yet!`)
|
101 |
}
|
src/app/server/actions/generateWithGradioApi.ts
ADDED
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { generateSeed } from "@/lib/generateSeed"
|
2 |
+
import { VideoOptions } from "@/types"
|
3 |
+
|
4 |
+
const gradioApi = `${process.env.VIDEO_HOTSHOT_XL_API_GRADIO || ""}`
|
5 |
+
const accessToken = `${process.env.AUTH_HOTSHOT_XL_API_GRADIO_ACCESS_TOKEN || ""}`
|
6 |
+
|
7 |
+
export async function generateVideoWithGradioAPI({
|
8 |
+
positivePrompt = "",
|
9 |
+
negativePrompt = "",
|
10 |
+
size = "512x512",
|
11 |
+
huggingFaceLora,
|
12 |
+
// replicateLora, // not supported yet
|
13 |
+
nbFrames = 8,
|
14 |
+
duration = 1000,
|
15 |
+
steps = 30,
|
16 |
+
}: VideoOptions): Promise<string> {
|
17 |
+
const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
|
18 |
+
method: "POST",
|
19 |
+
headers: {
|
20 |
+
"Content-Type": "application/json",
|
21 |
+
// Authorization: `Bearer ${token}`,
|
22 |
+
},
|
23 |
+
body: JSON.stringify({
|
24 |
+
fn_index: 0,
|
25 |
+
data: [
|
26 |
+
accessToken,
|
27 |
+
positivePrompt,
|
28 |
+
negativePrompt,
|
29 |
+
huggingFaceLora,
|
30 |
+
size,
|
31 |
+
generateSeed(),
|
32 |
+
steps,
|
33 |
+
nbFrames,
|
34 |
+
duration,
|
35 |
+
],
|
36 |
+
}),
|
37 |
+
cache: "no-store",
|
38 |
+
// we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
|
39 |
+
// next: { revalidate: 1 }
|
40 |
+
})
|
41 |
+
|
42 |
+
const { data } = await res.json()
|
43 |
+
|
44 |
+
// Recommendation: handle errors
|
45 |
+
if (res.status !== 200) {
|
46 |
+
// This will activate the closest `error.js` Error Boundary
|
47 |
+
throw new Error('Failed to fetch data')
|
48 |
+
}
|
49 |
+
// console.log("data:", data.slice(0, 50))
|
50 |
+
|
51 |
+
return data
|
52 |
+
}
|
src/app/server/actions/generateWithReplicateAPI.ts
ADDED
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import Replicate from "replicate"
|
2 |
+
|
3 |
+
import { generateSeed } from "@/lib/generateSeed"
|
4 |
+
import { VideoOptions } from "@/types"
|
5 |
+
import { sleep } from "@/lib/sleep"
|
6 |
+
|
7 |
+
const replicateToken = `${process.env.AUTH_REPLICATE_API_TOKEN || ""}`
|
8 |
+
const replicateModel = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL || ""}`
|
9 |
+
const replicateModelVersion = `${process.env.VIDEO_HOTSHOT_XL_API_REPLICATE_MODEL_VERSION || ""}`
|
10 |
+
|
11 |
+
export async function generateVideoWithReplicateAPI({
|
12 |
+
positivePrompt = "",
|
13 |
+
negativePrompt = "",
|
14 |
+
size = "512x512",
|
15 |
+
huggingFaceLora,
|
16 |
+
replicateLora,
|
17 |
+
steps = 30
|
18 |
+
}: VideoOptions): Promise<string> {
|
19 |
+
if (!replicateToken) {
|
20 |
+
throw new Error(`you need to configure your AUTH_REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
|
21 |
+
}
|
22 |
+
if (!replicateModel) {
|
23 |
+
throw new Error(`you need to configure your RENDERING_REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
|
24 |
+
}
|
25 |
+
|
26 |
+
if (!replicateModelVersion) {
|
27 |
+
throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
|
28 |
+
}
|
29 |
+
const replicate = new Replicate({ auth: replicateToken })
|
30 |
+
|
31 |
+
const [width, height] = size.split("x").map(x => Number(x))
|
32 |
+
|
33 |
+
// console.log("Calling replicate..")
|
34 |
+
const seed = generateSeed()
|
35 |
+
const prediction = await replicate.predictions.create({
|
36 |
+
version: replicateModelVersion,
|
37 |
+
input: {
|
38 |
+
prompt,
|
39 |
+
negative_prompt: negativePrompt,
|
40 |
+
|
41 |
+
// this is not a URL but a model name
|
42 |
+
hf_lora_url: replicateLora ? undefined : huggingFaceLora,
|
43 |
+
|
44 |
+
// this is a URL to the .tar (we can get it from the "trainings" page)
|
45 |
+
replicate_weights_url: huggingFaceLora ? undefined : replicateLora,
|
46 |
+
|
47 |
+
width,
|
48 |
+
height,
|
49 |
+
|
50 |
+
// those are used to create an upsampling or downsampling
|
51 |
+
// original_width: width,
|
52 |
+
// original_height: height,
|
53 |
+
// target_width: width,
|
54 |
+
// target_height: height,
|
55 |
+
|
56 |
+
video_length: 8, // nb frames
|
57 |
+
video_duration: 1000, // video duration in ms
|
58 |
+
|
59 |
+
seed
|
60 |
+
}
|
61 |
+
})
|
62 |
+
|
63 |
+
// console.log("prediction:", prediction)
|
64 |
+
|
65 |
+
// response times are random on Replicate, so this sleep doesn't really help us
|
66 |
+
// we should of course do it like on the AI Comic Factory (use the async loading/polling)
|
67 |
+
// but on the other hand if it takes 2 minutes to answer, it's not really useful either
|
68 |
+
await sleep(50000)
|
69 |
+
|
70 |
+
try {
|
71 |
+
const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
|
72 |
+
method: "GET",
|
73 |
+
headers: {
|
74 |
+
Authorization: `Token ${replicateToken}`,
|
75 |
+
},
|
76 |
+
cache: 'no-store',
|
77 |
+
})
|
78 |
+
|
79 |
+
if (res.status !== 200) {
|
80 |
+
throw new Error("failed")
|
81 |
+
}
|
82 |
+
|
83 |
+
const response = (await res.json()) as any
|
84 |
+
const error = `${response?.error || ""}`
|
85 |
+
if (error) {
|
86 |
+
throw new Error(error)
|
87 |
+
}
|
88 |
+
return `${response?.output || ""}`
|
89 |
+
} catch (err) {
|
90 |
+
// OK let's give replicate a second chance
|
91 |
+
await sleep(10000)
|
92 |
+
|
93 |
+
const res = await fetch(`https://api.replicate.com/v1/predictions/${prediction.id}`, {
|
94 |
+
method: "GET",
|
95 |
+
headers: {
|
96 |
+
Authorization: `Token ${replicateToken}`,
|
97 |
+
},
|
98 |
+
cache: 'no-store',
|
99 |
+
// we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
|
100 |
+
// next: { revalidate: 1 }
|
101 |
+
})
|
102 |
+
|
103 |
+
if (res.status !== 200) {
|
104 |
+
throw new Error('Failed to fetch data')
|
105 |
+
}
|
106 |
+
|
107 |
+
const response = (await res.json()) as any
|
108 |
+
const error = `${response?.error || ""}`
|
109 |
+
if (error) {
|
110 |
+
throw new Error(error)
|
111 |
+
}
|
112 |
+
return `${response?.output || ""}`
|
113 |
+
}
|
114 |
+
}
|
src/types.ts
CHANGED
@@ -1,11 +1,3 @@
|
|
1 |
-
export type ImageInferenceSize =
|
2 |
-
| '320x768'
|
3 |
-
| '384x672'
|
4 |
-
| '416x608'
|
5 |
-
| '512x512'
|
6 |
-
| '608x416'
|
7 |
-
| '672x384'
|
8 |
-
| '768x320'
|
9 |
|
10 |
export type ProjectionMode = 'cartesian' | 'spherical'
|
11 |
|
@@ -273,3 +265,38 @@ export type CurrentPanel =
|
|
273 |
| "join"
|
274 |
| "play"
|
275 |
| "results"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
|
2 |
export type ProjectionMode = 'cartesian' | 'spherical'
|
3 |
|
|
|
265 |
| "join"
|
266 |
| "play"
|
267 |
| "results"
|
268 |
+
|
269 |
+
// vendor-specific types
|
270 |
+
|
271 |
+
export type HotshotImageInferenceSize =
|
272 |
+
| '320x768'
|
273 |
+
| '384x672'
|
274 |
+
| '416x608'
|
275 |
+
| '512x512'
|
276 |
+
| '608x416'
|
277 |
+
| '672x384'
|
278 |
+
| '768x320'
|
279 |
+
| '1024x1024' // custom ratio - this isn't supported / supposed to work properly
|
280 |
+
| '1024x512' // custom panoramic ratio - this isn't supported / supposed to work properly
|
281 |
+
| '1024x576' // movie ratio (16:9) this isn't supported / supposed to work properly
|
282 |
+
| '576x1024' // tiktok ratio (9:16) this isn't supported / supposed to work properly
|
283 |
+
|
284 |
+
export type VideoOptions = {
|
285 |
+
positivePrompt: string
|
286 |
+
|
287 |
+
negativePrompt?: string
|
288 |
+
|
289 |
+
size?: HotshotImageInferenceSize
|
290 |
+
|
291 |
+
/**
|
292 |
+
* Must be a model *name*
|
293 |
+
*/
|
294 |
+
huggingFaceLora?: string
|
295 |
+
|
296 |
+
replicateLora?: string
|
297 |
+
|
298 |
+
nbFrames?: number // FPS (eg. 8)
|
299 |
+
duration?: number // in milliseconds
|
300 |
+
|
301 |
+
steps?: number
|
302 |
+
}
|