Skip to content

Commit 20894ab

Browse files
fix: respect user timeouts on low-memory systems (heygen-com#1221)
Closes heygen-com#1219 ## Problem On 8GB RAM machines, renders time out at 5% with `Runtime.callFunctionOn timed out` during the duration probe. User-set timeout env vars (`PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS`) are silently ignored by the calibration path, and there are no CLI flags to control timeouts directly. ## Root causes 1. **Calibration timeout cap overrides user settings** — `createCaptureCalibrationConfig` used `Math.min(cfg.protocolTimeout, 30_000)`, meaning even if the user set 300s, calibration still capped at 30s. On slow hardware this causes unnecessary timeouts. 2. **8GB systems get no low-memory treatment** — `getLowMemoryFlags()`, `getGpuMemBudgetMb()`, `memoryAdaptiveCacheLimit()`, and `memoryAdaptiveCacheBytesMb()` all used `< 8192` as the threshold. Systems reporting exactly 8192 MB (common for 8GB machines) fell through to the "plenty of memory" path, getting no Chrome heap reduction or cache limits. 3. **No CLI flags for key timeouts** — Users had to discover the correct env var names (`PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS`, `PRODUCER_PLAYER_READY_TIMEOUT_MS`) by reading source. The non-existent `PUPPETEER_PROTOCOL_TIMEOUT` and `--browser-timeout` were common guesses that did nothing. ## Changes - `captureCost.ts`: `Math.min` → `Math.max` so the 30s calibration default is a floor, not a ceiling. User-set higher timeouts are now respected. - `browserManager.ts`: `>= 8192` → `> 8192` in `getLowMemoryFlags()` and `<= 8192` in `getGpuMemBudgetMb()` so 8GB systems get reduced Chrome heap and GPU memory budget. - `config.ts`: `< 8192` → `<= 8192` in `memoryAdaptiveCacheLimit()` and `memoryAdaptiveCacheBytesMb()` so 8GB systems get reduced frame cache limits. - `render.ts`: Added `--protocol-timeout <ms>` and `--player-ready-timeout <ms>` CLI flags, wired through `resolveConfig` overrides. - Updated calibration tests to match the new floor-not-ceiling behavior. - Added fallow suppressions for pre-existing unused exports in `captureCost.ts`. ## Test plan - [x] Engine config tests pass (`vitest run src/config.test.ts`) - [x] Browser manager tests pass (`vitest run src/services/browserManager.test.ts`) - [x] Calibration safeguard tests pass (4/4 in `renderOrchestrator.test.ts`) - [x] TypeScript compiles cleanly for engine and cli packages - [ ] CI pipeline
1 parent 1bdb2d4 commit 20894ab

13 files changed

Lines changed: 113 additions & 21 deletions

File tree

.fallowrc.jsonc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,20 @@
9696
"file": "packages/cli/src/commands/render.ts",
9797
"exports": ["resolveBrowserGpuForCli", "renderLocal"],
9898
},
99+
// captureCost.ts: constants and helpers consumed by the runCaptureCalibration
100+
// orchestration function and tests, but the entry-point graph doesn't
101+
// reach them because the orchestrator's caller resolves them dynamically.
102+
{
103+
"file": "packages/producer/src/services/render/captureCost.ts",
104+
"exports": [
105+
"CAPTURE_CALIBRATION_TARGET_MS",
106+
"MAX_MEASURED_CAPTURE_COST_MULTIPLIER",
107+
"CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS",
108+
"measureCaptureCostFromSession",
109+
"logCaptureCalibrationResult",
110+
"createFailedCaptureCalibrationEstimate",
111+
],
112+
},
99113
],
100114
"ignoreDependencies": [
101115
// Runtime/dynamic deps not visible to static analysis: tsup `external`,

packages/cli/src/commands/render.ts

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,20 @@ export default defineCommand({
249249
"readiness poll has its own 45s budget). " +
250250
"Env fallback: PRODUCER_PAGE_NAVIGATION_TIMEOUT_MS (MILLISECONDS).",
251251
},
252+
"protocol-timeout": {
253+
type: "string",
254+
description:
255+
"CDP protocol timeout in ms. Increase on slow/low-memory machines " +
256+
"where Chrome operations time out. Default: 300000 (5 min). " +
257+
"Env: PRODUCER_PUPPETEER_PROTOCOL_TIMEOUT_MS.",
258+
},
259+
"player-ready-timeout": {
260+
type: "string",
261+
description:
262+
"Timeout in ms for the composition player to become ready. " +
263+
"Increase for complex compositions on slow hardware. Default: 45000 (45 s). " +
264+
"Env: PRODUCER_PLAYER_READY_TIMEOUT_MS.",
265+
},
252266
},
253267
// `run` is the citty handler for `hyperframes render` — sequential flag
254268
// validation + render dispatch. Inherited CRITICAL on main (CRAP 1290);
@@ -326,6 +340,32 @@ export default defineCommand({
326340
workers = parsed;
327341
}
328342

343+
// ── Validate timeout overrides ─────────────────────────────────────
344+
let protocolTimeout: number | undefined;
345+
if (args["protocol-timeout"] != null) {
346+
const parsed = parseInt(args["protocol-timeout"], 10);
347+
if (isNaN(parsed) || parsed < 1000) {
348+
errorBox(
349+
"Invalid protocol-timeout",
350+
`Got "${args["protocol-timeout"]}". Must be a number >= 1000 (ms).`,
351+
);
352+
process.exit(1);
353+
}
354+
protocolTimeout = parsed;
355+
}
356+
let playerReadyTimeout: number | undefined;
357+
if (args["player-ready-timeout"] != null) {
358+
const parsed = parseInt(args["player-ready-timeout"], 10);
359+
if (isNaN(parsed) || parsed < 1000) {
360+
errorBox(
361+
"Invalid player-ready-timeout",
362+
`Got "${args["player-ready-timeout"]}". Must be a number >= 1000 (ms).`,
363+
);
364+
process.exit(1);
365+
}
366+
playerReadyTimeout = parsed;
367+
}
368+
329369
// ── Wire opt-in: page-side compositing ───────────────────────────────
330370
if (args["page-side-compositing"] === false) {
331371
process.env.HF_PAGE_SIDE_COMPOSITING = "false";
@@ -347,6 +387,7 @@ export default defineCommand({
347387
// ── Resolve output path ───────────────────────────────────────────────
348388
const rendersDir = resolve("renders");
349389
const ext = FORMAT_EXT[format] ?? ".mp4";
390+
// fallow-ignore-next-line code-duplication
350391
const now = new Date();
351392
const datePart = now.toISOString().slice(0, 10);
352393
const timePart = now.toTimeString().slice(0, 8).replace(/:/g, "-");
@@ -528,6 +569,8 @@ export default defineCommand({
528569
outputResolution,
529570
pageSideCompositing: args["page-side-compositing"] !== false,
530571
pageNavigationTimeoutMs,
572+
protocolTimeout,
573+
playerReadyTimeout,
531574
exitAfterComplete: true,
532575
});
533576
} else {
@@ -547,6 +590,8 @@ export default defineCommand({
547590
entryFile,
548591
outputResolution,
549592
pageNavigationTimeoutMs,
593+
protocolTimeout,
594+
playerReadyTimeout,
550595
exitAfterComplete: true,
551596
});
552597
}
@@ -583,6 +628,10 @@ interface RenderOptions {
583628
* producer's EngineConfig override.
584629
*/
585630
pageNavigationTimeoutMs?: number;
631+
/** CDP protocol timeout override (ms). */
632+
protocolTimeout?: number;
633+
/** Player-ready timeout override (ms). */
634+
playerReadyTimeout?: number;
586635
}
587636

588637
/**
@@ -848,6 +897,7 @@ async function renderDocker(
848897
if (options.exitAfterComplete) scheduleRenderProcessExit();
849898
}
850899

900+
// fallow-ignore-next-line complexity
851901
export async function renderLocal(
852902
projectDir: string,
853903
outputPath: string,
@@ -885,6 +935,8 @@ export async function renderLocal(
885935
...(options.pageNavigationTimeoutMs != null
886936
? { pageNavigationTimeout: options.pageNavigationTimeoutMs }
887937
: {}),
938+
...(options.protocolTimeout != null && { protocolTimeout: options.protocolTimeout }),
939+
...(options.playerReadyTimeout != null && { playerReadyTimeout: options.playerReadyTimeout }),
888940
}),
889941
hdrMode: options.hdrMode,
890942
crf: options.crf,

packages/core/scripts/test-hyperframe-runtime-seek.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,18 @@ function testGsapAdapterPreservesTotalTime(): void {
9393
const { calls, timeline } = createTimeline(true);
9494
const adapter = createGsapAdapter({ getTimeline: () => timeline });
9595

96-
adapter.seek({ time: 2.033333333333333 });
96+
const seekTime = 2.033333333333333;
97+
adapter.seek({ time: seekTime });
9798

9899
assert.deepEqual(
99100
calls,
100-
[{ method: "pause" }, { method: "totalTime", time: 2.033333333333333, suppressEvents: false }],
101-
"GSAP adapter should not downgrade deterministic seeks back to seek()",
101+
[
102+
{ method: "pause" },
103+
// Nudge to force GSAP 3.x dirty state before the real seek
104+
{ method: "totalTime", time: seekTime + 0.001, suppressEvents: true },
105+
{ method: "totalTime", time: seekTime, suppressEvents: false },
106+
],
107+
"GSAP adapter should nudge then seek via totalTime() (not downgrade to seek())",
102108
);
103109
}
104110

packages/core/src/parsers/gsapParser.stress.test.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -869,16 +869,18 @@ describe("Additional edge cases", () => {
869869
expect(result.animations[1].targetSelector).toBe("#el2");
870870
});
871871

872-
it("skips a variable target that is not bound to a DOM lookup", () => {
872+
it("marks a variable target that is not bound to a DOM lookup as __unresolved__", () => {
873873
const script = `
874874
const tl = gsap.timeline({ paused: true });
875875
tl.to(mysteryTarget, { opacity: 1, duration: 0.5 }, 0);
876876
tl.to("#el2", { x: 100, duration: 0.5 }, 0);
877877
`;
878878
const result = parseGsapScript(script);
879-
// mysteryTarget has no resolvable selector binding — only the literal survives.
880-
expect(result.animations).toHaveLength(1);
881-
expect(result.animations[0].targetSelector).toBe("#el2");
879+
// mysteryTarget has no resolvable selector binding — kept with __unresolved__ marker.
880+
expect(result.animations).toHaveLength(2);
881+
expect(result.animations[0].targetSelector).toBe("__unresolved__");
882+
expect(result.animations[0].hasUnresolvedSelector).toBe(true);
883+
expect(result.animations[1].targetSelector).toBe("#el2");
882884
});
883885

884886
it("boolean values in vars are not included in properties", () => {

packages/core/src/parsers/gsapParser.test.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -920,14 +920,16 @@ describe("variable-target resolution (querySelector pattern)", () => {
920920
expect(result.animations[2].extras?.stagger).toBe("__raw:0.1");
921921
});
922922

923-
it("leaves unresolvable variable targets out of the animation list", () => {
923+
it("marks unresolvable variable targets with __unresolved__ and hasUnresolvedSelector", () => {
924924
const script = `
925925
const tl = gsap.timeline({ paused: true });
926926
tl.to(someUnknownThing, { opacity: 1, duration: 0.5 }, 0);
927927
tl.to(".real", { opacity: 1, duration: 0.5 }, 1);
928928
`;
929929
const result = parseGsapScript(script);
930-
expect(result.animations.map((a) => a.targetSelector)).toEqual([".real"]);
930+
expect(result.animations.map((a) => a.targetSelector)).toEqual(["__unresolved__", ".real"]);
931+
expect(result.animations[0].hasUnresolvedSelector).toBe(true);
932+
expect(result.animations[1].hasUnresolvedSelector).toBeUndefined();
931933
});
932934
});
933935

packages/core/src/studio-api/routes/render.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ export function registerRenderRoutes(api: Hono, adapter: StudioApiAdapter): void
8686
composition = body.composition;
8787
}
8888

89+
// fallow-ignore-next-line code-duplication
8990
const now = new Date();
9091
const datePart = now.toISOString().slice(0, 10);
9192
const timePart = now.toTimeString().slice(0, 8).replace(/:/g, "-");

packages/engine/src/config.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,14 +228,14 @@ function getSystemTotalMb(): number {
228228
function memoryAdaptiveCacheLimit(): number {
229229
const total = getSystemTotalMb();
230230
if (total < 4096) return 32;
231-
if (total < 8192) return 64;
231+
if (total <= 8192) return 64;
232232
return DEFAULT_CONFIG.frameDataUriCacheLimit;
233233
}
234234

235235
function memoryAdaptiveCacheBytesMb(): number {
236236
const total = getSystemTotalMb();
237237
if (total < 4096) return 128;
238-
if (total < 8192) return 256;
238+
if (total <= 8192) return 256;
239239
return DEFAULT_CONFIG.frameDataUriCacheBytesLimitMb;
240240
}
241241

packages/engine/src/services/browserManager.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,7 @@ export async function acquireBrowser(
331331
return launchPromise;
332332
}
333333

334+
// fallow-ignore-next-line complexity
334335
async function launchBrowser(
335336
chromeArgs: string[],
336337
config?: Partial<
@@ -509,13 +510,13 @@ function getGpuMemBudgetMb(): number {
509510

510511
const total = getTotalMemMb();
511512
if (total < 4096) return 512;
512-
if (total < 8192) return 1024;
513+
if (total <= 8192) return 1024;
513514
return Math.min(Math.floor(total / 2), 16384);
514515
}
515516

516517
function getLowMemoryFlags(): string[] {
517518
const total = getTotalMemMb();
518-
if (total >= 8192) return [];
519+
if (total > 8192) return [];
519520
const heapMb = total < 4096 ? 256 : 512;
520521
return [`--js-flags=--max-old-space-size=${heapMb}`];
521522
}

packages/producer/src/services/render/captureCost.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ export function resolveRenderWorkerCount(
161161
export function createCaptureCalibrationConfig(cfg: EngineConfig): EngineConfig {
162162
return {
163163
...cfg,
164-
protocolTimeout: Math.min(cfg.protocolTimeout, CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS),
164+
protocolTimeout: Math.max(cfg.protocolTimeout, CAPTURE_CALIBRATION_PROTOCOL_TIMEOUT_MS),
165165
};
166166
}
167167

@@ -282,6 +282,7 @@ export interface CaptureCalibrationOutcome {
282282
* the fallback fires (BeginFrame is no longer the active capture mode,
283283
* so the probe session is no longer reusable).
284284
*/
285+
// fallow-ignore-next-line complexity
285286
export async function runCaptureCalibration(input: {
286287
cfg: EngineConfig;
287288
fileServer: FileServerHandle;

packages/producer/src/services/render/stages/compileStage.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ function createCfg(overrides: Partial<EngineConfig> = {}): EngineConfig {
3737
chromeArgs: [],
3838
chromePath: undefined,
3939
captureCostMultiplier: 1,
40+
// fallow-ignore-next-line code-duplication
4041
format: "jpeg",
4142
jpegQuality: 80,
4243
concurrency: "auto",

0 commit comments

Comments
 (0)