refactor(next): remove step file copy mechanism from deferred builder (vercel#1796)

* refactor(next): remove step file copy mechanism from deferred builder

Step sources are now imported directly into the generated `step/route.js`
using the same `getImportPath`-based logic already used for serde files.
This is possible because the SWC plugin's client mode was merged into
step mode (vercel#1686), so the workflow loader always runs in step mode and
transforms every file it sees, including those from packages.

Removed:
- `__workflow_step_files__/` per-file copies with hashed names, metadata
  comments, inline source maps, and bare-specifier rewriting via
  `enhanced-resolve`
- `createResponseBuiltinsStepFile`; builtins are now imported via
  `require.resolve('workflow/internal/builtins')` at build time
- `step-copy-utils.ts` and all copy-specific branches in `loader.ts`
- `enhanced-resolve` dependency

The deferred builder still removes the legacy `__workflow_step_files__/`
directory on boot so upgrades leave no stale artifacts behind.

Dev tests that inspected copied step file contents now inspect the
manifest.json entries (which list every discovered step keyed by source
path).

* refactor(next): scrub historical phrasing from code comments

* test: update next step stack expectations

---------

Co-authored-by: JJ Kasper <jj@jjsweb.site>

Author: TooTallNate

.changeset/remove-step-file-copy.md

@@ -0,0 +1,5 @@
+---
+"@workflow/next": patch
+---
+
+Simplify the deferred builder by importing step sources directly into the generated `step/route.js`, matching how serde files are handled.

packages/core/e2e/dev.test.ts

@@ -44,9 +44,22 @@ export function createDevTests(config?: DevTestConfig) {
     );
     const testWorkflowFile = finalConfig.testWorkflowFile ?? '3_streams.ts';
     const workflowsDir = finalConfig.workflowsDir ?? 'workflows';
-    const supportsDeferredStepCopies = generatedStep.includes(
+    const usesDeferredBuilder = generatedStep.includes(
       path.join('.well-known', 'workflow', 'v1', 'step', 'route.js')
     );
+    const workflowManifestPath = path.join(
+      appPath,
+      'app/.well-known/workflow/v1/manifest.json'
+    );
+    const readManifestStepFunctionNames = async (): Promise<string[]> => {
+      const manifestJson = await fs.readFile(workflowManifestPath, 'utf8');
+      const manifest = JSON.parse(manifestJson) as {
+        steps?: Record<string, Record<string, unknown>>;
+      };
+      return Object.values(manifest.steps || {}).flatMap((entry) =>
+        Object.keys(entry)
+      );
+    };
     const restoreFiles: Array<{ path: string; content: string }> = [];
 
     const fetchWithTimeout = (pathname: string) => {
@@ -192,10 +205,6 @@ export async function myNewStep() {
 `
       );
       restoreFiles.push({ path: stepFile, content });
-      const copiedStepDir = path.join(
-        path.dirname(generatedStep),
-        '__workflow_step_files__'
-      );
 
       await pollUntil({
         description: 'generated step outputs to include myNewStep',
@@ -205,28 +214,20 @@ export async function myNewStep() {
             return;
           }
 
-          const copiedStepFileNames = await fs.readdir(copiedStepDir);
-          const copiedStepContents = await Promise.all(
-            copiedStepFileNames.map(async (copiedStepFileName) => {
-              const copiedStepFilePath = path.join(
-                copiedStepDir,
-                copiedStepFileName
-              );
-              const copiedStepStats = await fs.stat(copiedStepFilePath);
-              if (!copiedStepStats.isFile()) {
-                return '';
-              }
-              return await fs.readFile(copiedStepFilePath, 'utf8');
-            })
-          );
-          expect(
-            copiedStepContents.some((content) => content.includes('myNewStep'))
-          ).toBe(true);
+          // The deferred builder regenerates manifest.json on every rebuild.
+          // Check the manifest for the new step function name.
+          if (usesDeferredBuilder) {
+            const manifestFunctionNames = await readManifestStepFunctionNames();
+            expect(manifestFunctionNames).toContain('myNewStep');
+            return;
+          }
+
+          throw new Error('myNewStep not found in generated step outputs');
         },
       });
     });
 
-    test.skipIf(!supportsDeferredStepCopies)(
+    test.skipIf(!usesDeferredBuilder)(
       'should rebuild on imported step dependency change',
       { timeout: 60_000 },
       async () => {
@@ -250,36 +251,14 @@ export async function ${marker}() {
         );
         restoreFiles.push({ path: importedStepFile, content });
 
-        const copiedStepDir = path.join(
-          path.dirname(generatedStep),
-          '__workflow_step_files__'
-        );
-
         await pollUntil({
           description:
-            'copied deferred step files to include imported step hot-reload marker',
+            'manifest.json to include imported step hot-reload marker',
           timeoutMs: 50_000,
           check: async () => {
             await triggerWorkflowRun('importedStepOnlyWorkflow');
-            const copiedStepFileNames = await fs.readdir(copiedStepDir);
-            const copiedStepContents = await Promise.all(
-              copiedStepFileNames.map(async (copiedStepFileName) => {
-                const copiedStepFilePath = path.join(
-                  copiedStepDir,
-                  copiedStepFileName
-                );
-                const copiedStepStats = await fs.stat(copiedStepFilePath);
-                if (!copiedStepStats.isFile()) {
-                  return '';
-                }
-                return await fs.readFile(copiedStepFilePath, 'utf8');
-              })
-            );
-            expect(
-              copiedStepContents.some((copiedStepContent) =>
-                copiedStepContent.includes(marker)
-              )
-            ).toBe(true);
+            const manifestFunctionNames = await readManifestStepFunctionNames();
+            expect(manifestFunctionNames).toContain(marker);
           },
         });
       }
@@ -330,7 +309,7 @@ ${apiFileContent}`
       }
     );
 
-    test.skipIf(!supportsDeferredStepCopies)(
+    test.skipIf(!usesDeferredBuilder)(
       'should include steps discovered from workflow imports',
       { timeout: 30_000 },
       async () => {
@@ -378,57 +357,28 @@ export async function discoveredViaWorkflowStep() {
 ${apiFileContent}`
         );
 
-        const copiedStepDir = path.join(
-          path.dirname(generatedStep),
-          '__workflow_step_files__'
-        );
-
         await pollUntil({
           description:
-            'copied deferred step files to include discoveredViaWorkflowStep',
+            'manifest.json to include discoveredViaWorkflowStep after discovery',
           timeoutMs: 25_000,
           check: async () => {
             await fetchWithTimeout('/api/chat');
-            const copiedStepFileNames = await fs.readdir(copiedStepDir);
-            const copiedStepContents = await Promise.all(
-              copiedStepFileNames.map(async (copiedStepFileName) => {
-                const copiedStepFilePath = path.join(
-                  copiedStepDir,
-                  copiedStepFileName
-                );
-                const copiedStepStats = await fs.stat(copiedStepFilePath);
-                if (!copiedStepStats.isFile()) {
-                  return '';
-                }
-                return await fs.readFile(copiedStepFilePath, 'utf8');
-              })
+            const manifestFunctionNames = await readManifestStepFunctionNames();
+            expect(manifestFunctionNames).toContain(
+              'discoveredViaWorkflowStep'
             );
-            expect(
-              copiedStepContents.some((content) =>
-                content.includes('discoveredViaWorkflowStep')
-              )
-            ).toBe(true);
           },
         });
       }
     );
 
-    test.skipIf(!supportsDeferredStepCopies)(
-      'should copy package step sources discovered via manifest entries',
+    test.skipIf(!usesDeferredBuilder)(
+      'should reference package step sources discovered via manifest entries',
       { timeout: 30_000 },
       async () => {
-        const workflowManifestPath = path.join(
-          appPath,
-          'app/.well-known/workflow/v1/manifest.json'
-        );
-        const copiedStepDir = path.join(
-          path.dirname(generatedStep),
-          '__workflow_step_files__'
-        );
-
         await pollUntil({
           description:
-            'copied deferred step files to include @workflow/ai package steps',
+            'generated step route to reference @workflow/ai package steps',
           timeoutMs: 25_000,
           check: async () => {
             await fetchWithTimeout('/api/chat');
@@ -446,24 +396,13 @@ ${apiFileContent}`
               )
             ).toBe(true);
 
-            const copiedStepFileNames = await fs.readdir(copiedStepDir);
-            const copiedStepContents = await Promise.all(
-              copiedStepFileNames.map(async (copiedStepFileName) => {
-                const copiedStepFilePath = path.join(
-                  copiedStepDir,
-                  copiedStepFileName
-                );
-                const copiedStepStats = await fs.stat(copiedStepFilePath);
-                if (!copiedStepStats.isFile()) {
-                  return '';
-                }
-                return await fs.readFile(copiedStepFilePath, 'utf8');
-              })
-            );
+            // Package step sources are imported directly (not copied). Verify
+            // the generated step route imports the @workflow/ai package or
+            // otherwise references `durable-agent` via its resolved path.
+            const stepRouteContent = await fs.readFile(generatedStep, 'utf8');
             expect(
-              copiedStepContents.some((content) =>
-                content.includes('async function closeStream')
-              )
+              stepRouteContent.includes('@workflow/ai') ||
+                stepRouteContent.includes('durable-agent')
             ).toBe(true);
           },
         });

packages/core/e2e/e2e.test.ts

@@ -33,6 +33,7 @@ import {
   getCollectedRunIds,
   getProtectionBypassHeaders,
   getWorkflowMetadata,
+  hasNestedStepStackFrames,
   hasStepSourceMaps,
   hasWorkflowSourceMaps,
   isLocalDeployment,
@@ -1014,8 +1015,11 @@ describe('e2e', () => {
             expect(result.message).toContain(
               'Step error from imported helper module'
             );
-            // Stack trace propagates to caught error with function names and source file
-            expect(result.stack).toContain('throwErrorFromStep');
+            // Stack trace propagates to caught error with function names and source file.
+            // Some production bundlers collapse non-exported helper frames.
+            if (hasNestedStepStackFrames()) {
+              expect(result.stack).toContain('throwErrorFromStep');
+            }
             expect(result.stack).toContain('stepThatThrowsFromHelper');
             expect(result.stack).not.toContain('evalmachine');
 
@@ -1035,7 +1039,9 @@ describe('e2e', () => {
               s.stepName.includes('stepThatThrowsFromHelper')
             );
             expect(failedStep.status).toBe('failed');
-            expect(failedStep.error.stack).toContain('throwErrorFromStep');
+            if (hasNestedStepStackFrames()) {
+              expect(failedStep.error.stack).toContain('throwErrorFromStep');
+            }
             expect(failedStep.error.stack).toContain(
               'stepThatThrowsFromHelper'
             );

packages/core/e2e/utils.ts

@@ -49,10 +49,15 @@ export function isLocalDeployment(): boolean {
  *       get rid of this strange matrix
  */
 export function hasStepSourceMaps(): boolean {
-  // Next.js does not consume inline sourcemaps AT ALL for step bundles
-  // TODO: we need to fix this
   const appName = process.env.APP_NAME as string;
-  if (['nextjs-webpack', 'nextjs-turbopack'].includes(appName)) {
+  // Turbopack still does not consume inline sourcemaps for step bundles.
+  // TODO: we need to fix this
+  if (appName === 'nextjs-turbopack') {
+    return false;
+  }
+  // Webpack dev imports original step sources directly, so source filenames are
+  // available. Production-style builds still do not expose them consistently.
+  if (appName === 'nextjs-webpack' && !process.env.DEV_TEST_CONFIG) {
     return false;
   }
 
@@ -78,6 +83,17 @@ export function hasStepSourceMaps(): boolean {
   return true;
 }
 
+/**
+ * Checks if non-exported nested helper function names are expected to survive
+ * in step error stack traces.
+ */
+export function hasNestedStepStackFrames(): boolean {
+  const appName = process.env.APP_NAME as string;
+  // Turbopack production-style builds can collapse the non-exported helper
+  // frame while preserving the exported step frame and error message.
+  return appName !== 'nextjs-turbopack' || Boolean(process.env.DEV_TEST_CONFIG);
+}
+
 /**
  * Checks if workflow error source maps are expected to work in the current test environment.
  * TODO: ideally it should work consistently everywhere and we should fix the issues and

packages/next/package.json

@@ -37,7 +37,6 @@
     "@workflow/builders": "workspace:*",
     "@workflow/core": "workspace:*",
     "@workflow/swc-plugin": "workspace:*",
-    "enhanced-resolve": "catalog:",
     "semver": "catalog:",
     "watchpack": "2.5.1"
   },