WIP Adapters API

packages/open-next/src/adapter.ts

@@ -0,0 +1,146 @@
+import fs from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+import type { NextConfig } from "types/next-types";
+import { compileCache } from "./build/compileCache.js";
+import { compileOpenNextConfig } from "./build/compileConfig.js";
+import { compileTagCacheProvider } from "./build/compileTagCacheProvider.js";
+import { createCacheAssets, createStaticAssets } from "./build/createAssets.js";
+import { createImageOptimizationBundle } from "./build/createImageOptimizationBundle.js";
+import { createMiddleware } from "./build/createMiddleware.js";
+import { createRevalidationBundle } from "./build/createRevalidationBundle.js";
+import { createServerBundle } from "./build/createServerBundle.js";
+import { createWarmerBundle } from "./build/createWarmerBundle.js";
+import { generateOutput } from "./build/generateOutput.js";
+import * as buildHelper from "./build/helper.js";
+import { addDebugFile } from "./debug.js";
+import type { ContentUpdater } from "./plugins/content-updater.js";
+import {
+  externalChunksPlugin,
+  inlineRouteHandler,
+} from "./plugins/inlineRouteHandlers.js";
+
+export type NextAdapterOutputs = {
+  pages: any[];
+  pagesApi: any[];
+  appPages: any[];
+  appRoutes: any[];
+};
+
+type NextAdapter = {
+  name: string;
+  modifyConfig: (
+    config: NextConfig,
+    { phase }: { phase: string },
+  ) => Promise<NextConfig>;
+  onBuildComplete: (props: {
+    routes: any;
+    outputs: NextAdapterOutputs;
+    projectDir: string;
+    repoRoot: string;
+    distDir: string;
+    config: NextConfig;
+    nextVersion: string;
+  }) => Promise<void>;
+}; //TODO: use the one provided by Next
+
+let buildOpts: buildHelper.BuildOptions;
+
+export default {
+  name: "OpenNext",
+  async modifyConfig(nextConfig, { phase }) {
+    // We have to precompile the cache here, probably compile OpenNext config as well
+    const { config, buildDir } = await compileOpenNextConfig(
+      "open-next.config.ts",
+      { nodeExternals: undefined },
+    );
+
+    const require = createRequire(import.meta.url);
+    const openNextDistDir = path.dirname(
+      require.resolve("@opennextjs/aws/index.js"),
+    );
+
+    buildOpts = buildHelper.normalizeOptions(config, openNextDistDir, buildDir);
+
+    buildHelper.initOutputDir(buildOpts);
+
+    const cache = compileCache(buildOpts);
+
+    // We then have to copy the cache files to the .next dir so that they are available at runtime
+    //TODO: use a better path, this one is temporary just to make it work
+    const tempCachePath = `${buildOpts.outputDir}/server-functions/default/.open-next/.build`;
+    fs.mkdirSync(tempCachePath, { recursive: true });
+    fs.copyFileSync(cache.cache, path.join(tempCachePath, "cache.cjs"));
+    fs.copyFileSync(
+      cache.composableCache,
+      path.join(tempCachePath, "composable-cache.cjs"),
+    );
+
+    //TODO: We should check the version of Next here, below 16 we'd throw or show a warning
+    return {
+      ...nextConfig,
+      cacheHandler: cache.cache, //TODO: compute that here,
+      cacheMaxMemorySize: 0,
+      experimental: {
+        ...nextConfig.experimental,
+        trustHostHeader: true,
+        cacheHandlers: {
+          default: cache.composableCache,
+        },
+      },
+    };
+  },
+  async onBuildComplete(outputs) {
+    console.log("OpenNext build will start now");
+
+    // TODO(vicb): save outputs
+    addDebugFile(buildOpts, "outputs.json", outputs);
+
+    // Compile middleware
+    await createMiddleware(buildOpts);
+    console.log("Middleware created");
+
+    createStaticAssets(buildOpts);
+    console.log("Static assets created");
+
+    if (buildOpts.config.dangerous?.disableIncrementalCache !== true) {
+      const { useTagCache } = createCacheAssets(buildOpts);
+      console.log("Cache assets created");
+      if (useTagCache) {
+        await compileTagCacheProvider(buildOpts);
+        console.log("Tag cache provider compiled");
+      }
+    }
+
+    await createServerBundle(
+      buildOpts,
+      {
+        additionalPlugins: getAdditionalPluginsFactory(
+          buildOpts,
+          outputs.outputs,
+        ),
+      },
+      outputs.outputs,
+    );
+
+    console.log("Server bundle created");
+    await createRevalidationBundle(buildOpts);
+    console.log("Revalidation bundle created");
+    await createImageOptimizationBundle(buildOpts);
+    console.log("Image optimization bundle created");
+    await createWarmerBundle(buildOpts);
+    console.log("Warmer bundle created");
+    await generateOutput(buildOpts);
+    console.log("Output generated");
+  },
+} satisfies NextAdapter;
+
+function getAdditionalPluginsFactory(
+  buildOpts: buildHelper.BuildOptions,
+  outputs: NextAdapterOutputs,
+) {
+  return (updater: ContentUpdater) => [
+    inlineRouteHandler(updater, outputs),
+    externalChunksPlugin(outputs),
+  ];
+}

packages/open-next/src/build/compileConfig.ts

@@ -23,6 +23,7 @@ export async function compileOpenNextConfig(
   { nodeExternals = "", compileEdge = false } = {},
 ) {
   const buildDir = fs.mkdtempSync(path.join(os.tmpdir(), "open-next-tmp"));
+
   let configPath = compileOpenNextConfigNode(
     openNextConfigPath,
     buildDir,

packages/open-next/src/build/copyAdapterFiles.ts

@@ -0,0 +1,76 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { NextAdapterOutputs } from "../adapter";
+import { addDebugFile } from "../debug.js";
+import type * as buildHelper from "./helper.js";
+
+export async function copyAdapterFiles(
+  options: buildHelper.BuildOptions,
+  fnName: string,
+  outputs: NextAdapterOutputs,
+) {
+  const filesToCopy = new Map<string, string>();
+
+  // Copying the files from outputs to the output dir
+  for (const [key, value] of Object.entries(outputs)) {
+    if (["pages", "pagesApi", "appPages", "appRoutes"].includes(key)) {
+      for (const route of value as any[]) {
+        const assets = route.assets;
+        // We need to copy the filepaths to the output dir
+        const relativeFilePath = path.relative(options.appPath, route.filePath);
+        // console.log(
+        //   "route.filePath",
+        //   route.filePath,
+        //   "relativeFilePath",
+        //   relativeFilePath,
+        // );
+        filesToCopy.set(
+          route.filePath,
+          `${options.outputDir}/server-functions/${fnName}/${relativeFilePath}`,
+        );
+
+        for (const [relative, from] of Object.entries(assets || {})) {
+          //          console.log("route.assets", from, relative);
+          filesToCopy.set(
+            from as string,
+            `${options.outputDir}/server-functions/${fnName}/${relative}`,
+          );
+        }
+        // copyFileSync(from, `${options.outputDir}/${relative}`);
+      }
+    }
+  }
+
+  console.log("\n### Copying adapter files");
+  const debugCopiedFiles: Record<string, string> = {};
+  for (const [from, to] of filesToCopy) {
+    debugCopiedFiles[from] = to;
+
+    //make sure the directory exists first
+    fs.mkdirSync(path.dirname(to), { recursive: true });
+    // For pnpm symlink we need to do that
+    // see https://github.com/vercel/next.js/blob/498f342b3552d6fc6f1566a1cc5acea324ce0dec/packages/next/src/build/utils.ts#L1932
+    let symlink = "";
+    try {
+      symlink = fs.readlinkSync(from);
+    } catch (e) {
+      //Ignore
+    }
+    if (symlink) {
+      try {
+        fs.symlinkSync(symlink, to);
+      } catch (e: any) {
+        if (e.code !== "EEXIST") {
+          throw e;
+        }
+      }
+    } else {
+      fs.copyFileSync(from, to);
+    }
+  }
+
+  // TODO(vicb): debug
+  addDebugFile(options, "copied_files.json", debugCopiedFiles);
+
+  return Array.from(filesToCopy.values());
+}

packages/open-next/src/build/createAssets.ts

@@ -89,7 +89,7 @@ export function createCacheAssets(options: buildHelper.BuildOptions) {
 
   const dotNextPath = path.join(
     appBuildOutputPath,
-    ".next/standalone",
+    options.config.dangerous?.useAdapterOutputs ? "" : ".next/standalone",
     packagePath,
   );
 

packages/open-next/src/build/createServerBundle.ts

@@ -3,8 +3,10 @@ import path from "node:path";
 
 import type { FunctionOptions, SplittedFunctionOptions } from "types/open-next";
 
+import { createRequire } from "node:module";
 import { loadMiddlewareManifest } from "config/util.js";
 import type { Plugin } from "esbuild";
+import type { NextAdapterOutputs } from "../adapter.js";
 import logger from "../logger.js";
 import { minifyAll } from "../minimize-js.js";
 import { ContentUpdater } from "../plugins/content-updater.js";
@@ -13,6 +15,7 @@ import { openNextResolvePlugin } from "../plugins/resolve.js";
 import { getCrossPlatformPathRegex } from "../utils/regex.js";
 import { bundleNextServer } from "./bundleNextServer.js";
 import { compileCache } from "./compileCache.js";
+import { copyAdapterFiles } from "./copyAdapterFiles.js";
 import { copyTracedFiles } from "./copyTracedFiles.js";
 import {
   copyMiddlewareResources,
@@ -22,6 +25,7 @@ import * as buildHelper from "./helper.js";
 import { installDependencies } from "./installDeps.js";
 import { type CodePatcher, applyCodePatches } from "./patch/codePatcher.js";
 import * as patches from "./patch/patches/index.js";
+const require = createRequire(import.meta.url);
 
 interface CodeCustomization {
   // These patches are meant to apply on user and next generated code
@@ -34,6 +38,7 @@ interface CodeCustomization {
 export async function createServerBundle(
   options: buildHelper.BuildOptions,
   codeCustomization?: CodeCustomization,
+  nextOutputs?: NextAdapterOutputs,
 ) {
   const { config } = options;
   const foundRoutes = new Set<string>();
@@ -55,7 +60,13 @@ export async function createServerBundle(
     if (fnOptions.runtime === "edge") {
       await generateEdgeBundle(name, options, fnOptions);
     } else {
-      await generateBundle(name, options, fnOptions, codeCustomization);
+      await generateBundle(
+        name,
+        options,
+        fnOptions,
+        codeCustomization,
+        nextOutputs,
+      );
     }
   });
 
@@ -108,19 +119,26 @@ export async function createServerBundle(
   }
 
   // Generate default function
-  await generateBundle("default", options, {
-    ...defaultFn,
-    // @ts-expect-error - Those string are RouteTemplate
-    routes: Array.from(remainingRoutes),
-    patterns: ["*"],
-  });
+  await generateBundle(
+    "default",
+    options,
+    {
+      ...defaultFn,
+      // @ts-expect-error - Those string are RouteTemplate
+      routes: Array.from(remainingRoutes),
+      patterns: ["*"],
+    },
+    codeCustomization,
+    nextOutputs,
+  );
 }
 
 async function generateBundle(
   name: string,
   options: buildHelper.BuildOptions,
   fnOptions: SplittedFunctionOptions,
   codeCustomization?: CodeCustomization,
+  nextOutputs?: NextAdapterOutputs,
 ) {
   const { appPath, appBuildOutputPath, config, outputDir, monorepoRoot } =
     options;
@@ -187,14 +205,25 @@ async function generateBundle(
   // Copy env files
   buildHelper.copyEnvFile(appBuildOutputPath, packagePath, outputPath);
 
+  let tracedFiles: string[] = [];
+  let manifests: any = {};
+
   // Copy all necessary traced files
-  const { tracedFiles, manifests } = await copyTracedFiles({
-    buildOutputPath: appBuildOutputPath,
-    packagePath,
-    outputDir: outputPath,
-    routes: fnOptions.routes ?? ["app/page.tsx"],
-    bundledNextServer: isBundled,
-  });
+  if (config.dangerous?.useAdapterOutputs) {
+    tracedFiles = await copyAdapterFiles(options, name, nextOutputs!);
+    //TODO: we should load manifests here
+  } else {
+    const oldTracedFileOutput = await copyTracedFiles({
+      buildOutputPath: appBuildOutputPath,
+      packagePath,
+      outputDir: outputPath,
+      routes: fnOptions.routes ?? ["app/page.tsx"],
+      bundledNextServer: isBundled,
+      skipServerFiles: options.config.dangerous?.useAdapterOutputs === true,
+    });
+    tracedFiles = oldTracedFileOutput.tracedFiles;
+    manifests = oldTracedFileOutput.manifests;
+  }
 
   const additionalCodePatches = codeCustomization?.additionalCodePatches ?? [];
 
@@ -250,6 +279,8 @@ async function generateBundle(
     "15.4.0",
   );
 
+  const useAdapterHandler = config.dangerous?.useAdapterOutputs === true;
+
   const disableRouting = isBefore13413 || config.middleware?.external;
 
   const updater = new ContentUpdater(options);
@@ -268,6 +299,7 @@ async function generateBundle(
         ...(isAfter142 ? ["patchAsyncStorage"] : []),
         ...(isAfter141 ? ["appendPrefetch"] : []),
         ...(isAfter154 ? [] : ["setInitialURL"]),
+        ...(useAdapterHandler ? ["useRequestHandler"] : ["useAdapterHandler"]),
       ],
     }),
     openNextReplacementPlugin({
@@ -281,6 +313,8 @@ async function generateBundle(
           : ["stableIncrementalCache"]),
         ...(isAfter152 ? [] : ["composableCache"]),
       ],
+      replacements: [require.resolve("../core/util.adapter.js")],
+      entireFile: useAdapterHandler,
     }),
 
     openNextResolvePlugin({
@@ -309,6 +343,7 @@ async function generateBundle(
           "const require = topLevelCreateRequire(import.meta.url);",
           "import bannerUrl from 'url';",
           "const __dirname = bannerUrl.fileURLToPath(new URL('.', import.meta.url));",
+          "const __filename = bannerUrl.fileURLToPath(import.meta.url);",
           name === "default" ? "" : `globalThis.fnName = "${name}";`,
         ].join(""),
       },