Turbopack: add caching to chunking

turbopack/crates/turbopack-browser/src/chunking_context.rs

@@ -695,6 +695,8 @@ impl ChunkingContext for BrowserChunkingContext {
             )
             .await?;
 
+            let chunks = chunks.await?;
+
             let mut assets = chunks
                 .iter()
                 .map(|chunk| self.generate_chunk(**chunk).to_resolved())
@@ -761,6 +763,8 @@ impl ChunkingContext for BrowserChunkingContext {
             )
             .await?;
 
+            let chunks = chunks.await?;
+
             let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks
                 .iter()
                 .map(|chunk| self.generate_chunk(**chunk).to_resolved())

turbopack/crates/turbopack-core/src/chunk/chunk_group.rs

@@ -6,12 +6,12 @@ use turbo_rcstr::rcstr;
 use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
 
 use super::{
-    Chunk, ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
+    ChunkGroupContent, ChunkItemWithAsyncModuleInfo, ChunkingContext,
     availability_info::AvailabilityInfo, chunking::make_chunks,
 };
 use crate::{
     chunk::{
-        ChunkableModule, ChunkingType,
+        ChunkableModule, ChunkingType, Chunks,
         available_modules::AvailableModuleItem,
         chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
     },
@@ -35,7 +35,7 @@ use crate::{
 };
 
 pub struct MakeChunkGroupResult {
-    pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
+    pub chunks: ResolvedVc<Chunks>,
     pub referenced_output_assets: Vec<ResolvedVc<Box<dyn OutputAsset>>>,
     pub references: Vec<ResolvedVc<Box<dyn OutputAssetsReference>>>,
     pub availability_info: AvailabilityInfo,
@@ -154,11 +154,12 @@ pub async fn make_chunk_group(
     // Pass chunk items to chunking algorithm
     let chunks = make_chunks(
         module_graph,
-        chunking_context,
-        chunk_items,
-        chunk_item_batch_groups,
+        *chunking_context,
+        Vc::cell(chunk_items),
+        Vc::cell(chunk_item_batch_groups),
         rcstr!(""),
     )
+    .to_resolved()
     .await?;
 
     Ok(MakeChunkGroupResult {

turbopack/crates/turbopack-core/src/chunk/chunk_item_batch.rs

@@ -61,6 +61,9 @@ type ChunkItemOrBatchWithAsyncModuleInfoByChunkType = Either<
     ReadRef<ChunkItemBatchWithAsyncModuleInfoByChunkType>,
 >;
 
+#[turbo_tasks::value(transparent)]
+pub struct ChunkItemOrBatchWithAsyncModuleInfos(Vec<ChunkItemOrBatchWithAsyncModuleInfo>);
+
 impl ChunkItemOrBatchWithAsyncModuleInfo {
     pub async fn from_chunkable_module_or_batch(
         chunkable_module_or_batch: ChunkableModuleOrBatch,
@@ -222,6 +225,9 @@ type ChunkItemBatchGroupByChunkTypeT = SmallVec<
 #[turbo_tasks::value(transparent)]
 pub struct ChunkItemBatchGroupByChunkType(ChunkItemBatchGroupByChunkTypeT);
 
+#[turbo_tasks::value(transparent)]
+pub struct ChunkItemBatchGroups(Vec<ResolvedVc<ChunkItemBatchGroup>>);
+
 #[turbo_tasks::value]
 pub struct ChunkItemBatchGroup {
     pub items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,

turbopack/crates/turbopack-core/src/chunk/chunking/mod.rs

@@ -13,10 +13,11 @@ use turbo_tasks::{
 
 use crate::{
     chunk::{
-        Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkType, ChunkingContext, batch_info,
+        Chunk, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkType, ChunkingContext, Chunks,
+        batch_info,
         chunk_item_batch::{
-            ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo,
-            ChunkItemOrBatchWithAsyncModuleInfo,
+            ChunkItemBatchGroup, ChunkItemBatchGroups, ChunkItemBatchWithAsyncModuleInfo,
+            ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemOrBatchWithAsyncModuleInfos,
         },
         chunking::{
             dev::{app_vendors_split, expand_batches},
@@ -280,15 +281,17 @@ async fn batch_chunk_items_with_info_with_type(
 }
 
 /// Creates chunks based on heuristics for the passed `chunk_items`.
-#[tracing::instrument(level = Level::TRACE, skip_all)]
+#[turbo_tasks::function]
 pub async fn make_chunks(
     module_graph: Vc<ModuleGraph>,
     chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
-    chunk_items_or_batches: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
-    batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
+    chunk_items_or_batches: ResolvedVc<ChunkItemOrBatchWithAsyncModuleInfos>,
+    batch_groups: ResolvedVc<ChunkItemBatchGroups>,
     key_prefix: RcStr,
-) -> Result<Vec<ResolvedVc<Box<dyn Chunk>>>> {
+) -> Result<Vc<Chunks>> {
     let chunking_configs = &*chunking_context.chunking_configs().await?;
+    let chunk_items_or_batches = chunk_items_or_batches.await?;
+    let batch_groups = batch_groups.await?;
 
     let span = tracing::trace_span!(
         "get chunk item info",
@@ -384,7 +387,7 @@ pub async fn make_chunks(
         .try_join()
         .await?;
 
-    Ok(resolved_chunks)
+    Ok(Vc::cell(resolved_chunks))
 }
 
 struct SplitContext<'a> {

turbopack/crates/turbopack-nodejs/src/chunking_context.rs

@@ -472,6 +472,8 @@ impl ChunkingContext for NodeJsChunkingContext {
             )
             .await?;
 
+            let chunks = chunks.await?;
+
             let assets = chunks
                 .iter()
                 .map(|chunk| self.generate_chunk(**chunk).to_resolved())
@@ -524,6 +526,8 @@ impl ChunkingContext for NodeJsChunkingContext {
             )
             .await?;
 
+            let chunks = chunks.await?;
+
             let extra_chunks = extra_chunks.await?;
             let mut other_chunks = chunks
                 .iter()