new: Support multiple package workspaces. (#1806)

* Update platform. * Fix lints. * Update type. * Rework rust check. * Fix lints.
moonrepo · Jan 29, 2025 · 7824c0c · 7824c0c
1 parent 697a02d
commit 7824c0c
Show file tree

Hide file tree

Showing 17 changed files with 261 additions and 79 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 #### 🚀 Updates
 
+- Added unstable support for multiple package workspaces when installing dependencies.
 - Updated Pkl (`.pkl`) based configuration to always be enabled, and is no longer hidden behind an
   experimental flag.
   - Added official documentation: https://moonrepo.dev/docs/guides/pkl-config
@@ -19,6 +20,8 @@
   - Added timeout and concurrency limit to clients.
   - Added a `MOON_DEBUG_REMOTE` environment variable, which can be used to debug internal errors for
     diagnosing connection/integration issues.
+- Improved the Rust toolchain.
+  - The root-level project is now properly taken into account when detecting the package workspaces.
 
 #### 🐞 Fixes
 

diff --git a/crates/action-graph/src/action_graph_builder.rs b/crates/action-graph/src/action_graph_builder.rs
@@ -175,6 +175,7 @@ impl<'app> ActionGraphBuilder<'app> {
             .map(|t| &t.toolchains)
             .unwrap_or_else(|| &project.toolchains);
         let mut primary_toolchain = toolchains[0].to_owned();
+        let mut packages_root = WorkspaceRelativePathBuf::default();
 
         // If Bun and Node.js are enabled, they will both attempt to install
         // dependencies in the target root. We need to avoid this problem,
@@ -195,7 +196,11 @@ impl<'app> ActionGraphBuilder<'app> {
         // If project is NOT in the package manager workspace, then we should
         // install dependencies in the project, not the workspace root.
         if let Ok(platform) = self.platform_manager.get_by_toolchain(&primary_toolchain) {
-            if !platform.is_project_in_dependency_workspace(project.source.as_str())? {
+            packages_root = platform.find_dependency_workspace_root(project.source.as_str())?;
+
+            if !platform
+                .is_project_in_dependency_workspace(&packages_root, project.source.as_str())?
+            {
                 in_project = true;
 
                 debug!(
@@ -213,6 +218,7 @@ impl<'app> ActionGraphBuilder<'app> {
         } else {
             ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
                 runtime: self.get_runtime(project, &primary_toolchain, false),
+                root: packages_root,
             })
         };
 

diff --git a/crates/action-graph/tests/action_graph_test.rs b/crates/action-graph/tests/action_graph_test.rs
@@ -128,7 +128,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     })
                 ]
             );
@@ -155,7 +156,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     })
                 ]
             );
@@ -184,7 +186,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::install_project_deps(InstallProjectDepsNode {
                         project: Id::raw("out"),
@@ -222,7 +225,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     })
                 ]
             );
@@ -244,7 +248,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     })
                 ]
             );
@@ -278,7 +283,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -318,7 +324,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -508,7 +515,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -574,7 +582,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -631,7 +640,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -697,7 +707,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -763,7 +774,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),
@@ -843,7 +855,8 @@ mod action_graph {
                         runtime: create_node_runtime()
                     }),
                     ActionNode::install_workspace_deps(InstallWorkspaceDepsNode {
-                        runtime: create_node_runtime()
+                        runtime: create_node_runtime(),
+                        root: WorkspaceRelativePathBuf::new(),
                     }),
                     ActionNode::sync_project(SyncProjectNode {
                         project: Id::raw("bar"),

diff --git a/crates/action-pipeline/src/action_runner.rs b/crates/action-pipeline/src/action_runner.rs
@@ -120,6 +120,7 @@ pub async fn run_action(
                 workspace_graph.clone(),
                 &inner.runtime,
                 None,
+                Some(&inner.root),
             )
             .await;
 
@@ -151,6 +152,7 @@ pub async fn run_action(
                 workspace_graph.clone(),
                 &inner.runtime,
                 Some(&project),
+                None,
             )
             .await;
 

diff --git a/crates/action-pipeline/tests/report_estimate.rs b/crates/action-pipeline/tests/report_estimate.rs
@@ -1,5 +1,6 @@
 use moon_action::*;
 use moon_action_pipeline::reports::estimate::{Estimate, TaskEstimate};
+use moon_common::path::WorkspaceRelativePathBuf;
 use moon_toolchain::Runtime;
 use rustc_hash::FxHashMap;
 use std::sync::Arc;
@@ -131,6 +132,7 @@ mod estimate {
                     node: Arc::new(ActionNode::install_workspace_deps(
                         InstallWorkspaceDepsNode {
                             runtime: Runtime::system(),
+                            root: WorkspaceRelativePathBuf::new(),
                         },
                     )),
                     ..Action::default()
@@ -205,6 +207,7 @@ mod estimate {
                     node: Arc::new(ActionNode::install_workspace_deps(
                         InstallWorkspaceDepsNode {
                             runtime: Runtime::system(),
+                            root: WorkspaceRelativePathBuf::new(),
                         },
                     )),
                     ..Action::default()
@@ -280,6 +283,7 @@ mod estimate {
                     node: Arc::new(ActionNode::install_workspace_deps(
                         InstallWorkspaceDepsNode {
                             runtime: Runtime::system(),
+                            root: WorkspaceRelativePathBuf::new(),
                         },
                     )),
                     ..Action::default()

diff --git a/crates/action/src/action_node.rs b/crates/action/src/action_node.rs
@@ -1,3 +1,4 @@
+use moon_common::path::WorkspaceRelativePathBuf;
 use moon_common::Id;
 use moon_target::Target;
 use moon_toolchain::Runtime;
@@ -6,12 +7,15 @@ use serde::Serialize;
 use std::hash::{Hash, Hasher};
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
-pub struct RuntimeNode {
+pub struct SetupToolchainNode {
     pub runtime: Runtime,
 }
 
-pub type InstallWorkspaceDepsNode = RuntimeNode;
-pub type SetupToolchainNode = RuntimeNode;
+#[derive(Clone, Debug, Eq, PartialEq, Serialize)]
+pub struct InstallWorkspaceDepsNode {
+    pub runtime: Runtime,
+    pub root: WorkspaceRelativePathBuf,
+}
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
 pub struct ScopedRuntimeNode {
@@ -156,7 +160,15 @@ impl ActionNode {
     pub fn label(&self) -> String {
         match self {
             Self::InstallWorkspaceDeps(inner) => {
-                format!("InstallWorkspaceDeps({})", inner.runtime.target())
+                if inner.root.as_str().is_empty() {
+                    format!("InstallWorkspaceDeps({})", inner.runtime.target())
+                } else {
+                    format!(
+                        "InstallWorkspaceDeps({}, {})",
+                        inner.runtime.target(),
+                        inner.root
+                    )
+                }
             }
             Self::InstallProjectDeps(inner) => {
                 format!(

diff --git a/crates/actions/src/actions/install_deps.rs b/crates/actions/src/actions/install_deps.rs
@@ -3,7 +3,7 @@ use moon_action::{Action, ActionStatus, Operation};
 use moon_action_context::ActionContext;
 use moon_app_context::AppContext;
 use moon_cache_item::cache_item;
-use moon_common::path::encode_component;
+use moon_common::path::{encode_component, WorkspaceRelativePath};
 use moon_common::{color, is_ci, Id};
 use moon_platform::{BoxedPlatform, PlatformManager, Runtime};
 use moon_project::Project;
@@ -30,6 +30,7 @@ pub async fn install_deps(
     workspace_graph: WorkspaceGraph,
     runtime: &Runtime,
     project: Option<&Project>,
+    packages_root: Option<&WorkspaceRelativePath>,
 ) -> miette::Result<ActionStatus> {
     if runtime.is_system() {
         return Ok(ActionStatus::Skipped);
@@ -123,7 +124,13 @@ pub async fn install_deps(
             .as_ref()
             .is_some_and(|hash| hash != &state.data.last_hash)
     {
-        let working_dir = project.map_or(&app_context.workspace_root, |proj| &proj.root);
+        let working_dir = match project {
+            Some(proj) => proj.root.clone(),
+            None => match packages_root {
+                Some(pr) => pr.to_logical_path(&app_context.workspace_root),
+                None => app_context.workspace_root.clone(),
+            },
+        };
 
         // To avoid nested installs caused by child processes, we set this environment
         // variable with the current process ID and compare against it. If the IDs are
@@ -134,12 +141,12 @@ pub async fn install_deps(
         debug!(
             "Installing {} dependencies in {}",
             log_label,
-            color::path(working_dir)
+            color::path(&working_dir)
         );
 
         action.operations.extend(
             platform
-                .install_deps(&action_context, runtime, working_dir)
+                .install_deps(&action_context, runtime, &working_dir)
                 .await?,
         );
 

diff --git a/crates/cli/tests/run_bun_test.rs b/crates/cli/tests/run_bun_test.rs
@@ -1,39 +1,37 @@
-use moon_config::PartialBunConfig;
-use moon_test_utils::{
-    assert_snapshot, create_sandbox, create_sandbox_with_config, get_bun_fixture_configs,
-    predicates::prelude::*, Sandbox,
-};
-use std::fs;
-
-fn bun_sandbox() -> Sandbox {
-    bun_sandbox_with_config(|_| {})
-}
-
-fn bun_sandbox_with_config<C>(callback: C) -> Sandbox
-where
-    C: FnOnce(&mut PartialBunConfig),
-{
-    let (workspace_config, mut toolchain_config, tasks_config) = get_bun_fixture_configs();
-
-    if let Some(bun_config) = &mut toolchain_config.bun {
-        callback(bun_config);
+// These are very flaky in CI, as they error with "Text file busy" nonstop
+#[cfg(not(target_os = "linux"))]
+mod bun {
+    use moon_config::PartialBunConfig;
+    use moon_test_utils::{
+        assert_snapshot, create_sandbox, create_sandbox_with_config, get_bun_fixture_configs,
+        predicates::prelude::*, Sandbox,
+    };
+    use std::fs;
+
+    fn bun_sandbox() -> Sandbox {
+        bun_sandbox_with_config(|_| {})
     }
 
-    let sandbox = create_sandbox_with_config(
-        "bun",
-        Some(workspace_config),
-        Some(toolchain_config),
-        Some(tasks_config),
-    );
+    fn bun_sandbox_with_config<C>(callback: C) -> Sandbox
+    where
+        C: FnOnce(&mut PartialBunConfig),
+    {
+        let (workspace_config, mut toolchain_config, tasks_config) = get_bun_fixture_configs();
 
-    sandbox.enable_git();
-    sandbox
-}
+        if let Some(bun_config) = &mut toolchain_config.bun {
+            callback(bun_config);
+        }
 
-// These are very flaky in CI, as they error with "Text file busy" nonstop
-#[cfg(not(target_os = "linux"))]
-mod bun {
-    use super::*;
+        let sandbox = create_sandbox_with_config(
+            "bun",
+            Some(workspace_config),
+            Some(toolchain_config),
+            Some(tasks_config),
+        );
+
+        sandbox.enable_git();
+        sandbox
+    }
 
     #[test]
     fn runs_self() {

diff --git a/crates/config/src/config_finder.rs b/crates/config/src/config_finder.rs
@@ -4,7 +4,9 @@ use std::fs;
 use std::path::{Path, PathBuf};
 
 #[derive(Clone, Debug, Default)]
-pub struct ConfigFinder;
+pub struct ConfigFinder {
+    _ignored: bool,
+}
 
 impl ConfigFinder {
     pub fn get_project_files(&self, project_root: &Path) -> Vec<PathBuf> {
@@ -84,6 +86,7 @@ impl ConfigFinder {
         vec![format!("{name}.yml"), format!("{name}.pkl")]
     }
 
+    #[allow(clippy::only_used_in_recursion)]
     pub fn get_from_dir(&self, dir: PathBuf) -> miette::Result<Vec<PathBuf>> {
         let mut files = vec![];