feat(core): add support for additional project directories

Add native workspace support for additional project roots alongside the main workspace root. This enables workspaces to define multiple project directories for better organization and flexibility.

Changes include:
- New Rust module for handling additional project directories
- Updated workspace context to support multiple project roots
- Extended project graph utilities to recognize additional directories
- Updated package.json plugin creation with new API

Author: FrozenPandaz

packages/devkit/src/utils/replace-project-configuration-with-plugin.ts

@@ -10,6 +10,7 @@ import {
   updateProjectConfiguration,
 } from 'nx/src/devkit-exports';
 import { findProjectForPath, hashObject } from 'nx/src/devkit-internals';
+import { multiGlobInAdditionalProjectDirectories } from 'nx/src/native';
 
 export async function replaceProjectConfigurationsWithPlugin<T = unknown>(
   tree: Tree,
@@ -37,9 +38,16 @@ export async function replaceProjectConfigurationsWithPlugin<T = unknown>(
   const [pluginGlob, createNodesFunction] = createNodes;
   const configFiles = glob(tree, [pluginGlob]);
 
+  const additionalProjectConfigurationFiles =
+    multiGlobInAdditionalProjectDirectories(
+      tree.root,
+      nxJson.additionalProjectDirectories ?? [],
+      [pluginGlob]
+    )[0];
   const results = await createNodesFunction(configFiles, pluginOptions, {
     workspaceRoot: tree.root,
     nxJsonConfiguration: readNxJson(tree),
+    additionalProjectConfigurationFiles,
   });
 
   for (const [configFile, nodes] of results) {

packages/nx/bin/nx.ts

@@ -59,10 +59,8 @@ async function main() {
     process.env.NX_DAEMON = 'false';
     require('nx/src/command-line/nx-commands').commandsObject.argv;
   } else {
-    const additionalProjectDirectories =
-      readNxJson(workspace.dir).additionalProjectDirectories ?? [];
     if (!daemonClient.enabled() && workspace !== null) {
-      setupWorkspaceContext(workspace.dir, additionalProjectDirectories);
+      setupWorkspaceContext(workspace.dir);
     }
 
     // polyfill rxjs observable to avoid issues with multiple version of Observable installed in node_modules

packages/nx/src/daemon/server/server.ts

@@ -650,9 +650,7 @@ const handleOutputsChanges: FileWatcherCallback = async (err, changeEvents) => {
 };
 
 export async function startServer(): Promise<Server> {
-  const additionalProjectDirectories =
-    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
-  setupWorkspaceContext(workspaceRoot, additionalProjectDirectories);
+  setupWorkspaceContext(workspaceRoot);
 
   // Persist metadata about the background process so that it can be cleaned up later if needed
   await writeDaemonJsonProcessCache({

packages/nx/src/native/index.d.ts

@@ -135,7 +135,7 @@ export declare class Watcher {
 
 export declare class WorkspaceContext {
   workspaceRoot: string
-  constructor(workspaceRoot: string, additionalProjectDirectories: Array<string>, cacheDir: string)
+  constructor(workspaceRoot: string, cacheDir: string)
   getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
   /**
@@ -264,6 +264,10 @@ export declare export declare function isEditorInstalled(editor: SupportedEditor
 
 export declare export declare function logDebug(message: string): void
 
+export declare export declare function multiGlobInAdditionalProjectDirectories(workspaceRoot: string, additionalProjectDirectories: Array<string>, globs: Array<string>, exclude?: Array<string> | undefined | null): Array<Array<string>>
+
+export declare export declare function multiHashGlobInAdditionalProjectDirectories(workspaceRoot: string, additionalProjectDirectories: Array<string>, globGroups: Array<Array<string>>): Array<string>
+
 /** Stripped version of the NxJson interface for use in rust */
 export interface NxJson {
   namedInputs?: Record<string, Array<JsInputs>>

packages/nx/src/native/native-bindings.js

@@ -397,6 +397,8 @@ module.exports.IS_WASM = nativeBinding.IS_WASM
 module.exports.isAiAgent = nativeBinding.isAiAgent
 module.exports.isEditorInstalled = nativeBinding.isEditorInstalled
 module.exports.logDebug = nativeBinding.logDebug
+module.exports.multiGlobInAdditionalProjectDirectories = nativeBinding.multiGlobInAdditionalProjectDirectories
+module.exports.multiHashGlobInAdditionalProjectDirectories = nativeBinding.multiHashGlobInAdditionalProjectDirectories
 module.exports.parseTaskStatus = nativeBinding.parseTaskStatus
 module.exports.remove = nativeBinding.remove
 module.exports.restoreTerminal = nativeBinding.restoreTerminal

packages/nx/src/native/workspace/additional_project_directories.rs

@@ -0,0 +1,104 @@
+use std::path::{Path, PathBuf, Component};
+
+use crate::native::glob::glob_files::glob_files;
+use crate::native::types::FileData;
+use crate::native::walker::{nx_walker, NxFile};
+use rayon::prelude::*;
+use xxhash_rust::xxh3;
+
+fn join_paths(base: &Path, relative: &str) -> PathBuf {
+    let mut path = base.to_path_buf();
+
+    for component in Path::new(relative).components() {
+        match component {
+            Component::ParentDir => {
+                path.pop();
+            }
+            Component::CurDir => {}
+            Component::Normal(c) => path.push(c),
+            _ => {}
+        }
+    }
+
+    path
+}
+
+pub fn get_files_in_additional_project_directories(
+    workspace_root: String,
+    additional_project_directories: Vec<String>,
+) -> Vec<Vec<NxFile>> {
+    let root_path = PathBuf::from(&workspace_root);
+    additional_project_directories
+        .into_par_iter()
+        .map(|additional_project_directory| {
+            let joined_path = join_paths(&root_path, &additional_project_directory);
+            let full_path = joined_path.to_string_lossy().to_string();
+            nx_walker(&full_path, true).collect()
+        })
+        .collect()
+}
+
+#[napi]
+pub fn multi_glob_in_additional_project_directories(
+    workspace_root: String,
+    additional_project_directories: Vec<String>,
+    globs: Vec<String>,
+    exclude: Option<Vec<String>>,
+) -> napi::Result<Vec<Vec<String>>> {
+    let files = get_files_in_additional_project_directories(workspace_root, additional_project_directories);
+
+    globs
+        .iter()
+        .map(|glob| -> napi::Result<Vec<String>> {
+            let mut result = Vec::new();
+            for dir_files in &files {
+                let file_data: Vec<FileData> = dir_files
+                    .iter()
+                    .map(|f| FileData {
+                        file: f.full_path.clone(),
+                        hash: String::new(),
+                    })
+                    .collect();
+                let globbed_files: Vec<String> = glob_files(&file_data, vec![glob.clone()], exclude.clone())?
+                    .map(|f| f.file.to_owned())
+                    .collect();
+                result.extend(globbed_files);
+            }
+            Ok(result)
+        })
+        .collect()
+}
+
+#[napi]
+pub fn multi_hash_glob_in_additional_project_directories(
+    workspace_root: String,
+    additional_project_directories: Vec<String>,
+    glob_groups: Vec<Vec<String>>,
+) -> napi::Result<Vec<String>> {
+    let files = get_files_in_additional_project_directories(workspace_root, additional_project_directories);
+
+    glob_groups
+        .iter()
+        .map(|glob_group| -> napi::Result<String> {
+            let mut hasher = xxh3::Xxh3::new();
+            for dir_files in &files {
+                let file_data: Vec<FileData> = dir_files
+                    .iter()
+                    .map(|f| FileData {
+                        file: f.full_path.clone(),
+                        hash: String::new(),
+                    })
+                    .collect();
+
+                let globbed_files: Vec<_> =
+                    glob_files(&file_data, glob_group.clone(), None)?.collect();
+                for file in globbed_files {
+                    hasher.update(file.file.as_bytes());
+                    hasher.update(file.hash.as_bytes());
+                }
+            }
+
+            Ok(hasher.digest().to_string())
+        })
+        .collect()
+}

packages/nx/src/native/workspace/context.rs

@@ -30,15 +30,15 @@ pub struct WorkspaceContext {
 
 type Files = Vec<(PathBuf, String)>;
 
-fn gather_and_hash_files(directory: &Path, cache_dir: &String) -> Vec<(PathBuf, String)> {
-    let archived_files = read_files_archive(cache_dir);
+fn gather_and_hash_files(workspace_root: &Path, cache_dir: String) -> Vec<(PathBuf, String)> {
+    let archived_files = read_files_archive(&cache_dir);
 
-    trace!("Gathering files in {}", directory.display());
+    trace!("Gathering files in {}", workspace_root.display());
     let now = std::time::Instant::now();
     let file_hashes = if let Some(archived_files) = archived_files {
-        selective_files_hash(directory, archived_files)
+        selective_files_hash(workspace_root, archived_files)
     } else {
-        full_files_hash(directory)
+        full_files_hash(workspace_root)
     };
 
     let mut files = file_hashes
@@ -57,11 +57,7 @@ fn gather_and_hash_files(directory: &Path, cache_dir: &String) -> Vec<(PathBuf,
 struct FilesWorker(Option<Arc<(NxMutex<Files>, NxCondvar)>>);
 impl FilesWorker {
     #[cfg(not(target_arch = "wasm32"))]
-    fn gather_files(
-        workspace_root: &Path,
-        additional_project_directories: Vec<PathBuf>,
-        cache_dir: String,
-    ) -> Self {
+    fn gather_files(workspace_root: &Path, cache_dir: String) -> Self {
         if !workspace_root.exists() {
             warn!(
                 "workspace root does not exist: {}",
@@ -79,14 +75,7 @@ impl FilesWorker {
             trace!("Initially locking files");
             let mut workspace_files = lock.lock().expect("Should be the first time locking files");
 
-            let mut files = gather_and_hash_files(&workspace_root, &cache_dir);
-
-            for additional_project_directory in additional_project_directories {
-                let additional_files =
-                    gather_and_hash_files(&additional_project_directory, &cache_dir);
-
-                files.extend(additional_files);
-            }
+            let files = gather_and_hash_files(&workspace_root, cache_dir);
 
             *workspace_files = files;
             let files_len = workspace_files.len();
@@ -211,27 +200,15 @@ impl FilesWorker {
 #[napi]
 impl WorkspaceContext {
     #[napi(constructor)]
-    pub fn new(
-        workspace_root: String,
-        additional_project_directories: Vec<String>,
-        cache_dir: String,
-    ) -> Self {
+    pub fn new(workspace_root: String, cache_dir: String) -> Self {
         enable_logger();
 
         trace!(?workspace_root);
 
         let workspace_root_path = PathBuf::from(&workspace_root);
-        let additional_project_directories = additional_project_directories
-            .iter()
-            .map(|s| PathBuf::from(s))
-            .collect::<Vec<PathBuf>>();
 
         WorkspaceContext {
-            files_worker: FilesWorker::gather_files(
-                &workspace_root_path,
-                additional_project_directories,
-                cache_dir.clone(),
-            ),
+            files_worker: FilesWorker::gather_files(&workspace_root_path, cache_dir.clone()),
             workspace_root,
             workspace_root_path,
         }

packages/nx/src/native/workspace/mod.rs

@@ -9,6 +9,7 @@ mod files_archive;
 mod files_hashing;
 pub mod types;
 pub mod workspace_files;
+pub mod additional_project_directories;
 
 #[napi]
 // should only be used in tests to transfer the file map from the JS world to the Rust world

packages/nx/src/plugins/package-json/create-nodes.ts

@@ -1,6 +1,6 @@
 import { minimatch } from 'minimatch';
 import { existsSync } from 'node:fs';
-import { dirname, join } from 'node:path';
+import { dirname, join, basename, resolve, relative } from 'node:path';
 
 import { NxJsonConfiguration, readNxJson } from '../../config/nx-json';
 import type { ProjectConfiguration } from '../../config/workspace-json-project-json';
@@ -21,7 +21,6 @@ import {
   createNodesFromFiles,
   CreateNodesV2,
 } from '../../project-graph/plugins';
-import { basename } from 'path';
 import { hashObject } from '../../hasher/file-hasher';
 import {
   PackageJsonConfigurationCache,
@@ -36,7 +35,9 @@ export const createNodesV2: CreateNodesV2 = [
     '**/project.json'
   ),
   (configFiles, _, context) => {
-    const { packageJsons, projectJsonRoots } = splitConfigFiles(configFiles);
+    const { packageJsons, projectJsonRoots } = splitConfigFiles(
+      configFiles.concat(context.additionalProjectConfigurationFiles)
+    );
 
     const readJson = (f) => readJsonFile(join(context.workspaceRoot, f));
     const isInPackageJsonWorkspaces =
@@ -52,8 +53,9 @@ export const createNodesV2: CreateNodesV2 = [
 
     return createNodesFromFiles(
       (packageJsonPath, options, context) => {
-        const isInPackageManagerWorkspaces =
-          isInPackageJsonWorkspaces(packageJsonPath);
+        const isInPackageManagerWorkspaces = isInPackageJsonWorkspaces(
+          relative(context.workspaceRoot, packageJsonPath)
+        );
         if (
           !isInPackageManagerWorkspaces &&
           !isNextToProjectJson(packageJsonPath)
@@ -137,7 +139,7 @@ export function createNodeFromPackageJson(
   cache: PackageJsonConfigurationCache,
   isInPackageManagerWorkspaces: boolean
 ) {
-  const json: PackageJson = readJsonFile(join(workspaceRoot, pkgJsonPath));
+  const json: PackageJson = readJsonFile(resolve(workspaceRoot, pkgJsonPath));
 
   const projectRoot = dirname(pkgJsonPath);
 
@@ -180,8 +182,9 @@ export function buildProjectConfigurationFromPackageJson(
   nxJson: NxJsonConfiguration,
   isInPackageManagerWorkspaces: boolean
 ): ProjectConfiguration & { name: string } {
-  const normalizedPath = packageJsonPath.split('\\').join('/');
-  const projectRoot = dirname(normalizedPath);
+  const projectRoot = dirname(relative(workspaceRoot, packageJsonPath))
+    .split('\\')
+    .join('/');
 
   const siblingProjectJson = tryReadJson<ProjectConfiguration>(
     join(workspaceRoot, projectRoot, 'project.json')
@@ -208,7 +211,7 @@ export function buildProjectConfigurationFromPackageJson(
     );
   }
 
-  let name = packageJson.name ?? toProjectName(normalizedPath);
+  let name = packageJson.name ?? toProjectName(projectRoot);
 
   const projectConfiguration: ProjectConfiguration & { name: string } = {
     root: projectRoot,

packages/nx/src/project-graph/plugins/public-api.ts

@@ -16,6 +16,7 @@ import type { TaskResults } from '../../tasks-runner/life-cycle';
 export interface CreateNodesContextV2 {
   readonly nxJsonConfiguration: NxJsonConfiguration;
   readonly workspaceRoot: string;
+  readonly additionalProjectConfigurationFiles?: string[];
 }
 
 export type CreateNodesResultV2 = Array<