feat(core): add support for additional project directories

packages/devkit/src/utils/replace-project-configuration-with-plugin.ts

@@ -9,7 +9,11 @@ import {
   updateNxJson,
   updateProjectConfiguration,
 } from 'nx/src/devkit-exports';
-import { findProjectForPath, hashObject } from 'nx/src/devkit-internals';
+import {
+  findProjectForPath,
+  hashObject,
+  multiGlobInAdditionalProjectDirectories,
+} from 'nx/src/devkit-internals';
 
 export async function replaceProjectConfigurationsWithPlugin<T = unknown>(
   tree: Tree,
@@ -37,9 +41,16 @@ export async function replaceProjectConfigurationsWithPlugin<T = unknown>(
   const [pluginGlob, createNodesFunction] = createNodes;
   const configFiles = glob(tree, [pluginGlob]);
 
+  const additionalProjectConfigurationFiles =
+    multiGlobInAdditionalProjectDirectories(
+      tree.root,
+      nxJson.additionalProjectDirectories ?? [],
+      [pluginGlob]
+    )[0];
   const results = await createNodesFunction(configFiles, pluginOptions, {
     workspaceRoot: tree.root,
     nxJsonConfiguration: readNxJson(tree),
+    additionalProjectConfigurationFiles,
   });
 
   for (const [configFile, nodes] of results) {

packages/nx/bin/post-install.ts

@@ -1,15 +1,12 @@
-import { buildProjectGraphAndSourceMapsWithoutDaemon } from '../src/project-graph/project-graph';
 import { workspaceRoot } from '../src/utils/workspace-root';
 import { fileExists } from '../src/utils/fileutils';
 import { join } from 'path';
-import { daemonClient } from '../src/daemon/client/client';
 import { assertSupportedPlatform } from '../src/native/assert-supported-platform';
 import { verifyOrUpdateNxCloudClient } from '../src/nx-cloud/update-manager';
 import { getCloudOptions } from '../src/nx-cloud/utilities/get-cloud-options';
 import { isNxCloudUsed } from '../src/utils/nx-cloud-utils';
 import { readNxJson } from '../src/config/nx-json';
 import { logger } from '../src/utils/logger';
-import { setupWorkspaceContext } from '../src/utils/workspace-context';
 
 // The post install is not critical, to avoid any chance that it may hang
 // we will kill this process after 30 seconds.

packages/nx/src/adapter/compat.ts

@@ -87,6 +87,7 @@ export const allowedWorkspaceExtensions = [
   'maxCacheSize',
   'tui',
   'owners',
+  'additionalProjectDirectories',
 ] as const;
 
 if (!patched) {

packages/nx/src/config/nx-json.ts

@@ -855,6 +855,11 @@ export interface NxJsonConfiguration<T = '*' | string[]> {
    */
   sync?: NxSyncConfiguration;
 
+  /**
+   * Additional directories to discover projects within relative to the nx.json
+   */
+  additionalProjectDirectories?: string[];
+
   /**
    * Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
    */

packages/nx/src/devkit-internals.ts

@@ -35,6 +35,7 @@ export {
   createProjectRootMappingsFromProjectConfigurations,
   findProjectForPath,
 } from './project-graph/utils/find-project-for-path';
+export { multiGlobInAdditionalProjectDirectories } from './native';
 export { retrieveProjectConfigurations } from './project-graph/utils/retrieve-workspace-files';
 export { LoadedNxPlugin } from './project-graph/plugins/loaded-nx-plugin';
 export * from './project-graph/error-types';

packages/nx/src/hasher/hash-plan-inspector.ts

@@ -44,6 +44,7 @@ export class HashPlanInspector {
       false
     );
     this.inspector = new NativeHashPlanInspector(
+      workspaceRoot,
       externalReferences.allWorkspaceFiles,
       this.projectGraphRef,
       externalReferences.projectFiles

packages/nx/src/native/index.d.ts

@@ -43,7 +43,7 @@ export declare class FileLock {
 }
 
 export declare class HashPlanInspector {
-  constructor(allWorkspaceFiles: ExternalObject<Array<FileData>>, projectGraph: ExternalObject<ProjectGraph>, projectFileMap: ExternalObject<Record<string, Array<FileData>>>)
+  constructor(workspaceRoot: string, allWorkspaceFiles: ExternalObject<Array<FileData>>, projectGraph: ExternalObject<ProjectGraph>, projectFileMap: ExternalObject<Record<string, Array<FileData>>>)
   inspect(hashPlans: ExternalObject<Record<string, Array<HashInstruction>>>): Record<string, string[]>
 }
 
@@ -136,7 +136,7 @@ export declare class Watcher {
 export declare class WorkspaceContext {
   workspaceRoot: string
   constructor(workspaceRoot: string, cacheDir: string)
-  getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
+  getWorkspaceFiles(additionalProjectDirectories: Array<string>, projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
   /**
    * Performs multiple glob pattern matches against workspace files in parallel
@@ -264,6 +264,10 @@ export declare export declare function isEditorInstalled(editor: SupportedEditor
 
 export declare export declare function logDebug(message: string): void
 
+export declare export declare function multiGlobInAdditionalProjectDirectories(workspaceRoot: string, additionalProjectDirectories: Array<string>, globs: Array<string>, exclude?: Array<string> | undefined | null): Array<Array<string>>
+
+export declare export declare function multiHashGlobInAdditionalProjectDirectories(workspaceRoot: string, additionalProjectDirectories: Array<string>, globGroups: Array<Array<string>>): Array<string>
+
 /** Stripped version of the NxJson interface for use in rust */
 export interface NxJson {
   namedInputs?: Record<string, Array<JsInputs>>

packages/nx/src/native/native-bindings.js

@@ -397,6 +397,8 @@ module.exports.IS_WASM = nativeBinding.IS_WASM
 module.exports.isAiAgent = nativeBinding.isAiAgent
 module.exports.isEditorInstalled = nativeBinding.isEditorInstalled
 module.exports.logDebug = nativeBinding.logDebug
+module.exports.multiGlobInAdditionalProjectDirectories = nativeBinding.multiGlobInAdditionalProjectDirectories
+module.exports.multiHashGlobInAdditionalProjectDirectories = nativeBinding.multiHashGlobInAdditionalProjectDirectories
 module.exports.parseTaskStatus = nativeBinding.parseTaskStatus
 module.exports.remove = nativeBinding.remove
 module.exports.restoreTerminal = nativeBinding.restoreTerminal

packages/nx/src/native/tasks/hash_plan_inspector.rs

@@ -9,6 +9,7 @@ use std::collections::HashMap;
 
 #[napi]
 pub struct HashPlanInspector {
+    workspace_root: String,
     all_workspace_files: External<Vec<FileData>>,
     project_graph: External<ProjectGraph>,
     project_file_map: External<HashMap<String, Vec<FileData>>>,
@@ -18,11 +19,13 @@ pub struct HashPlanInspector {
 impl HashPlanInspector {
     #[napi(constructor)]
     pub fn new(
+        workspace_root: String,
         all_workspace_files: External<Vec<FileData>>,
         project_graph: External<ProjectGraph>,
         project_file_map: External<HashMap<String, Vec<FileData>>>,
     ) -> Self {
         Self {
+            workspace_root,
             all_workspace_files,
             project_graph,
             project_file_map,
@@ -58,6 +61,7 @@ impl HashPlanInspector {
                         .ok_or_else(|| anyhow!("project {} not found", project_name))?;
 
                     let files = collect_project_files(
+                        &self.workspace_root,
                         project_name,
                         &project.root,
                         file_sets,

packages/nx/src/native/tasks/hashers/hash_project_files.rs

@@ -1,20 +1,27 @@
-use std::collections::HashMap;
-
 use anyhow::*;
+use std::collections::HashMap;
+use std::path::PathBuf;
 use tracing::{trace, trace_span};
 
 use crate::native::glob::build_glob_set;
 use crate::native::types::FileData;
+use crate::native::utils::path::get_relative_path;
 
 pub fn hash_project_files(
+    workspace_root: &String,
     project_name: &str,
     project_root: &str,
     file_sets: &[String],
     project_file_map: &HashMap<String, Vec<FileData>>,
 ) -> Result<String> {
     let _span = trace_span!("hash_project_files", project_name).entered();
-    let collected_files =
-        collect_project_files(project_name, project_root, file_sets, project_file_map)?;
+    let collected_files = collect_project_files(
+        workspace_root,
+        project_name,
+        project_root,
+        file_sets,
+        project_file_map,
+    )?;
     trace!("collected_files: {:?}", collected_files.len());
     let mut hasher = xxhash_rust::xxh3::Xxh3::new();
     for file in collected_files {
@@ -26,6 +33,7 @@ pub fn hash_project_files(
 
 /// base function that should be testable (to make sure that we're getting the proper files back)
 pub fn collect_project_files<'a>(
+    workspace_root: &String,
     project_name: &str,
     project_root: &str,
     file_sets: &[String],
@@ -52,7 +60,15 @@ pub fn collect_project_files<'a>(
             let now = std::time::Instant::now();
             let hashes = files
                 .iter()
-                .filter(|file| glob_set.is_match(&file.file))
+                .filter(|file| {
+                    let path = if PathBuf::from(&file.file).is_absolute() {
+                        &get_relative_path(workspace_root, &file.file)
+                    } else {
+                        &file.file
+                    };
+                    trace!("{} - {}", path, file.hash);
+                    glob_set.is_match(path)
+                })
                 .collect::<Vec<_>>();
             trace!("hash_files for {}: {:?}", project_name, now.elapsed());
             Ok(hashes)
@@ -68,6 +84,7 @@ mod tests {
 
     #[test]
     fn test_collect_files() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         let proj_root = "test/root";
         let file_sets = &[
@@ -101,11 +118,14 @@ mod tests {
             ],
         );
 
-        let result = collect_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let result =
+            collect_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
 
         assert_eq!(result, vec![&tsfile_1, &tsfile_2]);
 
         let result = collect_project_files(
+            &workspace_root,
             proj_name,
             proj_root,
             &["!{projectRoot}/**/*.spec.ts".into()],
@@ -124,6 +144,7 @@ mod tests {
 
     #[test]
     fn should_hash_deterministically() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         let proj_root = "test/root";
         let file_sets = &[
@@ -156,7 +177,9 @@ mod tests {
                 file_data4.clone(),
             ],
         );
-        let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let hash_result =
+            hash_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
         assert_eq!(
             hash_result,
             hash(
@@ -173,6 +196,7 @@ mod tests {
 
     #[test]
     fn should_hash_projects_with_root_as_dot() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         // having "." as the project root means that this would be a standalone project
         let proj_root = ".";
@@ -206,7 +230,9 @@ mod tests {
                 file_data4.clone(),
             ],
         );
-        let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let hash_result =
+            hash_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
         assert_eq!(
             hash_result,
             hash(

packages/nx/src/native/tasks/task_hasher.rs

@@ -220,6 +220,7 @@ impl TaskHasher {
                     .get(project_name)
                     .ok_or_else(|| anyhow!("project {} not found", project_name))?;
                 let hashed_project_files = hash_project_files(
+                    &self.workspace_root,
                     project_name,
                     &project.root,
                     file_sets,

packages/nx/src/native/tests/workspace_files.spec.ts

@@ -54,7 +54,7 @@ describe('Workspace Context', () => {
       fs.tempDir,
       cacheDirectoryForWorkspace(fs.tempDir)
     );
-    let { projectFileMap, globalFiles } = await context.getWorkspaceFiles({
+    let { projectFileMap, globalFiles } = await context.getWorkspaceFiles([], {
       'libs/project1': 'project1',
       'libs/project2': 'project2',
       'libs/project3': 'project3',
@@ -156,9 +156,12 @@ describe('Workspace Context', () => {
       cacheDirectoryForWorkspace(fs.tempDir)
     );
 
-    const { globalFiles, projectFileMap } = await context.getWorkspaceFiles({
-      '.': 'repo-name',
-    });
+    const { globalFiles, projectFileMap } = await context.getWorkspaceFiles(
+      [],
+      {
+        '.': 'repo-name',
+      }
+    );
 
     expect(globalFiles).toEqual([]);
     expect(projectFileMap['repo-name']).toMatchInlineSnapshot(`