feat(core): update additional project directories handling

Refine workspace file handling and path utilities to properly support
additional project directories throughout the native and TypeScript layers.

Author: FrozenPandaz

packages/devkit/src/utils/calculate-hash-for-create-nodes.ts

@@ -14,11 +14,10 @@ export async function calculateHashForCreateNodes(
   additionalGlobs: string[] = []
 ): Promise<string> {
   return hashArray([
-    await hashWithWorkspaceContext(
-      context.workspaceRoot,
-      context.nxJsonConfiguration.additionalProjectDirectories ?? [],
-      [join(projectRoot, '**/*'), ...additionalGlobs]
-    ),
+    await hashWithWorkspaceContext(context.workspaceRoot, [
+      join(projectRoot, '**/*'),
+      ...additionalGlobs,
+    ]),
     hashObject(options),
   ]);
 }

packages/devkit/src/utils/replace-project-configuration-with-plugin.ts

@@ -9,8 +9,11 @@ import {
   updateNxJson,
   updateProjectConfiguration,
 } from 'nx/src/devkit-exports';
-import { findProjectForPath, hashObject } from 'nx/src/devkit-internals';
-import { multiGlobInAdditionalProjectDirectories } from 'nx/src/native';
+import {
+  findProjectForPath,
+  hashObject,
+  multiGlobInAdditionalProjectDirectories,
+} from 'nx/src/devkit-internals';
 
 export async function replaceProjectConfigurationsWithPlugin<T = unknown>(
   tree: Tree,

packages/nx/bin/nx.ts

@@ -21,7 +21,6 @@ import { performance } from 'perf_hooks';
 import { setupWorkspaceContext } from '../src/utils/workspace-context';
 import { daemonClient } from '../src/daemon/client/client';
 import { removeDbConnections } from '../src/utils/db-connection';
-import { readNxJson } from '../src/config/nx-json';
 
 async function main() {
   if (

packages/nx/src/daemon/server/server.ts

@@ -142,7 +142,6 @@ import {
   REGISTER_PROJECT_GRAPH_LISTENER,
 } from '../message-types/register-project-graph-listener';
 import { deserialize, serialize } from 'v8';
-import { readNxJson } from '../../config/nx-json';
 
 let performanceObserver: PerformanceObserver | undefined;
 let workspaceWatcherError: Error | undefined;

packages/nx/src/devkit-internals.ts

@@ -35,6 +35,7 @@ export {
   createProjectRootMappingsFromProjectConfigurations,
   findProjectForPath,
 } from './project-graph/utils/find-project-for-path';
+export { multiGlobInAdditionalProjectDirectories } from './native';
 export { retrieveProjectConfigurations } from './project-graph/utils/retrieve-workspace-files';
 export { LoadedNxPlugin } from './project-graph/plugins/loaded-nx-plugin';
 export * from './project-graph/error-types';

packages/nx/src/hasher/hash-plan-inspector.ts

@@ -44,6 +44,7 @@ export class HashPlanInspector {
       false
     );
     this.inspector = new NativeHashPlanInspector(
+      workspaceRoot,
       externalReferences.allWorkspaceFiles,
       this.projectGraphRef,
       externalReferences.projectFiles

packages/nx/src/native/index.d.ts

@@ -43,7 +43,7 @@ export declare class FileLock {
 }
 
 export declare class HashPlanInspector {
-  constructor(allWorkspaceFiles: ExternalObject<Array<FileData>>, projectGraph: ExternalObject<ProjectGraph>, projectFileMap: ExternalObject<Record<string, Array<FileData>>>)
+  constructor(workspaceRoot: string, allWorkspaceFiles: ExternalObject<Array<FileData>>, projectGraph: ExternalObject<ProjectGraph>, projectFileMap: ExternalObject<Record<string, Array<FileData>>>)
   inspect(hashPlans: ExternalObject<Record<string, Array<HashInstruction>>>): Record<string, string[]>
 }
 
@@ -136,7 +136,7 @@ export declare class Watcher {
 export declare class WorkspaceContext {
   workspaceRoot: string
   constructor(workspaceRoot: string, cacheDir: string)
-  getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
+  getWorkspaceFiles(additionalProjectDirectories: Array<string>, projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
   /**
    * Performs multiple glob pattern matches against workspace files in parallel

packages/nx/src/native/tasks/hash_plan_inspector.rs

@@ -9,6 +9,7 @@ use std::collections::HashMap;
 
 #[napi]
 pub struct HashPlanInspector {
+    workspace_root: String,
     all_workspace_files: External<Vec<FileData>>,
     project_graph: External<ProjectGraph>,
     project_file_map: External<HashMap<String, Vec<FileData>>>,
@@ -18,11 +19,13 @@ pub struct HashPlanInspector {
 impl HashPlanInspector {
     #[napi(constructor)]
     pub fn new(
+        workspace_root: String,
         all_workspace_files: External<Vec<FileData>>,
         project_graph: External<ProjectGraph>,
         project_file_map: External<HashMap<String, Vec<FileData>>>,
     ) -> Self {
         Self {
+            workspace_root,
             all_workspace_files,
             project_graph,
             project_file_map,
@@ -58,6 +61,7 @@ impl HashPlanInspector {
                         .ok_or_else(|| anyhow!("project {} not found", project_name))?;
 
                     let files = collect_project_files(
+                        &self.workspace_root,
                         project_name,
                         &project.root,
                         file_sets,

packages/nx/src/native/tasks/hashers/hash_project_files.rs

@@ -1,20 +1,27 @@
-use std::collections::HashMap;
-
 use anyhow::*;
+use std::collections::HashMap;
+use std::path::PathBuf;
 use tracing::{trace, trace_span};
 
 use crate::native::glob::build_glob_set;
 use crate::native::types::FileData;
+use crate::native::utils::path::get_relative_path;
 
 pub fn hash_project_files(
+    workspace_root: &String,
     project_name: &str,
     project_root: &str,
     file_sets: &[String],
     project_file_map: &HashMap<String, Vec<FileData>>,
 ) -> Result<String> {
     let _span = trace_span!("hash_project_files", project_name).entered();
-    let collected_files =
-        collect_project_files(project_name, project_root, file_sets, project_file_map)?;
+    let collected_files = collect_project_files(
+        workspace_root,
+        project_name,
+        project_root,
+        file_sets,
+        project_file_map,
+    )?;
     trace!("collected_files: {:?}", collected_files.len());
     let mut hasher = xxhash_rust::xxh3::Xxh3::new();
     for file in collected_files {
@@ -26,6 +33,7 @@ pub fn hash_project_files(
 
 /// base function that should be testable (to make sure that we're getting the proper files back)
 pub fn collect_project_files<'a>(
+    workspace_root: &String,
     project_name: &str,
     project_root: &str,
     file_sets: &[String],
@@ -52,7 +60,15 @@ pub fn collect_project_files<'a>(
             let now = std::time::Instant::now();
             let hashes = files
                 .iter()
-                .filter(|file| glob_set.is_match(&file.file))
+                .filter(|file| {
+                    let path = if PathBuf::from(&file.file).is_absolute() {
+                        &get_relative_path(workspace_root, &file.file)
+                    } else {
+                        &file.file
+                    };
+                    trace!("{} - {}", path, file.hash);
+                    glob_set.is_match(path)
+                })
                 .collect::<Vec<_>>();
             trace!("hash_files for {}: {:?}", project_name, now.elapsed());
             Ok(hashes)
@@ -68,6 +84,7 @@ mod tests {
 
     #[test]
     fn test_collect_files() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         let proj_root = "test/root";
         let file_sets = &[
@@ -101,11 +118,14 @@ mod tests {
             ],
         );
 
-        let result = collect_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let result =
+            collect_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
 
         assert_eq!(result, vec![&tsfile_1, &tsfile_2]);
 
         let result = collect_project_files(
+            &workspace_root,
             proj_name,
             proj_root,
             &["!{projectRoot}/**/*.spec.ts".into()],
@@ -124,6 +144,7 @@ mod tests {
 
     #[test]
     fn should_hash_deterministically() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         let proj_root = "test/root";
         let file_sets = &[
@@ -156,7 +177,9 @@ mod tests {
                 file_data4.clone(),
             ],
         );
-        let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let hash_result =
+            hash_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
         assert_eq!(
             hash_result,
             hash(
@@ -173,6 +196,7 @@ mod tests {
 
     #[test]
     fn should_hash_projects_with_root_as_dot() {
+        let workspace_root = String::from("");
         let proj_name = "test_project";
         // having "." as the project root means that this would be a standalone project
         let proj_root = ".";
@@ -206,7 +230,9 @@ mod tests {
                 file_data4.clone(),
             ],
         );
-        let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap();
+        let hash_result =
+            hash_project_files(&workspace_root, proj_name, proj_root, file_sets, &file_map)
+                .unwrap();
         assert_eq!(
             hash_result,
             hash(

packages/nx/src/native/tasks/task_hasher.rs

@@ -220,6 +220,7 @@ impl TaskHasher {
                     .get(project_name)
                     .ok_or_else(|| anyhow!("project {} not found", project_name))?;
                 let hashed_project_files = hash_project_files(
+                    &self.workspace_root,
                     project_name,
                     &project.root,
                     file_sets,