chore(repo): wip

Author: FrozenPandaz

packages/devkit/src/utils/calculate-hash-for-create-nodes.ts

@@ -14,10 +14,11 @@ export async function calculateHashForCreateNodes(
   additionalGlobs: string[] = []
 ): Promise<string> {
   return hashArray([
-    await hashWithWorkspaceContext(context.workspaceRoot, [
-      join(projectRoot, '**/*'),
-      ...additionalGlobs,
-    ]),
+    await hashWithWorkspaceContext(
+      context.workspaceRoot,
+      context.nxJsonConfiguration.additionalProjectDirectories ?? [],
+      [join(projectRoot, '**/*'), ...additionalGlobs]
+    ),
     hashObject(options),
   ]);
 }

packages/nx/bin/nx.ts

@@ -21,6 +21,7 @@ import { performance } from 'perf_hooks';
 import { setupWorkspaceContext } from '../src/utils/workspace-context';
 import { daemonClient } from '../src/daemon/client/client';
 import { removeDbConnections } from '../src/utils/db-connection';
+import { readNxJson } from '../src/config/nx-json';
 
 async function main() {
   if (
@@ -58,8 +59,10 @@ async function main() {
     process.env.NX_DAEMON = 'false';
     require('nx/src/command-line/nx-commands').commandsObject.argv;
   } else {
+    const additionalProjectDirectories =
+      readNxJson(workspace.dir).additionalProjectDirectories ?? [];
     if (!daemonClient.enabled() && workspace !== null) {
-      setupWorkspaceContext(workspace.dir);
+      setupWorkspaceContext(workspace.dir, additionalProjectDirectories);
     }
 
     // polyfill rxjs observable to avoid issues with multiple version of Observable installed in node_modules

packages/nx/bin/post-install.ts

@@ -1,15 +1,12 @@
-import { buildProjectGraphAndSourceMapsWithoutDaemon } from '../src/project-graph/project-graph';
 import { workspaceRoot } from '../src/utils/workspace-root';
 import { fileExists } from '../src/utils/fileutils';
 import { join } from 'path';
-import { daemonClient } from '../src/daemon/client/client';
 import { assertSupportedPlatform } from '../src/native/assert-supported-platform';
 import { verifyOrUpdateNxCloudClient } from '../src/nx-cloud/update-manager';
 import { getCloudOptions } from '../src/nx-cloud/utilities/get-cloud-options';
 import { isNxCloudUsed } from '../src/utils/nx-cloud-utils';
 import { readNxJson } from '../src/config/nx-json';
 import { logger } from '../src/utils/logger';
-import { setupWorkspaceContext } from '../src/utils/workspace-context';
 
 // The post install is not critical, to avoid any chance that it may hang
 // we will kill this process after 30 seconds.

packages/nx/src/adapter/compat.ts

@@ -87,6 +87,7 @@ export const allowedWorkspaceExtensions = [
   'maxCacheSize',
   'tui',
   'owners',
+  'additionalProjectDirectories',
 ] as const;
 
 if (!patched) {

packages/nx/src/config/nx-json.ts

@@ -854,6 +854,11 @@ export interface NxJsonConfiguration<T = '*' | string[]> {
    */
   sync?: NxSyncConfiguration;
 
+  /**
+   * Additional directories to discover projects within relative to the nx.json
+   */
+  additionalProjectDirectories?: string[];
+
   /**
    * Sets the maximum size of the local cache. Accepts a number followed by a unit (e.g. 100MB). Accepted units are B, KB, MB, and GB.
    */

packages/nx/src/daemon/server/server.ts

@@ -142,6 +142,7 @@ import {
   REGISTER_PROJECT_GRAPH_LISTENER,
 } from '../message-types/register-project-graph-listener';
 import { deserialize, serialize } from 'v8';
+import { readNxJson } from '../../config/nx-json';
 
 let performanceObserver: PerformanceObserver | undefined;
 let workspaceWatcherError: Error | undefined;
@@ -649,7 +650,9 @@ const handleOutputsChanges: FileWatcherCallback = async (err, changeEvents) => {
 };
 
 export async function startServer(): Promise<Server> {
-  setupWorkspaceContext(workspaceRoot);
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
+  setupWorkspaceContext(workspaceRoot, additionalProjectDirectories);
 
   // Persist metadata about the background process so that it can be cleaned up later if needed
   await writeDaemonJsonProcessCache({

packages/nx/src/native/index.d.ts

@@ -135,7 +135,7 @@ export declare class Watcher {
 
 export declare class WorkspaceContext {
   workspaceRoot: string
-  constructor(workspaceRoot: string, cacheDir: string)
+  constructor(workspaceRoot: string, additionalProjectDirectories: Array<string>, cacheDir: string)
   getWorkspaceFiles(projectRootMap: Record<string, string>): NxWorkspaceFiles
   glob(globs: Array<string>, exclude?: Array<string> | undefined | null): Array<string>
   /**

packages/nx/src/native/workspace/context.rs

@@ -30,15 +30,15 @@ pub struct WorkspaceContext {
 
 type Files = Vec<(PathBuf, String)>;
 
-fn gather_and_hash_files(workspace_root: &Path, cache_dir: String) -> Vec<(PathBuf, String)> {
-    let archived_files = read_files_archive(&cache_dir);
+fn gather_and_hash_files(directory: &Path, cache_dir: &String) -> Vec<(PathBuf, String)> {
+    let archived_files = read_files_archive(cache_dir);
 
-    trace!("Gathering files in {}", workspace_root.display());
+    trace!("Gathering files in {}", directory.display());
     let now = std::time::Instant::now();
     let file_hashes = if let Some(archived_files) = archived_files {
-        selective_files_hash(workspace_root, archived_files)
+        selective_files_hash(directory, archived_files)
     } else {
-        full_files_hash(workspace_root)
+        full_files_hash(directory)
     };
 
     let mut files = file_hashes
@@ -57,7 +57,11 @@ fn gather_and_hash_files(workspace_root: &Path, cache_dir: String) -> Vec<(PathB
 struct FilesWorker(Option<Arc<(NxMutex<Files>, NxCondvar)>>);
 impl FilesWorker {
     #[cfg(not(target_arch = "wasm32"))]
-    fn gather_files(workspace_root: &Path, cache_dir: String) -> Self {
+    fn gather_files(
+        workspace_root: &Path,
+        additional_project_directories: Vec<PathBuf>,
+        cache_dir: String,
+    ) -> Self {
         if !workspace_root.exists() {
             warn!(
                 "workspace root does not exist: {}",
@@ -75,7 +79,14 @@ impl FilesWorker {
             trace!("Initially locking files");
             let mut workspace_files = lock.lock().expect("Should be the first time locking files");
 
-            let files = gather_and_hash_files(&workspace_root, cache_dir);
+            let mut files = gather_and_hash_files(&workspace_root, &cache_dir);
+
+            for additional_project_directory in additional_project_directories {
+                let additional_files =
+                    gather_and_hash_files(&additional_project_directory, &cache_dir);
+
+                files.extend(additional_files);
+            }
 
             *workspace_files = files;
             let files_len = workspace_files.len();
@@ -200,15 +211,27 @@ impl FilesWorker {
 #[napi]
 impl WorkspaceContext {
     #[napi(constructor)]
-    pub fn new(workspace_root: String, cache_dir: String) -> Self {
+    pub fn new(
+        workspace_root: String,
+        additional_project_directories: Vec<String>,
+        cache_dir: String,
+    ) -> Self {
         enable_logger();
 
         trace!(?workspace_root);
 
         let workspace_root_path = PathBuf::from(&workspace_root);
+        let additional_project_directories = additional_project_directories
+            .iter()
+            .map(|s| PathBuf::from(s))
+            .collect::<Vec<PathBuf>>();
 
         WorkspaceContext {
-            files_worker: FilesWorker::gather_files(&workspace_root_path, cache_dir.clone()),
+            files_worker: FilesWorker::gather_files(
+                &workspace_root_path,
+                additional_project_directories,
+                cache_dir.clone(),
+            ),
             workspace_root,
             workspace_root_path,
         }

packages/nx/src/project-graph/utils/retrieve-workspace-files.ts

@@ -111,11 +111,13 @@ export async function retrieveProjectConfigurationsWithAngularProjects(
 
 export async function retrieveProjectConfigurationPaths(
   root: string,
+  additionalProjectDirectories: string[],
   plugins: Array<LoadedNxPlugin>
 ): Promise<string[]> {
   const projectGlobPatterns = getGlobPatternsOfPlugins(plugins);
   const pluginConfigFiles = await multiGlobWithWorkspaceContext(
     root,
+    additionalProjectDirectories,
     projectGlobPatterns
   );
   return pluginConfigFiles.flat();
@@ -132,6 +134,8 @@ export async function retrieveProjectConfigurationsWithoutPluginInference(
 ): Promise<Record<string, ProjectConfiguration>> {
   const nxJson = readNxJson(root);
   const plugins = await getOnlyDefaultPlugins(); // only load default plugins
+  const additionalProjectDirectories =
+    nxJson.additionalProjectDirectories ?? [];
   const projectGlobPatterns = getGlobPatternsOfPlugins(plugins);
   const cacheKey = root + ',' + projectGlobPatterns.join(',');
 
@@ -140,7 +144,8 @@ export async function retrieveProjectConfigurationsWithoutPluginInference(
   }
 
   const projectFiles =
-    (await multiGlobWithWorkspaceContext(root, projectGlobPatterns)) ?? [];
+    (await multiGlobWithWorkspaceContext(root,       additionalProjectDirectories,
+      projectGlobPatterns)) ?? [];
   const { projects } = await createProjectConfigurationsWithPlugins(
     root,
     nxJson,

packages/nx/src/utils/workspace-context.ts

@@ -3,15 +3,21 @@ import { performance } from 'perf_hooks';
 import { workspaceDataDirectoryForWorkspace } from './cache-directory';
 import { isOnDaemon } from '../daemon/is-on-daemon';
 import { daemonClient } from '../daemon/client/client';
+import { readNxJson } from '../config/configuration';
+import { resolve } from 'path';
 
 let workspaceContext: WorkspaceContext | undefined;
 
-export function setupWorkspaceContext(workspaceRoot: string) {
+export function setupWorkspaceContext(
+  workspaceRoot: string,
+  additionalProjectDirectories: string[]
+) {
   const { WorkspaceContext } =
     require('../native') as typeof import('../native');
   performance.mark('workspace-context');
   workspaceContext = new WorkspaceContext(
     workspaceRoot,
+    additionalProjectDirectories.map((p) => resolve(workspaceRoot, p)),
     workspaceDataDirectoryForWorkspace(workspaceRoot)
   );
   performance.mark('workspace-context:end');
@@ -27,8 +33,10 @@ export async function getNxWorkspaceFilesFromContext(
   projectRootMap: Record<string, string>,
   useDaemonProcess: boolean = true
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (!useDaemonProcess || isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.getWorkspaceFiles(projectRootMap);
   }
   return daemonClient.getWorkspaceFiles(projectRootMap);
@@ -46,7 +54,9 @@ export function globWithWorkspaceContextSync(
   globs: string[],
   exclude?: string[]
 ) {
-  ensureContextAvailable(workspaceRoot);
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
+  ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
   return workspaceContext.glob(globs, exclude);
 }
 
@@ -55,8 +65,10 @@ export async function globWithWorkspaceContext(
   globs: string[],
   exclude?: string[]
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (workspaceRoot === '/virtual' || isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.glob(globs, exclude);
   } else {
     return daemonClient.glob(globs, exclude);
@@ -68,8 +80,10 @@ export async function multiGlobWithWorkspaceContext(
   globs: string[],
   exclude?: string[]
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.multiGlob(globs, exclude);
   }
   return daemonClient.multiGlob(globs, exclude);
@@ -80,8 +94,10 @@ export async function hashWithWorkspaceContext(
   globs: string[],
   exclude?: string[]
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.hashFilesMatchingGlob(globs, exclude);
   }
   return daemonClient.hashGlob(globs, exclude);
@@ -91,8 +107,10 @@ export async function hashMultiGlobWithWorkspaceContext(
   workspaceRoot: string,
   globGroups: string[][]
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.hashFilesMatchingGlobs(globGroups);
   }
   return daemonClient.hashMultiGlob(globGroups);
@@ -132,13 +150,17 @@ export function updateFilesInContext(
   updatedFiles: string[],
   deletedFiles: string[]
 ) {
-  ensureContextAvailable(workspaceRoot);
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
+  ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
   return workspaceContext?.incrementalUpdate(updatedFiles, deletedFiles);
 }
 
 export async function getAllFileDataInContext(workspaceRoot: string) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.allFileData();
   }
   return daemonClient.getWorkspaceContextFileData();
@@ -148,8 +170,10 @@ export async function getFilesInDirectoryUsingContext(
   workspaceRoot: string,
   dir: string
 ) {
+  const additionalProjectDirectories =
+    readNxJson(workspaceRoot).additionalProjectDirectories ?? [];
   if (isOnDaemon() || !daemonClient.enabled()) {
-    ensureContextAvailable(workspaceRoot);
+    ensureContextAvailable(workspaceRoot, additionalProjectDirectories);
     return workspaceContext.getFilesInDirectory(dir);
   }
   return daemonClient.getFilesInDirectory(dir);
@@ -170,9 +194,12 @@ export function updateProjectFiles(
   );
 }
 
-function ensureContextAvailable(workspaceRoot: string) {
+function ensureContextAvailable(
+  workspaceRoot: string,
+  additionalProjectDirectories: string[]
+) {
   if (!workspaceContext || workspaceContext?.workspaceRoot !== workspaceRoot) {
-    setupWorkspaceContext(workspaceRoot);
+    setupWorkspaceContext(workspaceRoot, additionalProjectDirectories);
   }
 }