Add --workers option for multi-process operations

tests/test_cli.py

@@ -26,6 +26,7 @@
 from tests.common import build_temp_workspace, RunContext, temp_workspace
 
 from yamllint import cli, config
+from yamllint.linter import LintProblem
 
 
 # Check system's UTF-8 availability
@@ -733,6 +734,79 @@ def test_run_list_files(self):
             [os.path.join(self.wd, 'a.yaml')]
         )
 
+    def test_multiple_processes_w_2(self):
+        items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
+                 'en.yaml': os.path.join(self.wd, 'en.yaml'),
+                 'c.yaml': os.path.join(self.wd, 'c.yaml'),
+                 }
+
+        with RunContext(self) as ctx:
+            cli.run(["-w", "2", "-f", "parsable"] + list(items.values()))
+        self.assertEqual(ctx.returncode, 1)
+
+        path = items['a.yaml']
+        self.assertIn(
+            f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n',
+            ctx.stdout
+        )
+        self.assertIn(
+            f'{path}:3:4: [error] no new line character at the end of file '
+            '(new-line-at-end-of-file)\n',
+            ctx.stdout
+        )
+
+        path = items['en.yaml']
+        self.assertIn(
+            f'{path}:3:8: [error] no new line character at the end of file '
+            '(new-line-at-end-of-file)\n',
+            ctx.stdout
+        )
+
+        path = items['c.yaml']
+        self.assertIn(
+            f'{path}:3:8: [error] no new line character at the end of file '
+            '(new-line-at-end-of-file)\n',
+            ctx.stdout
+        )
+
+    def test_multiple_processes_w_0(self):
+        items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
+                 'en.yaml': os.path.join(self.wd, 'en.yaml'),
+                 'c.yaml': os.path.join(self.wd, 'c.yaml'),
+                 }
+        with RunContext(self) as ctx:
+            cli.run(["-w", "0", "-f", "parsable"] + list(items.values()))
+        self.assertEqual(ctx.returncode, 1)
+
+        path = items['a.yaml']
+        self.assertIn(
+            f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n',
+            ctx.stdout
+        )
+
+    def test_multiple_processes_non_existing_file(self):
+        items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
+                 'en.yaml': os.path.join(self.wd, 'en.yaml'),
+                 'c.yaml': os.path.join(self.wd, 'c.yaml'),
+                 'fails': os.path.join(self.wd, 'i-do-not-exist.yaml')
+                 }
+
+        with RunContext(self) as ctx:
+            cli.run(["-w", "2", "-f", "parsable"] + list(items.values()))
+        self.assertEqual(ctx.returncode, -1)
+        self.assertRegex(ctx.stderr, r'No such file or directory')
+
+    def test_multiple_processes_do_lint(self):
+        conf = config.YamlLintConfig('extends: default')
+        path = os.path.join(self.wd, 'a.yaml')
+        problems = cli.do_lint(path, conf, generator=False)
+        print(type(problems[0]))
+        self.assertEqual(
+            problems,
+            [LintProblem(2, 4, 'trailing spaces', 'trailing-spaces'),
+             LintProblem(3, 4, 'no new line character at the end of file',
+                         'new-line-at-end-of-file')])
+
 
 class CommandLineConfigTestCase(unittest.TestCase):
     def test_config_file(self):

yamllint/cli.py

@@ -14,6 +14,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
+import concurrent.futures
 import locale
 import os
 import platform
@@ -143,6 +144,15 @@ def find_project_config_filepath(path='.'):
     return find_project_config_filepath(path=os.path.join(path, '..'))
 
 
+def do_lint(filename, conf, generator=True):
+    path = filename[2:] if filename.startswith("./") else filename
+    with open(filename, newline="") as f:
+        if generator:
+            return linter.run(f, conf, path)
+        else:
+            return list(linter.run(f, conf, path))
+
+
 def run(argv=None):
     parser = argparse.ArgumentParser(prog=APP_NAME,
                                      description=APP_DESCRIPTION)
@@ -172,6 +182,12 @@ def run(argv=None):
     parser.add_argument('--no-warnings',
                         action='store_true',
                         help='output only error level problems')
+    parser.add_argument('-w', '--workers',
+                        action='store',
+                        type=int,
+                        default=1,
+                        help='maximum number of parallel processes to run '
+                             '(0 for auto, 1 for single-process)')
     parser.add_argument('-v', '--version', action='version',
                         version=f'{APP_NAME} {APP_VERSION}')
 
@@ -216,17 +232,42 @@ def run(argv=None):
 
     max_level = 0
 
-    for file in find_files_recursively(args.files, conf):
-        filepath = file[2:] if file.startswith('./') else file
-        try:
-            with open(file, newline='') as f:
-                problems = linter.run(f, conf, filepath)
-        except OSError as e:
-            print(e, file=sys.stderr)
-            sys.exit(-1)
-        prob_level = show_problems(problems, file, args_format=args.format,
-                                   no_warn=args.no_warnings)
-        max_level = max(max_level, prob_level)
+    if args.workers == 0:
+        args.workers = None
+
+    if args.workers == 1:
+        for file in find_files_recursively(args.files, conf):
+            try:
+                problems = do_lint(file, conf, generator=True)
+            except OSError as e:
+                print(e, file=sys.stderr)
+                sys.exit(-1)
+            prob_level = show_problems(problems, file, args_format=args.format,
+                                       no_warn=args.no_warnings)
+            max_level = max(max_level, prob_level)
+    else:
+        with concurrent.futures.ProcessPoolExecutor(
+                max_workers=args.workers
+        ) as executor:
+            futures = {
+                executor.submit(do_lint, file, conf, generator=False): file
+                for file in find_files_recursively(args.files, conf)
+            }
+
+            for future in concurrent.futures.as_completed(futures):
+                try:
+                    problems = future.result()
+                except OSError as e:
+                    print(e, file=sys.stderr)
+                    sys.exit(-1)
+
+                prob_level = show_problems(
+                    problems,
+                    futures[future],
+                    args_format=args.format,
+                    no_warn=args.no_warnings,
+                )
+                max_level = max(max_level, prob_level)
 
     # read yaml from stdin
     if args.stdin: