Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --workers option for multi-process operations #675

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+126 −11
Open

Add --workers option for multi-process operations #675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions tests/test_cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from tests.common import build_temp_workspace, RunContext, temp_workspace

from yamllint import cli, config
from yamllint.linter import LintProblem


# Check system's UTF-8 availability
Expand Down Expand Up @@ -733,6 +734,79 @@ def test_run_list_files(self):
[os.path.join(self.wd, 'a.yaml')]
)

def test_multiple_processes_w_2(self):
items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
'en.yaml': os.path.join(self.wd, 'en.yaml'),
'c.yaml': os.path.join(self.wd, 'c.yaml'),
}

with RunContext(self) as ctx:
cli.run(["-w", "2", "-f", "parsable"] + list(items.values()))
self.assertEqual(ctx.returncode, 1)

path = items['a.yaml']
self.assertIn(
f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n',
ctx.stdout
)
self.assertIn(
f'{path}:3:4: [error] no new line character at the end of file '
'(new-line-at-end-of-file)\n',
ctx.stdout
)

path = items['en.yaml']
self.assertIn(
f'{path}:3:8: [error] no new line character at the end of file '
'(new-line-at-end-of-file)\n',
ctx.stdout
)

path = items['c.yaml']
self.assertIn(
f'{path}:3:8: [error] no new line character at the end of file '
'(new-line-at-end-of-file)\n',
ctx.stdout
)

def test_multiple_processes_w_0(self):
items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
'en.yaml': os.path.join(self.wd, 'en.yaml'),
'c.yaml': os.path.join(self.wd, 'c.yaml'),
}
with RunContext(self) as ctx:
cli.run(["-w", "0", "-f", "parsable"] + list(items.values()))
self.assertEqual(ctx.returncode, 1)

path = items['a.yaml']
self.assertIn(
f'{path}:2:4: [error] trailing spaces (trailing-spaces)\n',
ctx.stdout
)

def test_multiple_processes_non_existing_file(self):
items = {'a.yaml': os.path.join(self.wd, 'a.yaml'),
'en.yaml': os.path.join(self.wd, 'en.yaml'),
'c.yaml': os.path.join(self.wd, 'c.yaml'),
'fails': os.path.join(self.wd, 'i-do-not-exist.yaml')
}

with RunContext(self) as ctx:
cli.run(["-w", "2", "-f", "parsable"] + list(items.values()))
self.assertEqual(ctx.returncode, -1)
self.assertRegex(ctx.stderr, r'No such file or directory')

def test_multiple_processes_do_lint(self):
conf = config.YamlLintConfig('extends: default')
path = os.path.join(self.wd, 'a.yaml')
problems = cli.do_lint(path, conf, generator=False)
print(type(problems[0]))
self.assertEqual(
problems,
[LintProblem(2, 4, 'trailing spaces', 'trailing-spaces'),
LintProblem(3, 4, 'no new line character at the end of file',
'new-line-at-end-of-file')])


class CommandLineConfigTestCase(unittest.TestCase):
def test_config_file(self):
Expand Down
63 changes: 52 additions & 11 deletions yamllint/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.

import argparse
import concurrent.futures
import locale
import os
import platform
Expand Down Expand Up @@ -143,6 +144,15 @@ def find_project_config_filepath(path='.'):
return find_project_config_filepath(path=os.path.join(path, '..'))


def do_lint(filename, conf, generator=True):
path = filename[2:] if filename.startswith("./") else filename
with open(filename, newline="") as f:
if generator:
return linter.run(f, conf, path)
else:
return list(linter.run(f, conf, path))


def run(argv=None):
parser = argparse.ArgumentParser(prog=APP_NAME,
description=APP_DESCRIPTION)
Expand Down Expand Up @@ -172,6 +182,12 @@ def run(argv=None):
parser.add_argument('--no-warnings',
action='store_true',
help='output only error level problems')
parser.add_argument('-w', '--workers',
action='store',
type=int,
default=1,
help='maximum number of parallel processes to run '
'(0 for auto, 1 for single-process)')
parser.add_argument('-v', '--version', action='version',
version=f'{APP_NAME} {APP_VERSION}')

Expand Down Expand Up @@ -216,17 +232,42 @@ def run(argv=None):

max_level = 0

for file in find_files_recursively(args.files, conf):
filepath = file[2:] if file.startswith('./') else file
try:
with open(file, newline='') as f:
problems = linter.run(f, conf, filepath)
except OSError as e:
print(e, file=sys.stderr)
sys.exit(-1)
prob_level = show_problems(problems, file, args_format=args.format,
no_warn=args.no_warnings)
max_level = max(max_level, prob_level)
if args.workers == 0:
args.workers = None

if args.workers == 1:
for file in find_files_recursively(args.files, conf):
try:
problems = do_lint(file, conf, generator=True)
except OSError as e:
print(e, file=sys.stderr)
sys.exit(-1)
prob_level = show_problems(problems, file, args_format=args.format,
no_warn=args.no_warnings)
max_level = max(max_level, prob_level)
else:
with concurrent.futures.ProcessPoolExecutor(
max_workers=args.workers
) as executor:
futures = {
executor.submit(do_lint, file, conf, generator=False): file
for file in find_files_recursively(args.files, conf)
}

for future in concurrent.futures.as_completed(futures):
try:
problems = future.result()
except OSError as e:
print(e, file=sys.stderr)
sys.exit(-1)

prob_level = show_problems(
problems,
futures[future],
args_format=args.format,
no_warn=args.no_warnings,
)
max_level = max(max_level, prob_level)

# read yaml from stdin
if args.stdin:
Expand Down
Loading