From 04553e045561f55735c7d98e38bb2a2b0455354f Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Fri, 22 Sep 2023 16:49:08 +0800 Subject: [PATCH 01/17] At least --help works --- blade.bat | 6 ++++++ src/blade/test_scheduler.py | 2 +- src/blade/toolchain.py | 2 +- src/blade/util.py | 19 +++++++++++++------ 4 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 blade.bat diff --git a/blade.bat b/blade.bat new file mode 100644 index 00000000..3fe0051f --- /dev/null +++ b/blade.bat @@ -0,0 +1,6 @@ +@echo off + +set python=python.exe +set blade_file=%~dp0src + +%python% %blade_file% %* diff --git a/src/blade/test_scheduler.py b/src/blade/test_scheduler.py index 29b1a0b6..98c187eb 100644 --- a/src/blade/test_scheduler.py +++ b/src/blade/test_scheduler.py @@ -36,7 +36,7 @@ def _signal_map(): if not name.startswith('SIG') or name.startswith('SIG_'): continue # SIGIOT and SIGABRT has the same value under linux and mac, but SIGABRT is more common. - if signal.SIGABRT == signal.SIGIOT and name == 'SIGIOT': + if name == 'SIGIOT' and signal.SIGABRT == signal.SIGIOT: continue result[-getattr(signal, name)] = name diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 1ba1930d..303c07be 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -34,7 +34,7 @@ class BuildArchitecture(object): 'bits': '32', }, 'x86_64': { - 'alias': ['amd64'], + 'alias': ['amd64', 'x64'], 'bits': '64', 'models': { '32': 'i386', diff --git a/src/blade/util.py b/src/blade/util.py index 0f8ff0d6..05ef1c7e 100644 --- a/src/blade/util.py +++ b/src/blade/util.py @@ -17,7 +17,6 @@ import ast import errno -import fcntl import hashlib import inspect import json @@ -28,6 +27,10 @@ import sys import zipfile +try: + import fcntl +except ImportError: + fcntl = None _IN_PY3 = sys.version_info[0] == 3 @@ -74,10 +77,11 @@ def md5sum(obj): def lock_file(filename): """lock file.""" try: - fd = os.open(filename, os.O_CREAT | os.O_RDWR) - old_fd_flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old_fd_flags | fcntl.FD_CLOEXEC) - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + fd = os.open(filename, os.O_CREAT | os.O_RDWR | os.O_EXCL) + if fcntl: + old_fd_flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, old_fd_flags | fcntl.FD_CLOEXEC) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) return fd, 0 except IOError as ex_value: return -1, ex_value.errno @@ -86,7 +90,8 @@ def lock_file(filename): def unlock_file(fd): """unlock file.""" try: - fcntl.flock(fd, fcntl.LOCK_UN) + if fcntl: + fcntl.flock(fd, fcntl.LOCK_UN) os.close(fd) except IOError: pass @@ -134,6 +139,8 @@ def get_cwd(): So in practice we simply use system('pwd') to get current working directory. """ + if os.name == 'nt': + return os.getcwd() p = subprocess.Popen(['pwd'], stdout=subprocess.PIPE, shell=True) return to_string(p.communicate()[0].strip()) From 9952a14f9fcaf7643946a1d5bbb10aa433655ae9 Mon Sep 17 00:00:00 2001 From: phongchen Date: Fri, 29 Sep 2023 11:16:58 +0800 Subject: [PATCH 02/17] Fix some functions for windows: - Workspace locking - Console color - ninja.build generation - CC toolchain --- src/blade/backend.py | 10 ++- src/blade/build_manager.py | 4 +- src/blade/command_line.py | 10 ++- src/blade/console.py | 31 +++++++- src/blade/toolchain.py | 147 +++++++++++++++++++++++++++++++++---- src/blade/util.py | 13 +++- 6 files changed, 188 insertions(+), 27 deletions(-) diff --git a/src/blade/backend.py b/src/blade/backend.py index 699ff6ae..1aa5a0f6 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import print_function +import codecs import os import subprocess import sys @@ -730,9 +731,12 @@ def generate_cuda_rules(self): description='CUDA LINK SHARED ${out}') def _builtin_command(self, builder, args=''): - cmd = ['PYTHONPATH=%s:$$PYTHONPATH' % self.blade_path] + if os.name == 'nt': + cmd = ['cmd /c set PYTHONPATH=%s:%%PYTHONPATH%%;' % self.blade_path] + else: + cmd = ['PYTHONPATH=%s:$$PYTHONPATH' % self.blade_path] python = os.environ.get('BLADE_PYTHON_INTERPRETER') or sys.executable - cmd.append('%s -m blade.builtin_tools %s' % (python, builder)) + cmd.append('"%s" -m blade.builtin_tools %s' % (python, builder)) if args: cmd.append(args) else: @@ -789,6 +793,6 @@ def generate_build_code(self): def generate_build_script(self): """Generate build script for underlying build system.""" code = self.generate_build_code() - script = open(self.script_path, 'w') + script = codecs.open(self.script_path, 'w', 'utf-8') script.writelines(code) script.close() diff --git a/src/blade/build_manager.py b/src/blade/build_manager.py index 4fecead6..2cce3960 100644 --- a/src/blade/build_manager.py +++ b/src/blade/build_manager.py @@ -24,8 +24,8 @@ from blade import maven from blade import ninja_runner from blade import target_pattern +from blade import toolchain from blade.binary_runner import BinaryRunner -from blade.toolchain import ToolChain from blade.build_accelerator import BuildAccelerator from blade.dependency_analyzer import analyze_deps from blade.load_build_files import load_targets @@ -101,7 +101,7 @@ def __init__(self, # Indicate whether the deps list is expanded by expander or not self.__targets_expanded = False - self.__build_toolchain = ToolChain() + self.__build_toolchain = toolchain.default() self.build_accelerator = BuildAccelerator(self.__build_toolchain) self.__build_jobs_num = 0 diff --git a/src/blade/command_line.py b/src/blade/command_line.py index 018715f1..5d828cd7 100644 --- a/src/blade/command_line.py +++ b/src/blade/command_line.py @@ -25,7 +25,7 @@ from blade import console from blade import constants from blade.toolchain import BuildArchitecture -from blade.toolchain import ToolChain +from blade.toolchain import CcToolChain # The 'version.py' is generated by the dist script, then it only exists in blade.zip try: @@ -98,6 +98,11 @@ def _check_test_options(self, options, targets): def _check_plat_and_profile_options(self, options, targets): """check platform and profile options.""" + arch = 'x64' + bits = 32 + options.arch = arch + options.bits = bits + return compiler_arch = self._compiler_target_arch() arch = BuildArchitecture.get_canonical_architecture(compiler_arch) if arch is None: @@ -414,6 +419,7 @@ def _add_dump_arguments(self, parser): group.add_argument( '--all-tags', dest='dump_all_tags', default=False, action='store_true', help='Dump all tags of targets in json format') + def _build_arg_parser(self): """Add command options, add options whthin this method.""" blade_cmd_help = 'blade [options...] [targets...]' @@ -462,7 +468,7 @@ def _build_arg_parser(self): def _compiler_target_arch(self): """Compiler(gcc) target architecture.""" - arch = ToolChain.get_cc_target_arch() + arch = CcToolChain.get_cc_target_arch() pos = arch.find('-') if pos == -1: console.fatal('Unknown target architecture %s from gcc.' % arch) diff --git a/src/blade/console.py b/src/blade/console.py index c18e6d3e..29bdf6cc 100644 --- a/src/blade/console.py +++ b/src/blade/console.py @@ -17,6 +17,7 @@ import datetime import os +import subprocess import sys import time @@ -25,10 +26,34 @@ ############################################################################## -# Global color enabled or not -_color_enabled = (sys.stdout.isatty() and - os.environ.get('TERM') not in ('emacs', 'dumb')) +def _windows_console_support_ansi_color(): + from ctypes import byref, windll, wintypes + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 + INVALID_HANDLE_VALUE = -1 + + handle = windll.kernel32.GetStdHandle(subprocess.STD_OUTPUT_HANDLE) + if handle == INVALID_HANDLE_VALUE: + return False + + mode = wintypes.DWORD() + if not windll.kernel32.GetConsoleMode(handle, byref(mode)): + return False + if not (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING): + if windll.kernel32.SetConsoleMode( + handle, + mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0: + print('kernel32.SetConsoleMode to enable ANSI sequences failed', + file=sys.stderr) + return True + +def _console_support_ansi_color(): + if os.name == 'nt': + return _windows_console_support_ansi_color() + return sys.stdout.isatty() and os.environ.get('TERM') not in ('emacs', 'dumb') + +# Global color enabled or not +_color_enabled = _console_support_ansi_color() # See http://en.wikipedia.org/wiki/ANSI_escape_code # colors diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 303c07be..d03a3a20 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -104,7 +104,47 @@ def get_model_architecture(arch, bits): return None -class ToolChain(object): +class CcToolChain(object): + """The build platform handles and gets the platform information.""" + + def __init__(self): + self.cc = '' + self.cxx = '' + self.ld = '' + self.cc_version = '' + self.ar = '' + + @staticmethod + def get_cc_target_arch(): + """Get the cc target architecture.""" + cc = CcToolChain._get_cc_command('CC', 'gcc') + returncode, stdout, stderr = run_command(cc + ' -dumpmachine', shell=True) + if returncode == 0: + return stdout.strip() + return '' + + def get_cc_commands(self): + return self.cc, self.cxx, self.ld + + def get_cc(self): + return self.cc + + def get_cc_version(self): + return self.cc_version + + def get_ar(self): + return self.ar + + def cc_is(self, vendor): + """Is cc is used for C/C++ compilation match vendor.""" + return vendor in self.cc + + def filter_cc_flags(self, flag_list, language='c'): + """Filter out the unrecognized compilation flags.""" + raise NotImplementedError + + +class CcToolChainGcc(CcToolChain): """The build platform handles and gets the platform information.""" def __init__(self): @@ -132,7 +172,78 @@ def _get_cc_version(self): @staticmethod def get_cc_target_arch(): """Get the cc target architecture.""" - cc = ToolChain._get_cc_command('CC', 'gcc') + cc = CcToolChain._get_cc_command('CC', 'gcc') + returncode, stdout, stderr = run_command(cc + ' -dumpmachine', shell=True) + if returncode == 0: + return stdout.strip() + return '' + + def filter_cc_flags(self, flag_list, language='c'): + """Filter out the unrecognized compilation flags.""" + flag_list = var_to_list(flag_list) + valid_flags, unrecognized_flags = [], [] + + # Put compilation output into test.o instead of /dev/null + # because the command line with '--coverage' below exit + # with status 1 which makes '--coverage' unsupported + # echo "int main() { return 0; }" | gcc -o /dev/null -c -x c --coverage - > /dev/null 2>&1 + fd, obj = tempfile.mkstemp('.o', 'filter_cc_flags_test') + cmd = ('export LC_ALL=C; echo "int main() { return 0; }" | ' + '%s -o %s -c -x %s -Werror %s -' % ( + self.cc, obj, language, ' '.join(flag_list))) + returncode, _, stderr = run_command(cmd, shell=True) + + try: + # In case of error, the `.o` file will be deleted by the compiler + os.remove(obj) + except OSError: + pass + os.close(fd) + + if returncode == 0: + return flag_list + for flag in flag_list: + # Example error messages: + # clang: warning: unknown warning option '-Wzzz' [-Wunknown-warning-option] + # gcc: gcc: error: unrecognized command line option '-Wxxx' + if " option '%s'" % flag in stderr: + unrecognized_flags.append(flag) + else: + valid_flags.append(flag) + + if unrecognized_flags: + console.warning('config: Unrecognized %s flags: %s' % ( + language, ', '.join(unrecognized_flags))) + + return valid_flags + + +class CcToolChainMsvc(CcToolChain): + """The build platform handles and gets the platform information.""" + + def __init__(self): + self.cc = 'cl.exe' + self.cxx = 'cl.exe' + self.ld = 'link.exe' + self.ar = 'lib.exe' + self.rc = 'rc.exe' + self.cc_version = self._get_cc_version() + + def _get_cc_version(self): + version = '' + returncode, stdout, stderr = run_command(self.cc, shell=True) + if returncode == 0: + m = re.search('Compiler Version ([\d.]+)', stderr.strip()) + if m: + version = m.group(1) + if not version: + console.fatal('Failed to obtain cc toolchain.') + return version + + @staticmethod + def get_cc_target_arch(): + """Get the cc target architecture.""" + cc = CcToolChain._get_cc_command('CC', 'gcc') returncode, stdout, stderr = run_command(cc + ' -dumpmachine', shell=True) if returncode == 0: return stdout.strip() @@ -163,32 +274,36 @@ def filter_cc_flags(self, flag_list, language='c'): # because the command line with '--coverage' below exit # with status 1 which makes '--coverage' unsupported # echo "int main() { return 0; }" | gcc -o /dev/null -c -x c --coverage - > /dev/null 2>&1 - fd, obj = tempfile.mkstemp('.o', 'filter_cc_flags_test') - cmd = ('export LC_ALL=C; echo "int main() { return 0; }" | ' - '%s -o %s -c -x %s -Werror %s -' % ( - self.cc, obj, language, ' '.join(flag_list))) - returncode, _, stderr = run_command(cmd, shell=True) - - try: - # In case of error, the `.o` file will be deleted by the compiler - os.remove(obj) - except OSError: - pass + suffix = language + if suffix == 'c++': + suffix = 'cpp' + fd, src = tempfile.mkstemp('.' + suffix, 'filter_cc_flags_test') + os.write(fd, b"int main() { return 0; }\n") os.close(fd) - if returncode == 0: - return flag_list for flag in flag_list: # Example error messages: # clang: warning: unknown warning option '-Wzzz' [-Wunknown-warning-option] # gcc: gcc: error: unrecognized command line option '-Wxxx' - if " option '%s'" % flag in stderr: + cmd = ('"%s" /nologo /FoNUL /c /WX %s "%s"' % (self.cc, flag, src)) + returncode, stdout, stderr = run_command(cmd, shell=True) + message = stdout + stderr + if "'%s'" % flag in message: unrecognized_flags.append(flag) else: valid_flags.append(flag) + try: + # In case of error, the `.o` file will be deleted by the compiler + os.remove(src) + except OSError: + pass if unrecognized_flags: console.warning('config: Unrecognized %s flags: %s' % ( language, ', '.join(unrecognized_flags))) return valid_flags + + +def default(): + return CcToolChainMsvc() diff --git a/src/blade/util.py b/src/blade/util.py index 05ef1c7e..edebe08f 100644 --- a/src/blade/util.py +++ b/src/blade/util.py @@ -32,6 +32,11 @@ except ImportError: fcntl = None +try: + import msvcrt +except ImportError: + msvcrt = None + _IN_PY3 = sys.version_info[0] == 3 @@ -77,13 +82,17 @@ def md5sum(obj): def lock_file(filename): """lock file.""" try: - fd = os.open(filename, os.O_CREAT | os.O_RDWR | os.O_EXCL) + fd = os.open(filename, os.O_CREAT | os.O_RDWR) if fcntl: old_fd_flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, old_fd_flags | fcntl.FD_CLOEXEC) fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + elif msvcrt: + msvcrt.locking(fd, msvcrt.LK_NBLCK, os.stat(fd).st_size) return fd, 0 except IOError as ex_value: + if msvcrt: # msvcrt did't set errno correctly + return -1, errno.EAGAIN return -1, ex_value.errno @@ -92,6 +101,8 @@ def unlock_file(fd): try: if fcntl: fcntl.flock(fd, fcntl.LOCK_UN) + elif msvcrt: + msvcrt.locking(fd, msvcrt.LK_UNLCK, os.stat(fd).st_size) os.close(fd) except IOError: pass From a9ab98c5a0710efbce4d2583811d2667daa6d944 Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Fri, 29 Sep 2023 16:15:38 +0800 Subject: [PATCH 03/17] More fix --- src/blade/backend.py | 34 ++++++++++++++++-------------- src/blade/build_manager.py | 10 ++++----- src/blade/cc_targets.py | 8 ++++--- src/blade/toolchain.py | 43 +++++++++++++++++++++++++++++++------- src/blade/util.py | 5 +++-- 5 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/blade/backend.py b/src/blade/backend.py index 1aa5a0f6..73ab6f0b 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -73,14 +73,14 @@ class _NinjaFileHeaderGenerator(object): for the underlying build system. """ # pylint: disable=too-many-public-methods - def __init__(self, command, options, build_dir, blade_path, build_toolchain, blade): + def __init__(self, command, options, build_dir, blade_path, cc_toolchain, blade): self.command = command self.options = options - self.build_dir = build_dir - self.blade_path = blade_path - self.build_toolchain = build_toolchain self.build_accelerator = blade.build_accelerator + self.build_dir = build_dir self.blade = blade + self.blade_path = blade_path + self.cc_toolchain = cc_toolchain self.rules_buf = [] self.__all_rule_names = set() @@ -193,7 +193,7 @@ def _get_intrinsic_cc_flags(self): cppflags.append('-Wno-error=coverage-mismatch') cppflags.append('-DPROFILE_GUIDED_OPTIMIZATION') - cppflags = self.build_toolchain.filter_cc_flags(cppflags) + cppflags = self.cc_toolchain.filter_cc_flags(cppflags) return cppflags, linkflags def _get_warning_flags(self): @@ -205,9 +205,9 @@ def _get_warning_flags(self): cflags = cc_config['c_warnings'] cuflags = cuda_config['cu_warnings'] - filtered_cppflags = self.build_toolchain.filter_cc_flags(cppflags) - filtered_cxxflags = self.build_toolchain.filter_cc_flags(cxxflags, 'c++') - filtered_cflags = self.build_toolchain.filter_cc_flags(cflags, 'c') + filtered_cppflags = self.cc_toolchain.filter_cc_flags(cppflags) + filtered_cxxflags = self.cc_toolchain.filter_cc_flags(cxxflags, 'c++') + filtered_cflags = self.cc_toolchain.filter_cc_flags(cflags, 'c') return filtered_cppflags, filtered_cxxflags, filtered_cflags, cuflags @@ -312,9 +312,11 @@ def _generate_cc_inclusion_check_rule(self): def _generate_cc_ar_rules(self): arflags = ''.join(config.get_item('cc_library_config', 'arflags')) ar = self.build_accelerator.get_ar_command() - self.generate_rule(name='ar', - command='rm -f $out; %s %s $out $in' % (ar, arflags), - description='AR ${out}') + if self.cc_toolchain.is_kind_of('msvc'): + command = '%s %s /OUT:$out $in' % (ar, arflags) + else: + command = 'rm -f $out; %s %s $out $in' % (ar, arflags) + self.generate_rule(name='ar', command=command, description='AR ${out}') def _generate_cc_link_rules(self, ld, linkflags): self._add_line('linkflags = %s' % ' '.join(config.get_item('cc_config', 'linkflags'))) @@ -666,8 +668,8 @@ def generate_package_rules(self): description='ZIP ${out}') def generate_version_rules(self): - cc = self.build_toolchain.get_cc() - cc_version = self.build_toolchain.get_cc_version() + cc = self.cc_toolchain.get_cc() + cc_version = self.cc_toolchain.get_cc_version() revision, url = util.load_scm(self.build_dir) args = '--scm=${out} --revision=${revision} --url=${url} --profile=${profile} --compiler="${compiler}"' @@ -686,7 +688,7 @@ def generate_version_rules(self): build %s: cxx %s cppflags = -w -O2 cxx_warnings = - ''') % (scm + '.o', scm)) + ''') % (self.cc_toolchain.object_file_of(scm), scm)) def generate_cuda_rules(self): nvcc_cmd = '${cmd}' @@ -769,7 +771,7 @@ def __init__(self, ninja_path, blade_path, blade): self.script_path = ninja_path self.blade_path = blade_path self.blade = blade - self.build_toolchain = blade.get_build_toolchain() + self.cc_toolchain = blade.get_cc_toolchain() self.build_dir = blade.get_build_dir() self.__all_rule_names = [] @@ -783,7 +785,7 @@ def generate_build_code(self): self.blade.get_options(), self.build_dir, self.blade_path, - self.build_toolchain, + self.cc_toolchain, self.blade) code = ninja_script_header_generator.generate() code += self.blade.generate_targets_build_code() diff --git a/src/blade/build_manager.py b/src/blade/build_manager.py index 2cce3960..9d031cd7 100644 --- a/src/blade/build_manager.py +++ b/src/blade/build_manager.py @@ -101,8 +101,8 @@ def __init__(self, # Indicate whether the deps list is expanded by expander or not self.__targets_expanded = False - self.__build_toolchain = toolchain.default() - self.build_accelerator = BuildAccelerator(self.__build_toolchain) + self.__cc_toolchain = toolchain.default() + self.build_accelerator = BuildAccelerator(self.__cc_toolchain) self.__build_jobs_num = 0 self.__build_script = os.path.join(self.__build_dir, 'build.ninja') @@ -556,9 +556,9 @@ def generate_targets_build_code(self): # code.append('default %s\n' % (_ALL_COMMAND_TARGETS)) return code - def get_build_toolchain(self): - """Return build toolchain instance.""" - return self.__build_toolchain + def get_cc_toolchain(self): + """Return cc build toolchain instance.""" + return self.__cc_toolchain def get_sources_keyword_list(self): """This keywords list is used to check the source files path. diff --git a/src/blade/cc_targets.py b/src/blade/cc_targets.py index 9299bc73..50647e7b 100644 --- a/src/blade/cc_targets.py +++ b/src/blade/cc_targets.py @@ -550,6 +550,7 @@ def _cc_objects(self, expanded_srcs, generated_headers=None): if secret: implicit_deps.append(self._source_file_path(self.attr['secret_revision_file'])) + toolchain = self.blade.get_cc_toolchain() objs_dir = self._target_file_path(self.name + '.objs') objs = [] for src, full_src in expanded_srcs: @@ -557,7 +558,7 @@ def _cc_objects(self, expanded_srcs, generated_headers=None): # to avoid file missing error if secret and path_under_dir(full_src, self.build_dir): self.generate_build('phony', full_src, inputs=[], clean=[]) - obj = os.path.join(objs_dir, src + '.o') + obj = os.path.join(objs_dir, toolchain.object_file_of(src)) rule = self._get_rule_from_suffix(src, secret) self.generate_build(rule, obj, inputs=full_src, implicit_deps=implicit_deps, @@ -1326,7 +1327,7 @@ def _get_rpath_links(self): def _generate_cc_binary_link_flags(self, dynamic_link): linkflags = [] - toolchain = self.blade.get_build_toolchain() + toolchain = self.blade.get_cc_toolchain() if not dynamic_link and toolchain.cc_is('gcc') and version_parse(toolchain.get_cc_version()) > version_parse('4.5'): linkflags += ['-static-libgcc', '-static-libstdc++'] if self.attr.get('export_dynamic'): @@ -1353,7 +1354,8 @@ def _cc_binary(self, objs, inclusion_check_result, dynamic_link): order_only_deps.append(inclusion_check_result) if self.attr['embed_version']: - scm = os.path.join(self.build_dir, 'scm.cc.o') + toolchain = self.blade.get_cc_toolchain() + scm = os.path.join(self.build_dir, toolchain.object_file_of('scm.cc')) objs.append(scm) order_only_deps.append(scm) output = self._target_file_path(self.name) diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index d03a3a20..d046f83e 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -135,14 +135,20 @@ def get_cc_version(self): def get_ar(self): return self.ar - def cc_is(self, vendor): + def is_kind_of(self, vendor): """Is cc is used for C/C++ compilation match vendor.""" - return vendor in self.cc + raise NotImplementedError def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" raise NotImplementedError + def object_file_of(self, source_file): + """ + Get the object file name from the source file. + """ + raise NotImplementedError + class CcToolChainGcc(CcToolChain): """The build platform handles and gets the platform information.""" @@ -178,6 +184,17 @@ def get_cc_target_arch(): return stdout.strip() return '' + def is_kind_of(self, vendor): + """Is cc is used for C/C++ compilation match vendor.""" + return vendor in ('gcc', 'clang', 'gcc') + + def object_file_of(self, source_file): + """ + Get the object file name from the source file. + """ + return source_file + '.o' + + def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" flag_list = var_to_list(flag_list) @@ -261,9 +278,16 @@ def get_cc_version(self): def get_ar(self): return self.ar - def cc_is(self, vendor): + def is_kind_of(self, vendor): """Is cc is used for C/C++ compilation match vendor.""" - return vendor in self.cc + return vendor in ('msvc') + + def object_file_of(self, source_file): + """ + Get the object file name from the source file. + """ + return source_file + '.obj' + def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" @@ -283,12 +307,15 @@ def filter_cc_flags(self, flag_list, language='c'): for flag in flag_list: # Example error messages: - # clang: warning: unknown warning option '-Wzzz' [-Wunknown-warning-option] - # gcc: gcc: error: unrecognized command line option '-Wxxx' - cmd = ('"%s" /nologo /FoNUL /c /WX %s "%s"' % (self.cc, flag, src)) + # Command line error D8021 : invalid numeric argument '/Wzzz' + if flag.startswith('-'): + testflag = '/' + flag[1:] + else: + testflag = flag + cmd = ('"%s" /nologo /FoNUL /c /WX %s "%s"' % (self.cc, testflag, src)) returncode, stdout, stderr = run_command(cmd, shell=True) message = stdout + stderr - if "'%s'" % flag in message: + if "'%s'" % testflag in message: unrecognized_flags.append(flag) else: valid_flags.append(flag) diff --git a/src/blade/util.py b/src/blade/util.py index edebe08f..a618cb61 100644 --- a/src/blade/util.py +++ b/src/blade/util.py @@ -167,9 +167,10 @@ def find_file_bottom_up(name, from_dir=None): path = os.path.join(finding_dir, name) if os.path.exists(path): return path - if finding_dir == '/': + new_finding_dir = os.path.dirname(finding_dir) + if new_finding_dir != finding_dir: break - finding_dir = os.path.dirname(finding_dir) + finding_dir = new_finding_dir return '' From cf8493cd428a4ddca22b858ba4ac7adea9495bce Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Fri, 29 Sep 2023 18:17:33 +0800 Subject: [PATCH 04/17] Support MSYS2 --- src/blade/toolchain.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index d046f83e..170aafd2 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -333,4 +333,6 @@ def filter_cc_flags(self, flag_list, language='c'): def default(): - return CcToolChainMsvc() + if os.name == 'nt': + return CcToolChainMsvc() + return CcToolChainGcc() From 2d67266267a8503f3e595ebe6b978bd056a499e2 Mon Sep 17 00:00:00 2001 From: phongchen Date: Tue, 3 Oct 2023 18:31:23 +0800 Subject: [PATCH 05/17] msvc build --- .gitignore | 1 + blade | 2 +- blade.conf | 6 +- src/blade/backend.py | 126 ++++++---------------------- src/blade/build_manager.py | 2 +- src/blade/cc_targets.py | 2 +- src/blade/main.py | 3 +- src/blade/toolchain.py | 164 ++++++++++++++++++++++++++++++++++++- 8 files changed, 194 insertions(+), 112 deletions(-) diff --git a/.gitignore b/.gitignore index 20a67894..c1d7ffca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea *.swp *.pyc +__pycache__ .DS_Store blade-bin diff --git a/blade b/blade index 3cc9a0f1..290e96fb 100755 --- a/blade +++ b/blade @@ -115,4 +115,4 @@ if [[ ! -e "$blade_file" ]]; then _fatal "Cannot find the core file $blade_file" fi -${BLADE_PYTHON_INTERPRETER:-$python} $blade_file "$@" +${BLADE_PYTHON_INTERPRETER:-$python} "$blade_file" "$@" diff --git a/blade.conf b/blade.conf index e1e1c537..a05f6440 100644 --- a/blade.conf +++ b/blade.conf @@ -4,7 +4,7 @@ global_config( cc_config( extra_incs=[], - warnings = [ + gcc_warnings = [ '-Wall', '-Wextra', # disable some warnings enabled by Wextra @@ -45,7 +45,7 @@ cc_config( '-Werror=write-strings', ], # C++ only warning flags - cxx_warnings = [ + gcc_cxx_warnings = [ '-Wno-invalid-offsetof', '-Wnon-virtual-dtor', '-Woverloaded-virtual', @@ -57,7 +57,7 @@ cc_config( '-Werror=vla', ], # C only warning flags - c_warnings = ['-Werror-implicit-function-declaration'], + gcc_c_warnings = ['-Werror-implicit-function-declaration'], optimize = ['-O2'], benchmark_libs=['//toft/base:benchmark'], benchmark_main_libs=['//toft/base:benchmark_main'], diff --git a/src/blade/backend.py b/src/blade/backend.py index 73ab6f0b..4fdcb70e 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -28,25 +28,6 @@ from blade import util -# To verify whether a header file is included without depends on the library it belongs to, -# we use the gcc's `-H` option to generate the inclusion stack information, see -# https://gcc.gnu.org/onlinedocs/gcc/Preprocessor-Options.html for details. -# But this information is output to stderr mixed with diagnostic messages. -# So we use this awk script to split them. -# -# The inclusion information is writes to stdout, other normal diagnostic message is write to stderr. -# -# NOTE the `$$` is required by ninja. and the `Multiple...` followed by multi lines of filepath are useless part. -# `多个防止重包含可能对其有用:` is the same as `Multiple...` in Chinese. -# After `Multiple...`, maybe there are still some useful messages, such as cuda error. -_INCLUSION_STACK_SPLITTER = (r"awk '" - r"""/Multiple include guards may be useful for:|多个防止重包含可能对其有用:/ {stop=1} """ # Can't exit here otherwise SIGPIPE maybe occurs. - r"""/^\.+ [^\/]/ && !end { started=1 ; print $$0} """ # Non absolute path, neither header list after error message. - r"""!/^\.+ / && started {end=1} """ # mark the error message when polling header list - r"""!/^\.+ / && (!stop || (!/Multiple include guards may be useful for:|多个防止重包含可能对其有用:/ && !/^[a-zA-Z0-9\.\/\+_-]+$$/ )) {print $$0 > "/dev/stderr"}""" # Maybe error messages - r"'" -) - def _incs_list_to_string(incs): """Convert incs list to string. @@ -60,11 +41,6 @@ def protoc_import_path_option(incs): return ' '.join(['-I=%s' % inc for inc in incs]) -def _shell_support_pipefail(): - """Whether current shell support the `pipefail` option.""" - return subprocess.call('set -o pipefail 2>/dev/null', shell=True) == 0 - - class _NinjaFileHeaderGenerator(object): """Generate global declarations and definitions for build script. @@ -243,67 +219,36 @@ def _generate_cc_vars(self): def _generate_cc_compile_rules(self, cc, cxx, cppflags): self._generate_cc_vars() - cc_config = config.get_section('cc_config') - cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags'] - cppflags = cc_config['cppflags'] + cppflags - includes = cc_config['extra_incs'] - includes = includes + ['.', self.build_dir] - includes = ' '.join(['-I%s' % inc for inc in includes]) - - template = self._cc_compile_command_wrapper_template('${out}.H') - - cc_command = ('%s -o ${out} -MMD -MF ${out}.d -c -fPIC %s %s ${optimize} ' - '${c_warnings} ${cppflags} %s ${includes} ${in}') % ( - cc, ' '.join(cflags), ' '.join(cppflags), includes) + cc_command, cxx_command, securecc_command, hdrs_command = self.cc_toolchain.get_compile_commands( + build_dir=self.build_dir, cppflags=cppflags, is_dump=self.command == 'dump' and self.options.dump_compdb) + if self.cc_toolchain.is_kind_of('msvc'): + deps = 'msvc' + depfile = None + else: + deps = 'gcc' + depfile = '${out}.d' self.generate_rule(name='cc', - command=template % cc_command, + command=cc_command, description='CC ${in}', - depfile='${out}.d', - deps='gcc') - - cxx_command = ('%s -o ${out} -MMD -MF ${out}.d -c -fPIC %s %s ${optimize} ' - '${cxx_warnings} ${cppflags} %s ${includes} ${in}') % ( - cxx, ' '.join(cxxflags), ' '.join(cppflags), includes) + depfile=depfile, + deps=deps) self.generate_rule(name='cxx', - command=template % cxx_command, + command=cxx_command, description='CXX ${in}', - depfile='${out}.d', - deps='gcc') + depfile=depfile, + deps=deps) self.generate_rule(name='secretcc', - command=template % (cc_config['secretcc'] + ' ' + cxx_command), + command=securecc_command, description='SECRET CC ${in}', - depfile='${out}.d') + depfile=depfile, + deps=deps) - self._generate_cc_hdrs_rule(cc, cxx, cppflags, cflags, cxxflags, includes) - - def _generate_cc_hdrs_rule(self, cc, cxx, cppflags, cflags, cxxflags, includes): - """ - Generate inclusion stack file for header file to check dependency missing. - See the '-H' in https://gcc.gnu.org/onlinedocs/gcc/Preprocessor-Options.html for details. - """ self.generate_rule(name='cxxhdrs', - command=self._hdrs_command(cxx, cxxflags, cppflags, includes), - depfile='${out}.d', deps='gcc', + command=hdrs_command, + depfile=depfile, deps=deps, description='CXX HDRS ${in}') - def _hdrs_command(self, cc, flags, cppflags, includes): - """Command to generate cc inclusion information file""" - args = (' -o /dev/null -E -MMD -MF ${out}.d %s %s -w ${cppflags} %s ${includes} ${in} ' - '2> ${out}.err' % (' '.join(flags), ' '.join(cppflags), includes)) - - # The `-fdirectives-only` option can significantly increase the speed of preprocessing, - # but errors may occur under certain boundary conditions (for example, - # `#if __COUNTER__ == __COUNTER__ + 1`), - # try the command again without it on error. - cmd1 = cc + ' -fdirectives-only' + args - cmd2 = cc + args - - # If the first cpp command fails, the second cpp command will be executed. - # The error message of the first command should be completely ignored. - return ('export LC_ALL=C; %s || %s; ec=$$?; %s ${out}.err > ${out}; ' - 'rm -f ${out}.err; exit $$ec') % (cmd1, cmd2, _INCLUSION_STACK_SPLITTER) - def _generate_cc_inclusion_check_rule(self): self.generate_rule(name='ccincchk', command=self._builtin_command('cc_inclusion_check'), @@ -336,43 +281,19 @@ def _generate_cc_link_rules(self, ld, linkflags): # Linking might have a lot of object files exceeding maximal length of a bash command line. # Using response file can resolve this problem. # Refer to: https://ninja-build.org/manual.html - link_args = '-o ${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' self.generate_rule(name='link', - command=ld + ' ' + link_args, + command=self.cc_toolchain.get_link_command(), rspfile='${out}.rsp', rspfile_content='${in}', description='LINK BINARY ${out}', pool=pool) self.generate_rule(name='solink', - command=ld + ' -shared ' + link_args, + command=self.cc_toolchain.get_shared_link_command(), rspfile='${out}.rsp', rspfile_content='${in}', description='LINK SHARED ${out}', pool=pool) - def _cc_compile_command_wrapper_template(self, inclusion_stack_file, cuda=False): - """Calculate the cc compile command wrapper template.""" - # When dumping compdb, a raw compile command without wrapper should be generated, - # otherwise some tools can't handle it. - if self.command == 'dump' and self.options.dump_compdb: - return '%s' - - print_header_option = '-H' - if cuda: - print_header_option = '-Xcompiler -H' - - if _shell_support_pipefail(): - # Use `pipefail` to ensure that the exit code is correct. - template = 'export LC_ALL=C; set -o pipefail; %%s %s 2>&1 | %s > %s' % ( - print_header_option, _INCLUSION_STACK_SPLITTER, inclusion_stack_file) - else: - # Some shell such as Ubuntu's `dash` doesn't support pipefail, make a workaround. - template = ('export LC_ALL=C; %%s %s 2> ${out}.err; ec=$$?; %s < ${out}.err > %s ; ' - 'rm -f ${out}.err; exit $$ec') % ( - print_header_option, _INCLUSION_STACK_SPLITTER, inclusion_stack_file) - - return template - def generate_proto_rules(self): proto_config = config.get_section('proto_library_config') protoc = proto_config['protoc'] @@ -691,6 +612,7 @@ def generate_version_rules(self): ''') % (self.cc_toolchain.object_file_of(scm), scm)) def generate_cuda_rules(self): + return nvcc_cmd = '${cmd}' cc_config = config.get_section('cc_config') @@ -705,8 +627,6 @@ def generate_cuda_rules(self): includes = includes + ['.', self.build_dir] includes = ' '.join(['-I%s' % inc for inc in includes]) - template = self._cc_compile_command_wrapper_template('${out}.H', cuda=True) - _, cxx, _ = self.build_accelerator.get_cc_commands() cu_command = '%s -ccbin %s -o ${out} -MMD -MF ${out}.d ' \ '-Xcompiler -fPIC %s %s %s ${optimize} ${cu_warnings} ' \ @@ -736,7 +656,7 @@ def _builtin_command(self, builder, args=''): if os.name == 'nt': cmd = ['cmd /c set PYTHONPATH=%s:%%PYTHONPATH%%;' % self.blade_path] else: - cmd = ['PYTHONPATH=%s:$$PYTHONPATH' % self.blade_path] + cmd = ['PYTHONPATH="%s":$$PYTHONPATH' % self.blade_path] python = os.environ.get('BLADE_PYTHON_INTERPRETER') or sys.executable cmd.append('"%s" -m blade.builtin_tools %s' % (python, builder)) if args: diff --git a/src/blade/build_manager.py b/src/blade/build_manager.py index 9d031cd7..5d432c40 100644 --- a/src/blade/build_manager.py +++ b/src/blade/build_manager.py @@ -101,7 +101,7 @@ def __init__(self, # Indicate whether the deps list is expanded by expander or not self.__targets_expanded = False - self.__cc_toolchain = toolchain.default() + self.__cc_toolchain = toolchain.default(options.m) self.build_accelerator = BuildAccelerator(self.__cc_toolchain) self.__build_jobs_num = 0 diff --git a/src/blade/cc_targets.py b/src/blade/cc_targets.py index 50647e7b..660558d5 100644 --- a/src/blade/cc_targets.py +++ b/src/blade/cc_targets.py @@ -1328,7 +1328,7 @@ def _get_rpath_links(self): def _generate_cc_binary_link_flags(self, dynamic_link): linkflags = [] toolchain = self.blade.get_cc_toolchain() - if not dynamic_link and toolchain.cc_is('gcc') and version_parse(toolchain.get_cc_version()) > version_parse('4.5'): + if not dynamic_link and toolchain.is_kind_of('gcc') and version_parse(toolchain.get_cc_version()) > version_parse('4.5'): linkflags += ['-static-libgcc', '-static-libstdc++'] if self.attr.get('export_dynamic'): linkflags.append('-rdynamic') diff --git a/src/blade/main.py b/src/blade/main.py index 5dea57e0..e5eadfb6 100644 --- a/src/blade/main.py +++ b/src/blade/main.py @@ -31,9 +31,8 @@ def load_config(options, root_dir): """Load the configuration file and parse.""" # Init global build attributes - build_attributes.initialize(options) config.load_files(root_dir, options.load_local_config) - + build_attributes.initialize(options) def setup_console(options): if options.color != 'auto': diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 170aafd2..c9f3873f 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -3,6 +3,7 @@ # # Author: Chong Peng # Date: October 20, 2011 +# coding: utf-8 """ @@ -13,15 +14,23 @@ from __future__ import print_function import os +import subprocess import re import tempfile +from blade import config from blade import console from blade.util import var_to_list, iteritems, run_command # example: Cuda compilation tools, release 11.0, V11.0.194 _nvcc_version_re = re.compile(r'V(\d+\.\d+\.\d+)') + +def _shell_support_pipefail(): + """Whether current shell support the `pipefail` option.""" + return subprocess.call('set -o pipefail 2>/dev/null', shell=True) == 0 + + class BuildArchitecture(object): """ The BuildArchitecture class manages architecture/bits configuration @@ -139,6 +148,14 @@ def is_kind_of(self, vendor): """Is cc is used for C/C++ compilation match vendor.""" raise NotImplementedError + def target_arch(self): + """Return architecture of target machine.""" + raise NotImplementedError + + def target_bits(self): + """Return number of bits of target machine.""" + raise NotImplementedError + def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" raise NotImplementedError @@ -149,6 +166,32 @@ def object_file_of(self, source_file): """ raise NotImplementedError + def library_file_name(self, name): + """ + Get the library file name from the name. + """ + raise NotImplementedError + + +# To verify whether a header file is included without depends on the library it belongs to, +# we use the gcc's `-H` option to generate the inclusion stack information, see +# https://gcc.gnu.org/onlinedocs/gcc/Preprocessor-Options.html for details. +# But this information is output to stderr mixed with diagnostic messages. +# So we use this awk script to split them. +# +# The inclusion information is writes to stdout, other normal diagnostic message is write to stderr. +# +# NOTE the `$$` is required by ninja. and the `Multiple...` followed by multi lines of filepath are useless part. +# `多个防止重包含可能对其有用:` is the same as `Multiple...` in Chinese. +# After `Multiple...`, maybe there are still some useful messages, such as cuda error. +_INCLUSION_STACK_SPLITTER = (r"awk '" + r"""/Multiple include guards may be useful for:|多个防止重包含可能对其有用:/ {stop=1} """ # Can't exit here otherwise SIGPIPE maybe occurs. + r"""/^\.+ [^\/]/ && !end { started=1 ; print $$0} """ # Non absolute path, neither header list after error message. + r"""!/^\.+ / && started {end=1} """ # mark the error message when polling header list + r"""!/^\.+ / && (!stop || (!/Multiple include guards may be useful for:|多个防止重包含可能对其有用:/ && !/^[a-zA-Z0-9\.\/\+_-]+$$/ )) {print $$0 > "/dev/stderr"}""" # Maybe error messages + r"'" +) + class CcToolChainGcc(CcToolChain): """The build platform handles and gets the platform information.""" @@ -194,6 +237,77 @@ def object_file_of(self, source_file): """ return source_file + '.o' + def _cc_compile_command_wrapper_template(self, inclusion_stack_file, cuda=False): + """Calculate the cc compile command wrapper template.""" + print_header_option = '-H' + if cuda: + print_header_option = '-Xcompiler -H' + + if _shell_support_pipefail(): + # Use `pipefail` to ensure that the exit code is correct. + template = 'export LC_ALL=C; set -o pipefail; %%s %s 2>&1 | %s > %s' % ( + print_header_option, _INCLUSION_STACK_SPLITTER, inclusion_stack_file) + else: + # Some shell such as Ubuntu's `dash` doesn't support pipefail, make a workaround. + template = ('export LC_ALL=C; %%s %s 2> ${out}.err; ec=$$?; %s < ${out}.err > %s ; ' + 'rm -f ${out}.err; exit $$ec') % ( + print_header_option, _INCLUSION_STACK_SPLITTER, inclusion_stack_file) + + return template + + def get_compile_commands(self, build_dir, cppflags, is_dump): + cc_config = config.get_section('cc_config') + cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags'] + cppflags = cc_config['cppflags'] + cppflags + includes = cc_config['extra_incs'] + includes = includes + ['.', build_dir] + includes = ' '.join(['-I%s' % inc for inc in includes]) + + cc_command = ('%s -o ${out} -MMD -MF ${out}.d -c -fPIC %s %s ${optimize} ' + '${c_warnings} ${cppflags} %s ${includes} ${in}') % ( + self.cc, ' '.join(cflags), ' '.join(cppflags), includes) + cxx_command = ('%s -o ${out} -MMD -MF ${out}.d -c -fPIC %s %s ${optimize} ' + '${cxx_warnings} ${cppflags} %s ${includes} ${in}') % ( + self.cxx, ' '.join(cxxflags), ' '.join(cppflags), includes) + securecc_command = (cc_config['secretcc'] + ' ' + cxx_command) + + hdrs_command = self._hdrs_command(cxxflags, cppflags, includes) + + if is_dump: + # When dumping compdb, a raw compile command without wrapper should be generated, + # otherwise some tools can't handle it. + return cc_command, cxx_command, securecc_command, hdrs_command + template = self._cc_compile_command_wrapper_template('${out}.H') + return template % cc_command, template % cxx_command, template % securecc_command, hdrs_command + + def _hdrs_command(self, flags, cppflags, includes): + """ + Command to generate cc inclusion information file for header file to check dependency missing. + See the '-H' in https://gcc.gnu.org/onlinedocs/gcc/Preprocessor-Options.html for details. + """ + args = (' -o /dev/null -E -MMD -MF ${out}.d %s %s -w ${cppflags} %s ${includes} ${in} ' + '2> ${out}.err' % (' '.join(flags), ' '.join(cppflags), includes)) + + # The `-fdirectives-only` option can significantly increase the speed of preprocessing, + # but errors may occur under certain boundary conditions (for example, + # `#if __COUNTER__ == __COUNTER__ + 1`), + # try the command again without it on error. + cmd1 = self.cc + ' -fdirectives-only' + args + cmd2 = self.cc + args + + # If the first cpp command fails, the second cpp command will be executed. + # The error message of the first command should be completely ignored. + return ('export LC_ALL=C; %s || %s; ec=$$?; %s ${out}.err > ${out}; ' + 'rm -f ${out}.err; exit $$ec') % (cmd1, cmd2, _INCLUSION_STACK_SPLITTER) + + def get_link_command(self): + return self.ld + self._get_link_args() + + def get_shared_link_command(self): + return self.ld + '-shared ' + self._get_link_args() + + def _get_link_args(self): + return ' -o ${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" @@ -331,8 +445,56 @@ def filter_cc_flags(self, flag_list, language='c'): return valid_flags + def get_compile_commands(self, build_dir, cppflags, is_dump): + cc_config = config.get_section('cc_config') + cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags'] + cppflags = cc_config['cppflags'] + cppflags + includes = cc_config['extra_incs'] + includes = includes + ['.', build_dir] + includes = ' '.join(['-I%s' % inc for inc in includes]) + + cc_command = ('%s /nologo /c /Fo${out} %s %s ${optimize} ' + '${c_warnings} ${cppflags} %s ${includes} ${in}') % ( + self.cc, ' '.join(cflags), ' '.join(cppflags), includes) + cxx_command = ('%s /nologo /c -Fo${out} %s %s ${optimize} ' + '${cxx_warnings} ${cppflags} %s ${includes} ${in}') % ( + self.cxx, ' '.join(cxxflags), ' '.join(cppflags), includes) + securecc_command = (cc_config['secretcc'] + ' ' + cxx_command) + + hdrs_command = self._hdrs_command(cxxflags, cppflags, includes) + + if is_dump: + # When dumping compdb, a raw compile command without wrapper should be generated, + # otherwise some tools can't handle it. + return cc_command, cxx_command, securecc_command, hdrs_command + template = self._cc_compile_command_wrapper_template('${out}.H') + return template % cc_command, template % cxx_command, template % securecc_command, hdrs_command + + def _cc_compile_command_wrapper_template(self, inclusion_stack_file, cuda=False): + return '%s' + + def _hdrs_command(self, flags, cppflags, includes): + """ + Command to generate cc inclusion information file for header file to check dependency missing. + See https://learn.microsoft.com/en-us/cpp/build/reference/showincludes-list-include-files for details. + """ + args = (' -Fonul /Zs %s %s /w ${cppflags} %s ${includes} ${in} ' + '2> ${out}.err' % (' '.join(flags), ' '.join(cppflags), includes)) + + # If the first cpp command fails, the second cpp command will be executed. + # The error message of the first command should be completely ignored. + return self.cc + args + + def get_link_command(self): + return self.ld + self._get_link_args() + + def get_shared_link_command(self): + return self.ld + ' /DLL ' + self._get_link_args() + + def _get_link_args(self): + return ' /nologo /OUT:${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' -def default(): +def default(bits): if os.name == 'nt': return CcToolChainMsvc() return CcToolChainGcc() From 283d88014da007447b3466acaa20d81582a3bb35 Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Tue, 3 Oct 2023 20:20:50 +0800 Subject: [PATCH 06/17] fix builtin-tools --- src/blade/backend.py | 32 ++++++++++++++++++++++++++------ src/blade/config.py | 3 ++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/blade/backend.py b/src/blade/backend.py index 4fdcb70e..6f2fc67f 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -57,6 +57,7 @@ def __init__(self, command, options, build_dir, blade_path, cc_toolchain, blade) self.blade = blade self.blade_path = blade_path self.cc_toolchain = cc_toolchain + self.__builtin_tools_prefix = '' self.rules_buf = [] self.__all_rule_names = set() @@ -652,13 +653,32 @@ def generate_cuda_rules(self): command=nvcc_cmd + ' -shared ' + link_args, description='CUDA LINK SHARED ${out}') - def _builtin_command(self, builder, args=''): - if os.name == 'nt': - cmd = ['cmd /c set PYTHONPATH=%s:%%PYTHONPATH%%;' % self.blade_path] - else: - cmd = ['PYTHONPATH="%s":$$PYTHONPATH' % self.blade_path] + def _builtin_tools_prefix(self): + """ + Get the command prefix to run the builtin tools. + """ python = os.environ.get('BLADE_PYTHON_INTERPRETER') or sys.executable - cmd.append('"%s" -m blade.builtin_tools %s' % (python, builder)) + if not self.__builtin_tools_prefix: + # we need to set the PYTHONPATH to the path of the blade directory before run python. + if os.name == 'nt': + # On Windows, we generate a batch file to set the PYTHONPATH. + builtin_tools_file = os.path.join(self.build_dir, 'builtin_tools.bat') + with open(builtin_tools_file, 'w') as f: + print('@echo off', file=f) + print('set "PYTHONPATH={};%PYTHONPATH%"'.format(self.blade_path), file=f) + print('{} %*'.format(python), file=f) + self.__builtin_tools_prefix = builtin_tools_file + else: + # On posix system, a simply environment prefix is enough to do it. + self.__builtin_tools_prefix = 'PYTHONPATH="%s":$$PYTHONPATH %s' % (self.blade_path, python) + return self.__builtin_tools_prefix + + def _builtin_command(self, builder, args=''): + """ + Generate blade builtin command line + """ + cmd = [self._builtin_tools_prefix()] + cmd.append('-m blade.builtin_tools %s' % builder) if args: cmd.append(args) else: diff --git a/src/blade/config.py b/src/blade/config.py index 41813a65..4a7bf85f 100644 --- a/src/blade/config.py +++ b/src/blade/config.py @@ -116,7 +116,8 @@ def __init__(self): # Options passed to ar/ranlib to control how # the archive is created, such as, let ar operate # in deterministic mode discarding timestamps - 'arflags': ['rcs'], + 'gcc_arflags': ['rcs'], + 'arflags': [''], 'ranlibflags': [], 'hdrs_missing_severity': 'error', 'hdrs_missing_suppress': set(), From db12a14d959e038cae4fc9246c594b3c22ae6337 Mon Sep 17 00:00:00 2001 From: phongchen Date: Wed, 4 Oct 2023 01:17:50 +0800 Subject: [PATCH 07/17] Add override --- src/blade/backend.py | 2 +- src/blade/toolchain.py | 49 +++++++++++++++++++++++------------------- src/blade/util.py | 48 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/src/blade/backend.py b/src/blade/backend.py index 6f2fc67f..6c26f44c 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -666,7 +666,7 @@ def _builtin_tools_prefix(self): with open(builtin_tools_file, 'w') as f: print('@echo off', file=f) print('set "PYTHONPATH={};%PYTHONPATH%"'.format(self.blade_path), file=f) - print('{} %*'.format(python), file=f) + print('"{}" %*'.format(python), file=f) self.__builtin_tools_prefix = builtin_tools_file else: # On posix system, a simply environment prefix is enough to do it. diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index c9f3873f..e4f4786b 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -20,7 +20,7 @@ from blade import config from blade import console -from blade.util import var_to_list, iteritems, run_command +from blade.util import var_to_list, iteritems, override, run_command # example: Cuda compilation tools, release 11.0, V11.0.194 _nvcc_version_re = re.compile(r'V(\d+\.\d+\.\d+)') @@ -166,12 +166,23 @@ def object_file_of(self, source_file): """ raise NotImplementedError - def library_file_name(self, name): + def static_library_name(self, name): + """ + Get the static library file name from the name. + """ + raise NotImplementedError + + def dynamic_library_name(self, name): """ Get the library file name from the name. """ raise NotImplementedError + def executable_file_name(self, name): + """ + Get the executable file name from the name. + """ + raise NotImplementedError # To verify whether a header file is included without depends on the library it belongs to, # we use the gcc's `-H` option to generate the inclusion stack information, see @@ -227,16 +238,20 @@ def get_cc_target_arch(): return stdout.strip() return '' + @override def is_kind_of(self, vendor): - """Is cc is used for C/C++ compilation match vendor.""" return vendor in ('gcc', 'clang', 'gcc') + @override def object_file_of(self, source_file): - """ - Get the object file name from the source file. - """ return source_file + '.o' + @override + def executable_file_name(self, name): + if os.name == 'nt': + return name + '.exe' + return name + def _cc_compile_command_wrapper_template(self, inclusion_stack_file, cuda=False): """Calculate the cc compile command wrapper template.""" print_header_option = '-H' @@ -380,28 +395,17 @@ def get_cc_target_arch(): return stdout.strip() return '' - def get_cc_commands(self): - return self.cc, self.cxx, self.ld - - def get_cc(self): - return self.cc - - def get_cc_version(self): - return self.cc_version - - def get_ar(self): - return self.ar - + @override def is_kind_of(self, vendor): - """Is cc is used for C/C++ compilation match vendor.""" return vendor in ('msvc') + @override def object_file_of(self, source_file): - """ - Get the object file name from the source file. - """ return source_file + '.obj' + @override + def executable_file_name(self, name): + return name + '.exe' def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" @@ -494,6 +498,7 @@ def get_shared_link_command(self): def _get_link_args(self): return ' /nologo /OUT:${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' + def default(bits): if os.name == 'nt': return CcToolChainMsvc() diff --git a/src/blade/util.py b/src/blade/util.py index a618cb61..466cc873 100644 --- a/src/blade/util.py +++ b/src/blade/util.py @@ -21,6 +21,7 @@ import inspect import json import os +import re import signal import string import subprocess @@ -379,3 +380,50 @@ def which(cmd): if returncode != 0: return None return stdout.strip() + + +def override(method): + """ + Check method override. + https://stackoverflow.com/a/14631397 + """ + + stack = inspect.stack() + base_classes = _get_bass_classes(stack) + + # TODO: check signature + # sig = inspect.signature(method) + error = "methid '%s' doesn't override any base class method" % method.__name__ + assert( any( hasattr(cls, method.__name__) for cls in base_classes ) ), error + return method + + +def _get_bass_classes(stack): + base_classes = re.search(r'class.+\((.+)\)\s*\:', stack[2][4][0]).group(1) + + # handle multiple inheritance + base_classes = [s.strip() for s in base_classes.split(',')] + if not base_classes: + raise ValueError('override decorator: unable to determine base class') + + # stack[0]=override, stack[1]=inside class def'n, stack[2]=outside class def'n + derived_class_locals = stack[2][0].f_locals + + # replace each class name in base_classes with the actual class type + for i, base_class in enumerate(base_classes): + + if '.' not in base_class: + base_classes[i] = derived_class_locals[base_class] + else: + components = base_class.split('.') + + # obj is either a module or a class + obj = derived_class_locals[components[0]] + + for c in components[1:]: + assert(inspect.ismodule(obj) or inspect.isclass(obj)) + obj = getattr(obj, c) + + base_classes[i] = obj + + return base_classes From b68dae32c24c00ba1eebc5d9d27a98ce040b8a61 Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Wed, 4 Oct 2023 19:18:27 +0800 Subject: [PATCH 08/17] Done with hdr command --- src/blade/toolchain.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index e4f4786b..b9caf7ef 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -482,12 +482,13 @@ def _hdrs_command(self, flags, cppflags, includes): Command to generate cc inclusion information file for header file to check dependency missing. See https://learn.microsoft.com/en-us/cpp/build/reference/showincludes-list-include-files for details. """ - args = (' -Fonul /Zs %s %s /w ${cppflags} %s ${includes} ${in} ' - '2> ${out}.err' % (' '.join(flags), ' '.join(cppflags), includes)) + cmd = ('cmd.exe /c %s /nologo /c /E /Zs /TP /showIncludes %s %s /w ${cppflags} %s ${includes} ${in} 2>&1 >nul |' + ' findstr "Note: including file" >${out} && type ${out}'% ( + self.cc, ' '.join(flags), ' '.join(cppflags), includes)) # If the first cpp command fails, the second cpp command will be executed. # The error message of the first command should be completely ignored. - return self.cc + args + return cmd def get_link_command(self): return self.ld + self._get_link_args() From b3ad4322810697eff79fb8ad5f95790659979491 Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Wed, 4 Oct 2023 20:44:47 +0800 Subject: [PATCH 09/17] Simplify cc_flags_filter, fix clean subcommand --- src/blade/build_manager.py | 7 +++++- src/blade/load_build_files.py | 5 ++++- src/blade/toolchain.py | 40 +++++++++++++---------------------- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/blade/build_manager.py b/src/blade/build_manager.py index 5d432c40..feb8c4c4 100644 --- a/src/blade/build_manager.py +++ b/src/blade/build_manager.py @@ -252,7 +252,12 @@ def _test(self): def _remove_paths(paths): # The rm command can delete a large number of files at once, which is much faster than # using python's own remove functions (only supports deleting a single path at a time). - subprocess.call(['rm', '-fr'] + paths) + if os.name == 'posix': + subprocess.call(['rm', '-fr'] + paths) + return + import shutil + for path in paths: + shutil.rmtree(path, ignore_errors=True) def clean(self): """Clean specific generated target files or directories""" diff --git a/src/blade/load_build_files.py b/src/blade/load_build_files.py index 872f942d..783af0df 100644 --- a/src/blade/load_build_files.py +++ b/src/blade/load_build_files.py @@ -367,7 +367,10 @@ def _check_under_skipped_dir(dirname): if os.path.exists(filepath): cache[dirname] = filepath return filepath - result = _check_under_skipped_dir(os.path.dirname(dirname)) + parent = os.path.dirname(dirname) + if parent == dirname: + return '' + result = _check_under_skipped_dir(parent) cache[dirname] = result return result diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index b9caf7ef..66cfe40b 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -412,36 +412,26 @@ def filter_cc_flags(self, flag_list, language='c'): flag_list = var_to_list(flag_list) valid_flags, unrecognized_flags = [], [] - # Put compilation output into test.o instead of /dev/null - # because the command line with '--coverage' below exit - # with status 1 which makes '--coverage' unsupported - # echo "int main() { return 0; }" | gcc -o /dev/null -c -x c --coverage - > /dev/null 2>&1 - suffix = language - if suffix == 'c++': - suffix = 'cpp' - fd, src = tempfile.mkstemp('.' + suffix, 'filter_cc_flags_test') - os.write(fd, b"int main() { return 0; }\n") - os.close(fd) - + # cl.exe only report the first error option, so we must test flags one by one. for flag in flag_list: - # Example error messages: - # Command line error D8021 : invalid numeric argument '/Wzzz' if flag.startswith('-'): testflag = '/' + flag[1:] else: testflag = flag - cmd = ('"%s" /nologo /FoNUL /c /WX %s "%s"' % (self.cc, testflag, src)) - returncode, stdout, stderr = run_command(cmd, shell=True) - message = stdout + stderr - if "'%s'" % testflag in message: - unrecognized_flags.append(flag) - else: - valid_flags.append(flag) - try: - # In case of error, the `.o` file will be deleted by the compiler - os.remove(src) - except OSError: - pass + cmd = ('"%s" /nologo /E /Zs /c /WX %s' % (self.cc, testflag)) + try: + returncode, stdout, stderr = run_command(cmd, shell=False) + if returncode == 0: + continue + output = stdout + stderr + # Example error messages: + # Command line error D8021 : invalid numeric argument '/Wzzz' + if "'%s'" % testflag in output: + unrecognized_flags.append(flag) + else: + valid_flags.append(flag) + except OSError as e: + console.fatal("filter_cc_flags error, cmd='%s', error='%s'" % (cmd, e)) if unrecognized_flags: console.warning('config: Unrecognized %s flags: %s' % ( From 0150f732e6010b9d2050cdc29fde7b082e7eec47 Mon Sep 17 00:00:00 2001 From: CHEN FENG Date: Thu, 5 Oct 2023 20:38:56 +0800 Subject: [PATCH 10/17] fixing inclusion check --- src/blade/backend.py | 16 ++++++-------- src/blade/inclusion_check.py | 31 ++++++++++++++++++++++++++- src/blade/toolchain.py | 41 ++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/blade/backend.py b/src/blade/backend.py index 6c26f44c..4e252ade 100644 --- a/src/blade/backend.py +++ b/src/blade/backend.py @@ -256,12 +256,7 @@ def _generate_cc_inclusion_check_rule(self): description='CC INCLUSION CHECK ${in}') def _generate_cc_ar_rules(self): - arflags = ''.join(config.get_item('cc_library_config', 'arflags')) - ar = self.build_accelerator.get_ar_command() - if self.cc_toolchain.is_kind_of('msvc'): - command = '%s %s /OUT:$out $in' % (ar, arflags) - else: - command = 'rm -f $out; %s %s $out $in' % (ar, arflags) + command = self.cc_toolchain.get_static_library_command() self.generate_rule(name='ar', command=command, description='AR ${out}') def _generate_cc_link_rules(self, ld, linkflags): @@ -660,25 +655,26 @@ def _builtin_tools_prefix(self): python = os.environ.get('BLADE_PYTHON_INTERPRETER') or sys.executable if not self.__builtin_tools_prefix: # we need to set the PYTHONPATH to the path of the blade directory before run python. + python_args = '-m blade.builtin_tools' if os.name == 'nt': # On Windows, we generate a batch file to set the PYTHONPATH. builtin_tools_file = os.path.join(self.build_dir, 'builtin_tools.bat') with open(builtin_tools_file, 'w') as f: print('@echo off', file=f) print('set "PYTHONPATH={};%PYTHONPATH%"'.format(self.blade_path), file=f) - print('"{}" %*'.format(python), file=f) + print('"{}" {} %*'.format(python, python_args), file=f) self.__builtin_tools_prefix = builtin_tools_file else: # On posix system, a simply environment prefix is enough to do it. - self.__builtin_tools_prefix = 'PYTHONPATH="%s":$$PYTHONPATH %s' % (self.blade_path, python) + self.__builtin_tools_prefix = 'PYTHONPATH="%s":$$PYTHONPATH "%s" %s' % ( + self.blade_path, python, python_args) return self.__builtin_tools_prefix def _builtin_command(self, builder, args=''): """ Generate blade builtin command line """ - cmd = [self._builtin_tools_prefix()] - cmd.append('-m blade.builtin_tools %s' % builder) + cmd = [self._builtin_tools_prefix(), builder] if args: cmd.append(args) else: diff --git a/src/blade/inclusion_check.py b/src/blade/inclusion_check.py index 4ba7caa1..ae680a61 100644 --- a/src/blade/inclusion_check.py +++ b/src/blade/inclusion_check.py @@ -61,6 +61,23 @@ def is_allowed_undeclared_hdr(self, hdr): return hdr in self._allowed_undeclared_hdrs +_MSVC_INCUSION_PREFIX = 'Note: including file:' + + +def _is_inclusion_line(line): + """Return True if the line is a header inclusion line.""" + return _is_msvc_inclusion_line(line) + + +def _is_msvc_inclusion_line(line): + return line.startswith(_MSVC_INCUSION_PREFIX) + + +def _is_gcc_inclusion_line(line): + """Return True if the line is a header inclusion line.""" + return line.startswith('#include') + + def _parse_inclusion_stacks(path, build_dir): """Parae headers inclusion stacks from file. @@ -118,7 +135,7 @@ def _process_hdr(level, hdr, current_level): with open(path) as f: for index, line in enumerate(f): line = line.rstrip() # Strip `\n` - if not line.startswith('.'): + if not _is_inclusion_line(line): # The remaining lines are useless for us break level, hdr = _parse_hdr_level_line(line) @@ -159,6 +176,10 @@ def _parse_hdr_level_line(line): Example: . ./common/rpc/rpc_client.h """ + return _parse_msvc_hdr_level_line(line) + + +def _parse_gcc_hdr_level_line(line): pos = line.find(' ') if pos == -1: return -1, '' @@ -169,6 +190,13 @@ def _parse_hdr_level_line(line): return level, hdr +def _parse_msvc_hdr_level_line(line): + line = line.removeprefix(_MSVC_INCUSION_PREFIX) + hdr = line.lstrip() + level = len(line) - len(hdr) + return level, hdr + + def _remove_build_dir_prefix(path, build_dir): """Remove the build dir prefix of path (e.g. build64_release/) Args: @@ -362,6 +390,7 @@ def check_file(src, full_src, is_header): if not path: console.warning('No inclusion file found for %s' % full_src) return + console.debug('Find inclusuon file %s for %s' % (path, full_src)) direct_hdrs, stacks = _parse_inclusion_stacks(path, self.build_dir) all_direct_hdrs.update(direct_hdrs) missing_dep_hdrs = set() diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 66cfe40b..25c12df1 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -183,6 +183,30 @@ def executable_file_name(self, name): Get the executable file name from the name. """ raise NotImplementedError + + def get_compile_commands(self, build_dir, cppflags, is_dump): + """ + Get the compile commands for the specified source file. + """ + raise NotImplementedError + + def get_link_command(self): + """ + Get the link command. + """ + raise NotImplementedError + + def get_shared_link_command(self): + """ + Get the link command for dynamic library. + """ + raise NotImplementedError + + def get_static_library_command(self): + """ + Get the static library command. + """ + raise NotImplementedError # To verify whether a header file is included without depends on the library it belongs to, # we use the gcc's `-H` option to generate the inclusion stack information, see @@ -270,6 +294,7 @@ def _cc_compile_command_wrapper_template(self, inclusion_stack_file, cuda=False) return template + @override def get_compile_commands(self, build_dir, cppflags, is_dump): cc_config = config.get_section('cc_config') cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags'] @@ -315,15 +340,23 @@ def _hdrs_command(self, flags, cppflags, includes): return ('export LC_ALL=C; %s || %s; ec=$$?; %s ${out}.err > ${out}; ' 'rm -f ${out}.err; exit $$ec') % (cmd1, cmd2, _INCLUSION_STACK_SPLITTER) + @override def get_link_command(self): return self.ld + self._get_link_args() + @override def get_shared_link_command(self): return self.ld + '-shared ' + self._get_link_args() + @override + def get_static_library_command(self): + flags = ''.join(config.get_item('cc_library_config', 'arflags')) + return 'rm -f $out; %s %s $out $in' % (self.ar, flags) + def _get_link_args(self): return ' -o ${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' + @override def filter_cc_flags(self, flag_list, language='c'): """Filter out the unrecognized compilation flags.""" flag_list = var_to_list(flag_list) @@ -439,6 +472,7 @@ def filter_cc_flags(self, flag_list, language='c'): return valid_flags + @override def get_compile_commands(self, build_dir, cppflags, is_dump): cc_config = config.get_section('cc_config') cflags, cxxflags = cc_config['cflags'], cc_config['cxxflags'] @@ -480,12 +514,19 @@ def _hdrs_command(self, flags, cppflags, includes): # The error message of the first command should be completely ignored. return cmd + @override def get_link_command(self): return self.ld + self._get_link_args() + @override def get_shared_link_command(self): return self.ld + ' /DLL ' + self._get_link_args() + @override + def get_static_library_command(self): + flags = ''.join(config.get_item('cc_library_config', 'arflags')) + return '%s /nologo %s /OUT:$out $in' % (self.ar, flags) + def _get_link_args(self): return ' /nologo /OUT:${out} ${intrinsic_linkflags} ${linkflags} ${target_linkflags} @${out}.rsp ${extra_linkflags}' From 3ba10a97e50f4c15a0d1d6e01faaf138de24b4ab Mon Sep 17 00:00:00 2001 From: phongchen Date: Fri, 6 Oct 2023 01:19:19 +0800 Subject: [PATCH 11/17] fixed header inclusion check --- src/blade/inclusion_check.py | 33 +++++++++++++++++++-------------- src/blade/util.py | 15 ++++++++++++--- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/blade/inclusion_check.py b/src/blade/inclusion_check.py index ae680a61..9292e560 100644 --- a/src/blade/inclusion_check.py +++ b/src/blade/inclusion_check.py @@ -31,6 +31,7 @@ def find_libs_by_header(hdr, hdr_targets_map, hdr_dir_targets_map): class GlobalDeclaration(object): """Global inclusion dependenct relationship declaration.""" + def __init__(self, declaration_file): self._declaration_file = declaration_file self._initialized = False @@ -118,7 +119,7 @@ def _parse_inclusion_stacks(path, build_dir): stacks, hdrs_stack = [], [] def _process_hdr(level, hdr, current_level): - if hdr.startswith('/'): + if os.path.isabs(hdr): skip_level = level elif hdr.startswith(build_dir): skip_level = level @@ -142,7 +143,8 @@ def _process_hdr(level, hdr, current_level): if level == -1: console.log('%s: Unrecognized line %s' % (path, line)) break - if level == 1 and not hdr.startswith('/'): + + if level == 1 and not os.path.isabs(hdr): direct_hdrs.append(_remove_build_dir_prefix(os.path.normpath(hdr), build_dir)) if level > current_level: if skip_level != -1 and level > skip_level: @@ -194,6 +196,9 @@ def _parse_msvc_hdr_level_line(line): line = line.removeprefix(_MSVC_INCUSION_PREFIX) hdr = line.lstrip() level = len(line) - len(hdr) + cwd = os.getcwd().lower() + if hdr.lower().startswith(cwd): + hdr = hdr[len(cwd)+1:] return level, hdr @@ -234,7 +239,6 @@ def __init__(self, target): inclusion_declaration_file = os.path.join(self.build_dir, 'inclusion_declaration.data') self.global_declaration = GlobalDeclaration(inclusion_declaration_file) - def _find_inclusion_file(self, src, is_header): """Find the '.H' file for the given src. @@ -276,9 +280,10 @@ def _check_direct_headers(self, full_src, direct_hdrs, suppressd_hdrs, libs = self.find_targets_by_private_hdr(hdr) if libs and self.key not in libs: msg.append(' "%s" is a private header file of %s' % ( - hdr, self._or_joined_libs(libs))) + hdr, self._or_joined_libs(libs))) continue - console.diagnose(self.source_location, 'debug', '"%s" is an undeclared header' % hdr) + console.diagnose(self.source_location, 'debug', + '"%s" is an undeclared header' % hdr) undeclared_hdrs.add(hdr) # We need also check suppressd_hdrs because target maybe not loaded in partial build if hdr not in suppressd_hdrs and not self.is_allowed_undeclared_hdr(hdr): @@ -316,7 +321,7 @@ def is_allowed_undeclared_hdr(self, hdr): return self.global_declaration.is_allowed_undeclared_hdr(hdr) def _header_undeclared_message(self, hdr): - msg = '"%s" is not declared in any cc target. ' % hdr + msg = '"%s" is not declared in any cc target. ' % util.to_unix_path(hdr) if util.path_under_dir(hdr, self.path): msg += 'If it belongs to this target, it should be declared in "src"' if self.type.endswith('_library'): @@ -359,7 +364,7 @@ def _check_generated_headers(self, full_src, stacks, direct_hdrs, suppressd_hdrs continue msg.append(' For %s' % self._hdr_declaration_message(generated_hdr)) if not stack: - msg.append(' In file included from "%s"' % full_src) + msg.append(' In file included from "%s"' % util.to_unix_path(full_src)) else: stack.reverse() msg.append(' In file included from %s' % self._hdr_declaration_message(stack[0])) @@ -395,16 +400,16 @@ def check_file(src, full_src, is_header): all_direct_hdrs.update(direct_hdrs) missing_dep_hdrs = set() self._check_direct_headers( - full_src, direct_hdrs, self.suppress.get(src, []), - missing_dep_hdrs, undeclared_hdrs, direct_check_msg) + full_src, direct_hdrs, self.suppress.get(src, []), + missing_dep_hdrs, undeclared_hdrs, direct_check_msg) for stack in stacks: all_generated_hdrs.add(stack[-1]) # But direct headers can not cover all, so it is still useful self._check_generated_headers( - full_src, stacks, direct_hdrs, - self.suppress.get(src, []), - missing_dep_hdrs, generated_check_msg) + full_src, stacks, direct_hdrs, + self.suppress.get(src, []), + missing_dep_hdrs, generated_check_msg) if missing_dep_hdrs: missing_details[src] = list(missing_dep_hdrs) @@ -418,10 +423,10 @@ def check_file(src, full_src, is_header): severity = self.severity if direct_check_msg: console.diagnose(self.source_location, severity, - '%s: Missing dependency declaration:\n%s' % (self.name, '\n'.join(direct_check_msg))) + '%s: Missing dependency declaration:\n%s' % (self.name, '\n'.join(direct_check_msg))) if generated_check_msg: console.diagnose(self.source_location, severity, - '%s: Missing indirect dependency declaration:\n%s' % (self.name, '\n'.join(generated_check_msg))) + '%s: Missing indirect dependency declaration:\n%s' % (self.name, '\n'.join(generated_check_msg))) ok = (severity != 'error' or not direct_check_msg and not generated_check_msg) diff --git a/src/blade/util.py b/src/blade/util.py index 466cc873..94ce3252 100644 --- a/src/blade/util.py +++ b/src/blade/util.py @@ -92,7 +92,7 @@ def lock_file(filename): msvcrt.locking(fd, msvcrt.LK_NBLCK, os.stat(fd).st_size) return fd, 0 except IOError as ex_value: - if msvcrt: # msvcrt did't set errno correctly + if msvcrt: # msvcrt did't set errno correctly return -1, errno.EAGAIN return -1, ex_value.errno @@ -183,6 +183,12 @@ def path_under_dir(path, dir): return dir == '.' or path == dir or path.startswith(dir) and path[len(dir)] == os.path.sep +def to_unix_path(path): + """Convert path to unix format + """ + return path.replace('\\', '/') + + def mkdir_p(path): """Make directory if it does not exist.""" try: @@ -274,15 +280,18 @@ def regular_variable_name(name): """convert some name to a valid identifier name""" return name.translate(_TRANS_TABLE) + # Some python 2/3 compatibility helpers. if _IN_PY3: def iteritems(d): return d.items() + def itervalues(d): return d.values() else: def iteritems(d): return d.iteritems() + def itervalues(d): return d.itervalues() @@ -394,7 +403,7 @@ def override(method): # TODO: check signature # sig = inspect.signature(method) error = "methid '%s' doesn't override any base class method" % method.__name__ - assert( any( hasattr(cls, method.__name__) for cls in base_classes ) ), error + assert (any(hasattr(cls, method.__name__) for cls in base_classes)), error return method @@ -421,7 +430,7 @@ def _get_bass_classes(stack): obj = derived_class_locals[components[0]] for c in components[1:]: - assert(inspect.ismodule(obj) or inspect.isclass(obj)) + assert (inspect.ismodule(obj) or inspect.isclass(obj)) obj = getattr(obj, c) base_classes[i] = obj From c195cd022131f620321940a4db9ac94e5bdf4f20 Mon Sep 17 00:00:00 2001 From: chen3feng Date: Wed, 11 Oct 2023 22:24:49 +0800 Subject: [PATCH 12/17] fix windows error --- src/blade/toolchain.py | 2 +- src/blade/workspace.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 25c12df1..4c2ef971 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -412,7 +412,7 @@ def _get_cc_version(self): version = '' returncode, stdout, stderr = run_command(self.cc, shell=True) if returncode == 0: - m = re.search('Compiler Version ([\d.]+)', stderr.strip()) + m = re.search(r'Compiler Version ([\d.]+)', stderr.strip()) if m: version = m.group(1) if not version: diff --git a/src/blade/workspace.py b/src/blade/workspace.py index 8449b97d..8e46f33d 100644 --- a/src/blade/workspace.py +++ b/src/blade/workspace.py @@ -113,9 +113,12 @@ def setup_build_dir(self): os.mkdir(build_dir) try: os.remove('blade-bin') - except os.error: + except: pass - os.symlink(os.path.abspath(build_dir), 'blade-bin') + try: + os.symlink(os.path.abspath(build_dir), 'blade-bin') + except OSError as e: + console.warning("Can't create symbolic link 'blade-bin', %s" % e) log_file = os.path.join(build_dir, 'blade.log') console.set_log_file(log_file) From 57286fe6897ed0c87561ba79df788e9a89f879c4 Mon Sep 17 00:00:00 2001 From: chen3feng Date: Thu, 12 Oct 2023 10:07:56 +0800 Subject: [PATCH 13/17] Fix lint --- src/blade/workspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blade/workspace.py b/src/blade/workspace.py index 8e46f33d..2e629da7 100644 --- a/src/blade/workspace.py +++ b/src/blade/workspace.py @@ -113,7 +113,7 @@ def setup_build_dir(self): os.mkdir(build_dir) try: os.remove('blade-bin') - except: + except OSError: pass try: os.symlink(os.path.abspath(build_dir), 'blade-bin') From 26c31cdaf8800315fca04b3cbbbceae402498edc Mon Sep 17 00:00:00 2001 From: chen3feng Date: Fri, 13 Oct 2023 02:54:44 -0700 Subject: [PATCH 14/17] Add .gitattributes --- .gitattributes | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..c9199558 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +* text=auto +*.bat text eol=crlf +*.cmd text eol=crlf +*.sh text eol=lf From f2525de11f51f91f08343388dac18b3ae78d526f Mon Sep 17 00:00:00 2001 From: chen3feng Date: Wed, 28 Feb 2024 14:55:49 +0800 Subject: [PATCH 15/17] Add setup.bat --- setup.bat | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.bat diff --git a/setup.bat b/setup.bat new file mode 100644 index 00000000..5ac15723 --- /dev/null +++ b/setup.bat @@ -0,0 +1,2 @@ +@echo off +set PATH=%~dp0;%PATH% From b93e4754e08e3b0473dc395c4883809767b79d5b Mon Sep 17 00:00:00 2001 From: chen3feng Date: Wed, 28 Feb 2024 18:40:26 +0800 Subject: [PATCH 16/17] Fix hdrs command error and deps format parsing error --- src/blade/target.py | 6 ++++-- src/blade/toolchain.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/blade/target.py b/src/blade/target.py index 3a90c67f..f9649919 100644 --- a/src/blade/target.py +++ b/src/blade/target.py @@ -90,7 +90,7 @@ def _parse_target(dep): result = ('', '', msgs) else: if path: - path = os.path.normpath(path) + path = os.path.normpath(path).replace('\\', '/') # windows result = (path, name, None) _parse_target.cache[dep] = result return result @@ -440,6 +440,7 @@ def _add_location_reference_target(self, m): def _unify_dep(self, dep): """Unify dep to key.""" (path, name, msgs) = _parse_target(dep) + console.warning(f'uni: dep={dep}, path={path}') if msgs: for msg in msgs: @@ -463,7 +464,7 @@ def _unify_dep(self, dep): else: # Depend on library in current directory path = self.path - + console.warning(f'_unify_dep: dep={dep}, path={path}') return '%s:%s' % (path, name) def _init_target_deps(self, deps): @@ -480,6 +481,7 @@ def _init_target_deps(self, deps): """ for d in deps: dkey = self._unify_dep(d) + console.warning(f'deps={deps}, dkey={dkey}') if dkey and dkey not in self.deps: self.deps.append(dkey) diff --git a/src/blade/toolchain.py b/src/blade/toolchain.py index 4c2ef971..ffd7d137 100644 --- a/src/blade/toolchain.py +++ b/src/blade/toolchain.py @@ -506,8 +506,10 @@ def _hdrs_command(self, flags, cppflags, includes): Command to generate cc inclusion information file for header file to check dependency missing. See https://learn.microsoft.com/en-us/cpp/build/reference/showincludes-list-include-files for details. """ + # If there is not #include in the file, findstr will fail, use "|| ver>nul" to ignore the error. + # https://stackoverflow.com/questions/22046780/whats-the-windows-command-shell-equivalent-of-bashs-true-command cmd = ('cmd.exe /c %s /nologo /c /E /Zs /TP /showIncludes %s %s /w ${cppflags} %s ${includes} ${in} 2>&1 >nul |' - ' findstr "Note: including file" >${out} && type ${out}'% ( + ' findstr "Note: including file" >${out} || ver>nul'% ( self.cc, ' '.join(flags), ' '.join(cppflags), includes)) # If the first cpp command fails, the second cpp command will be executed. From 063a37a8e635dd0695bd154e28619abbc2b21b6d Mon Sep 17 00:00:00 2001 From: chen3feng Date: Sun, 31 Mar 2024 17:30:59 +0800 Subject: [PATCH 17/17] Fix suffix and target finding --- .vscode/settings.json | 1 - src/blade/cc_targets.py | 2 ++ src/blade/load_build_files.py | 8 ++++++-- src/blade/target.py | 3 --- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 06696dec..bc392949 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "python.linting.pylintEnabled": true, "python.analysis.extraPaths": [ "src", "src/test" diff --git a/src/blade/cc_targets.py b/src/blade/cc_targets.py index 814b1d38..d8e64e63 100644 --- a/src/blade/cc_targets.py +++ b/src/blade/cc_targets.py @@ -1354,6 +1354,8 @@ def _cc_binary(self, objs, inclusion_check_result, dynamic_link): objs.append(scm) order_only_deps.append(scm) output = self._target_file_path(self.name) + if os.name == 'nt': + output += '.exe' self._cc_link(output, 'link', objs=objs, deps=usr_libs, sys_libs=sys_libs, linker_scripts=self.attr.get('lds_fullpath'), version_scripts=self.attr.get('vers_fullpath'), diff --git a/src/blade/load_build_files.py b/src/blade/load_build_files.py index 783af0df..3768066b 100644 --- a/src/blade/load_build_files.py +++ b/src/blade/load_build_files.py @@ -421,7 +421,6 @@ def load_targets(target_ids, excluded_targets, blade): # starting dirs mentioned in command line direct_targets, starting_dirs = _expand_target_patterns(blade, target_ids, excluded_trees) starting_dirs -= excluded_dirs - # to prevent duplicated loading of BUILD files processed_dirs = {} @@ -486,12 +485,17 @@ def under_excluded_trees(source_dir): for target_id in target_ids: source_dir, target_name = target_id.rsplit(':', 1) - if source_dir and not os.path.exists(source_dir): + + if source_dir == '': + source_dir = '.' + elif not os.path.exists(source_dir): _report_not_exist('Directory', source_dir, source_dir, blade) + skip_file = _check_under_skipped_dir(source_dir) if skip_file: console.warning('"%s" is under skipped directory due to "%s", ignored' % (target_id, skip_file)) continue + if target_name == '...': for root, dirs, files in os.walk(source_dir): # Note the dirs[:] = slice assignment; we are replacing the diff --git a/src/blade/target.py b/src/blade/target.py index f9649919..25ee71b3 100644 --- a/src/blade/target.py +++ b/src/blade/target.py @@ -440,7 +440,6 @@ def _add_location_reference_target(self, m): def _unify_dep(self, dep): """Unify dep to key.""" (path, name, msgs) = _parse_target(dep) - console.warning(f'uni: dep={dep}, path={path}') if msgs: for msg in msgs: @@ -464,7 +463,6 @@ def _unify_dep(self, dep): else: # Depend on library in current directory path = self.path - console.warning(f'_unify_dep: dep={dep}, path={path}') return '%s:%s' % (path, name) def _init_target_deps(self, deps): @@ -481,7 +479,6 @@ def _init_target_deps(self, deps): """ for d in deps: dkey = self._unify_dep(d) - console.warning(f'deps={deps}, dkey={dkey}') if dkey and dkey not in self.deps: self.deps.append(dkey)