Appleclang (WIP) (#222)

* untangle clang and appleclang

Author: DmitriyMusatkin

.gitignore

@@ -671,3 +671,5 @@ venv/
 # single dist/index.js file and commit THAT instead.
 node_modules/
 !.github/actions/check-submodules/dist/
+
+.DS_Store

README.md

@@ -31,7 +31,7 @@ Usage: ```builder.pyz [build|inspect|<action-name>] [spec] [OPTIONS]```
 * ```--cmake-extra``` - Extra cmake config arg applied to all projects. e.g ```--cmake-extra=-DBUILD_SHARED_LIBS=ON```. May be specified multiple times.
 * ```--coverage``` - Generate the test coverage report and upload it to codecov. Only supported when using cmake and gcc as compiler, error out on other cases. Use `--coverage-include` and `--coverage-exclude` to report the needed coverage file. The default code coverage report will include everything in the `source/` directory
     * ```--coverage-include``` - The relative (based on the project directory) path of files and folders to include in the test coverage report. May be specified multiple times.
-    * ```--coverage-exclude``` - The relative (based on the project directory) path of files and folders to exclude in the test coverage report. May be specified multiple times. Note: the include can override the exlucde path.
+    * ```--coverage-exclude``` - The relative (based on the project directory) path of files and folders to exclude in the test coverage report. May be specified multiple times. Note: the include can override the exclude path.
 
 ### Supported Targets:
 * linux: x86|i686, x64|x86_64, armv6, armv7, arm64|armv8|aarch64|arm64v8
@@ -52,7 +52,7 @@ work.
 
 ### Configuration (builder.json)
 Each project has a configuration file: builder.json in the root of the project. It may also have a .builder folder which contains python
-scripts that will be automatically loaded when the project is found by the builder. Both of these are techncially optional, if the builder
+scripts that will be automatically loaded when the project is found by the builder. Both of these are technically optional, if the builder
 finds something that looks like a git repo full of code in a directory with the same name as the project it is searching for, it will use
 that instead. There are a few external dependencies (s2n and libcrypto, for instance) which are configured by scripts embedded in builder
 (see imports/).
@@ -78,7 +78,7 @@ See builder/data.py for more info/defaults/possible values.
     // Whether or not this project needs a C/C++ compiler
     "needs_compiler": true,
 
-    // Variables for use in interpolation throughout the config. Note that these can be overriden
+    // Variables for use in interpolation throughout the config. Note that these can be overridden
     // per host/os/target/architecture/compiler/version (see below)
     // Variables can be references in braces: e.g. {my_variable}
     //

builder/core/data.py

@@ -478,9 +478,22 @@ class PKG_TOOLS(Enum):
             'default': {}
         }
     },
+    'appleclang': {
+        'hosts': ['macos'],
+        'targets': ['macos', 'ios', 'tvos', 'watchos'],
+
+        'versions': {
+            'default': {},
+
+            '11': {},
+            '12': {},
+            '13': {},
+            '14': {},
+        },
+    },
     'clang': {
-        'hosts': ['linux', 'macos', 'openbsd'],
-        'targets': ['linux', 'macos', 'openbsd', 'ios', 'tvos', 'watchos'],
+        'hosts': ['linux', 'openbsd'],
+        'targets': ['linux', 'openbsd'],
 
         'imports': ['llvm'],
 
@@ -514,17 +527,13 @@ class PKG_TOOLS(Enum):
                 'cmake_args': ['-DENABLE_FUZZ_TESTS=ON'],
                 'apt_compiler_packages': ['libstdc++-9-dev'],
             },
-            # 10 and 11 are XCode Apple clang/LLVM
-            '10': {
-                '!cmake_args': [],
-            },
-            '11': {
-                '!cmake_args': [],
-            },
+            '10': {},
+            '11': {},
             '12': {},
             '13': {},
             '14': {},
-            '15': {}
+            '15': {},
+            '16': {}
         },
         'architectures': {
             # No fuzz tests on ARM

builder/core/spec.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0.
 
 import os
-import sys
 
 from builder.core.data import *
 from builder.core.host import current_host, current_os, current_arch, normalize_arch
@@ -89,3 +88,14 @@ def __str__(self):
 
     def __repr__(self):
         return self.name
+
+    def update_compiler(self, compiler, compiler_version):
+        self.compiler = compiler
+        self.compiler_version = compiler_version
+
+        self.name = '-'.join([self.host, self.compiler,
+                              self.compiler_version, self.target, self.arch])
+        if self.downstream:
+            self.name += "-downstream"
+
+        validate_spec(self)

builder/core/toolchain.py

@@ -19,7 +19,7 @@ def _compiler_version(cc):
             # Apple clang
             m = re.match('Apple (LLVM|clang) version (\d+)', text)
             if m:
-                return 'clang', m.group(2)
+                return 'appleclang', m.group(2)
             # LLVM clang
             m = re.match('.*(LLVM|clang) version (\d+)', text)
             if m:
@@ -59,6 +59,14 @@ def _clang_versions():
     return versions
 
 
+def _appleclang_versions():
+    versions = [v for v in COMPILERS['appleclang']
+                ['versions'].keys() if v != 'default']
+    versions.sort()
+    versions.reverse()
+    return versions
+
+
 def _gcc_versions():
     versions = [v for v in COMPILERS['gcc']
                 ['versions'].keys() if v != 'default']
@@ -150,7 +158,7 @@ def cxx_compiler_path(self):
         compiler = self.compiler
         if self.compiler == 'default':
             compiler = Toolchain.default_compiler()[0]
-        if compiler == 'clang':
+        if compiler == 'clang' or compiler == 'appleclang':
             return Toolchain.find_compiler_tool(compiler, 'clang++', self.compiler_version)[0]
         elif compiler == 'gcc':
             return Toolchain.find_compiler_tool(compiler, 'g++', self.compiler_version)[0]
@@ -176,6 +184,14 @@ def find_llvm_tool(name, version=None):
         versions = [version] if version else _clang_versions()
         return _find_compiler_tool(name, versions)
 
+    @staticmethod
+    def find_apple_llvm_compiler(name, version=None):
+        """ Finds apple versions of clang compilers at a specific version, or the latest one available.
+            Note: apple does not ship tools like clang-tidy, so if installed
+            those will follow regular llvm versioning """
+        versions = [version] if version else _appleclang_versions()
+        return _find_compiler_tool(name, versions)
+
     @staticmethod
     def find_msvc(version=None):
         """ Finds MSVC at a specific version, or the latest one available """
@@ -214,7 +230,12 @@ def _find_msvc(version, install_vswhere=True):
     def find_compiler(compiler, version=None):
         """ Returns path, found_version for the requested compiler if it is installed """
         if compiler == 'clang':
-            return Toolchain.find_llvm_tool(compiler, version)
+            if current_os() == "macos":
+                return Toolchain.find_apple_llvm_compiler(compiler, version)
+            else:
+                return Toolchain.find_llvm_tool(compiler, version)
+        elif compiler == 'appleclang':
+            return Toolchain.find_apple_llvm_compiler('clang', version)
         elif compiler == 'gcc':
             return Toolchain.find_gcc_tool(compiler, version)
         elif compiler == 'msvc':
@@ -226,20 +247,13 @@ def find_compiler_tool(compiler, tool, version=None):
         """ Returns path, found_version for the requested tool if it is installed """
         if compiler == 'clang':
             return Toolchain.find_llvm_tool(tool, version)
+        elif compiler == 'appleclang':
+            return Toolchain.find_apple_llvm_compiler(tool, version)
         elif compiler == 'gcc':
             return Toolchain.find_gcc_tool(tool, version)
 
         return None, None
 
-    @staticmethod
-    def compiler_packages(compiler, version):
-        """ Returns a list of packages required to use the requested compiler """
-        compiler_config = COMPILERS.get(compiler, {}).get(
-            'versions', {}).get(version, None)
-        if compiler_config:
-            return compiler_config.get('packages', [])
-        return []
-
     @staticmethod
     def all_compilers():
         """ Returns a list of tuples of all available (compiler, version) """
@@ -248,10 +262,13 @@ def all_compilers():
             path, _version = Toolchain.find_gcc_tool('gcc', version)
             if path:
                 compilers.append(('gcc', version))
-        for version in _clang_versions():
+
+        is_mac = current_os() == "macos"
+        versions = _appleclang_versions() if is_mac else _clang_versions()
+        for version in versions:
             path, _version = Toolchain.find_llvm_tool('clang', version)
             if path:
-                compilers.append(('clang', version))
+                compilers.append(('appleclang' if is_mac else 'clang', version))
         if current_os() == 'windows':
             for version in _msvc_versions():
                 path, _version = Toolchain.find_msvc(version)
@@ -275,7 +292,11 @@ def _find_compiler():
             compiler = None
             version = None
             platform = current_os()
-            if platform != 'windows':
+
+            if platform == 'windows':
+                compiler = 'msvc'
+                version = Toolchain.find_msvc()[1]
+            else:
                 # resolve CC and /usr/bin/cc
                 for env_cc in (util.where(os.environ.get('CC', None)), util.where('cc')):
                     if env_cc:
@@ -284,8 +305,8 @@ def _find_compiler():
                             return cc, ccver
 
                 # Try to find clang or gcc
-                clang_path, clang_version = Toolchain.find_llvm_tool(
-                    'clang')
+                clang_path, clang_version = Toolchain.find_llvm_tool('clang')
+                print(clang_path)
                 gcc_path, gcc_version = Toolchain.find_gcc_tool('gcc')
                 if clang_path:
                     compiler = 'clang'
@@ -297,9 +318,6 @@ def _find_compiler():
                     print(
                         'Neither GCC or Clang could be found on this system, perhaps not installed yet?')
 
-            else:
-                compiler = 'msvc'
-                version = Toolchain.find_msvc()[1]
             if not compiler or not version:
                 print('WARNING: Default compiler could not be found')
 

builder/main.py

@@ -294,6 +294,12 @@ def main():
         print('No project specified and no project found in current directory')
         sys.exit(1)
 
+    if env.config.get('needs_compiler', True):
+        env.toolchain = Toolchain(spec=env.spec)
+        env.spec.update_compiler(env.toolchain.compiler, env.toolchain.compiler_version)
+        if env.spec.compiler == 'default' or env.spec.compiler_version == 'default':
+            raise Exception("Failed to resolve default compiler. None installed?")
+
     print('Using Spec:')
     print('  Host: {} {}'.format(spec.host, current_arch()))
     print('  Target: {} {}'.format(spec.target, spec.arch))
@@ -302,9 +308,6 @@ def main():
     if not env.config.get('enabled', True):
         raise Exception("The project is disabled in this configuration")
 
-    if env.config.get('needs_compiler', True):
-        env.toolchain = Toolchain(spec=env.spec)
-
     if args.dump_config:
         from pprint import pprint
         print('Spec: ', end='')