python a_compile.py on an Apple M1 mac results in error

[ahrib]

Attempting to run python a_compile.py on an Apple M1 mac results in error.

If it helps, building with cmake works as expected when using a similar project https://github.com/neurolabusc/pigz-bench

environment details

  System Version: macOS 12.6 (21G115)
  Kernel Version: Darwin 21.6.0
  Boot Volume: Macintosh HD
  Boot Mode: Normal
  Secure Virtual Memory: Enabled
  System Integrity Protection: Enabled
  Model Name: MacBook Pro
  Model Identifier: MacBookPro18,3
  Chip: Apple M1 Pro
  Total Number of Cores: 10 (8 performance and 2 efficiency)
  Memory: 16 GB
  System Firmware Version: 7459.141.1
  OS Loader Version: 7459.141.1

full console output below

  CMake Deprecation Warning at /Users/andrej/Documents/Projects/pigzbench-master/zlib-madler/CMakeLists.txt:1 (cmake_minimum_required):
   Compatibility with CMake < 2.8.12 will be removed from a future version of
   CMake.
 
   Update the VERSION argument <min> value or use a ...<max> suffix to tell
   CMake that the project does not need compatibility with older versions.
 
 
 -- Configuring done
 CMake Warning (dev):
   Policy CMP0042 is not set: MACOSX_RPATH is enabled by default.  Run "cmake
   --help-policy CMP0042" for policy details.  Use the cmake_policy command to
   set the policy and suppress this warning.
 
   MACOSX_RPATH is not specified for the following targets:
 
    zlib
 
 This warning is for project developers.  Use -Wno-dev to suppress it.
 
 -- Generating done
 -- Build files have been written to: /Users/andrej/Documents/Projects/pigzbench-master/pigz-madler/build
 Consolidate compiler generated dependencies of target zlibstatic
 [ 80%] Built target zlibstatic
 Consolidate compiler generated dependencies of target pigz
 [ 85%] Linking C executable pigz
 [100%] Built target pigz
 /Users/andrej/Documents/Projects/pigzbench-master/pigz-madler/build/pigz->/Users/andrej/Documents/Projects/pigzbench-master/exe/pigz-madler
 -- Configuring done
 -- Generating done
 -- Build files have been written to: /Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build
 Consolidate compiler generated dependencies of target zlib
 [ 81%] Built target zlib
 Consolidate compiler generated dependencies of target pigz
 [ 86%] Linking C executable pigz
 Undefined symbols for architecture arm64:
   "_get_crc_table", referenced from:
       _main in pigz.c.o
 ld: symbol(s) not found for architecture arm64
 clang: error: linker command failed with exit code 1 (use -v to see invocation)
 make[2]: *** [pigz] Error 1
 make[1]: *** [CMakeFiles/pigz.dir/all] Error 2
 make: *** [all] Error 2
 Traceback (most recent call last):
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 815, in move
     os.rename(src, real_dst)
 FileNotFoundError: [Errno 2] No such file or directory: '/Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build/Release/pigz' -> '/Users/andrej/Documents/Projects/pigzbench-master/exe/pigz-cloudflare'
 
 During handling of the above exception, another exception occurred:
 
 Traceback (most recent call last):
   File "/Users/andrej/Documents/Projects/pigzbench-master/a_compile.py", line 174, in <module>
     compile_pigz(args.rebuild)
   File "/Users/andrej/Documents/Projects/pigzbench-master/a_compile.py", line 164, in compile_pigz
     shutil.move(pigzexe, outnm)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 835, in move
     copy_function(src, real_dst)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 434, in copy2
     copyfile(src, dst, follow_symlinks=follow_symlinks)
   File "/Users/andrej/mambaforge/envs/python/lib/python3.10/shutil.py", line 254, in copyfile
     with open(src, 'rb') as fsrc:
 FileNotFoundError: [Errno 2] No such file or directory: '/Users/andrej/Documents/Projects/pigzbench-master/pigz-cloudflare/build/Release/pigz'

[nmoinvaz]

zlib may need to target a particular branch or commit. I don't have an M1 so that is all I can help.

[neurolabusc]

@nmoinvaz this version of a_compile.py works for my M2. It uses the cmake from @ningfei. I do think it only builds the develop branch of zlib-ng, but it does build the system (Madler), CloudFlare and zlib-ng variations.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import argparse
import stat
import shutil
import subprocess
import platform
import zipfile
from distutils.dir_util import copy_tree

def rmtree(top):
    """Delete folder and contents: shutil.rmtree has issues with read-only files on Windows"""

    for (root, dirs, files) in os.walk(top, topdown=False):
        for name in files:
            filename = os.path.join(root, name)
            os.chmod(filename, stat.S_IWUSR)
            os.remove(filename)
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    os.rmdir(top)


def install_silesia_corpus():
    """Install popular Silesia corpus"""

    basedir = os.getcwd()
    corpusdir = os.path.join(basedir, 'silesia')
    if os.path.isdir(corpusdir):
        return
    try:
        os.mkdir(corpusdir)
    except OSError:
        print('Creation of the directory {} failed' .format(corpusdir))
    cmd = 'git clone https://github.com/MiloszKrajewski/SilesiaCorpus silesia'
    print("Installing "+corpusdir)
    subprocess.call(cmd, shell=True)
    os.chdir(corpusdir)
    fnm = 'README.md'
    if os.path.isfile(fnm):
        os.remove(fnm)
    ext = '.zip'
    for item in os.listdir(corpusdir):  # loop through items in dir
        print("+"+item)
        if item.endswith(ext):  # check for ".zip" extension
            file_name = os.path.abspath(item)  # get full path of files
            print(file_name)
            zip_ref = zipfile.ZipFile(file_name)  # create zipfile object
            zip_ref.extractall(corpusdir)  # extract file to dir
            zip_ref.close()  # close file
            os.remove(file_name)  # delete zipped file
    os.chdir(basedir)


def install_neuro_corpus():
    """Install neuroimaging corpus"""

    basedir = os.getcwd()
    corpusdir = os.path.join(basedir, 'corpus')
    if os.path.isdir(corpusdir):
        return
    try:
        os.mkdir(corpusdir)
    except OSError:
        print('Creation of the directory {} failed' .format(exedir))
    cmd = 'git clone https://github.com/neurolabusc/zlib-bench.git'
    subprocess.call(cmd, shell=True)
    indir = os.path.join(basedir, 'zlib-bench', 'corpus')
    tmpfile = os.path.join(indir, 'README')
    if os.path.isfile(tmpfile):
        os.remove(tmpfile)
    copy_tree(indir, corpusdir)
    indir = os.path.join(basedir, 'zlib-bench')
    rmtree(indir)


def compile_pigz(rebuild=True):
    """compile variants of pigz"""

    methods = ['Cloudflare', 'ng','System']
    basedir = os.getcwd()
    exedir = os.path.join(basedir, 'exe')
    if os.path.isdir(exedir):
        rmtree(exedir)
    os.mkdir(exedir)
    repodir = os.path.join(basedir, 'pigz')
    if os.path.isdir(repodir):
        rmtree(repodir)
    ext = ''
    if platform.system() == 'Windows':
        ext = '.exe'
    cmd = 'git clone https://github.com/ningfei/pigz.git'
    subprocess.call(cmd, shell=True)
    if os.path.isdir(exedir):
        rmtree(exedir)
    os.mkdir(exedir)
    builddir = os.path.join(repodir, 'build')
    pigzexe = os.path.join(builddir, 'bin', 'pigz') + ext
    for method in methods:
        os.chdir(repodir)
        if os.path.isdir(builddir):
          rmtree(builddir)
        os.mkdir(builddir)
        os.chdir(builddir)
        cmd = 'cmake -DZLIB_IMPLEMENTATION=' + method + ' ..'
        subprocess.call(cmd, shell=True)
        cmd = 'make'
        if platform.system() == 'Windows':
            cmd = 'cmake --build . --config Release'
        subprocess.call(cmd, shell=True)
        outnm = os.path.join(exedir, 'pigz' + method + ext)
        print (pigzexe + '->' + outnm)
        shutil.move(pigzexe, outnm)


if __name__ == '__main__':
    """compile variants of pigz and sample compression corpus"""

    install_neuro_corpus()
    install_silesia_corpus()
    compile_pigz()