forgot to commit, add _no_io_ and *_retcodes_

.gitignore

@@ -13,6 +13,7 @@
 /.virtualenv/
 /requirements.txt
 /python_modules
+/pybuildtool/.coverage
 
 
 # waf

build.yml

@@ -35,6 +35,7 @@ pybuildtool:
   unittest:
     shell:
       options:
+        _noop_retcodes_: [5]
         command: ./run pybin coverage run --include=./* -m py.test
       rule_in:
         - "{_1}/pylint"
@@ -48,6 +49,7 @@ pybuildtool:
   coverage:
     shell:
       options:
+        _noop_retcodes_: [1]
         command: ./run pybin coverage report -m --skip-covered
       rule_in:
         - "{_1}/unittest/shell"

etc/pylint.rc

@@ -139,7 +139,20 @@ disable=print-statement,
         deprecated-sys-function,
         exception-escape,
         comprehension-escape,
-        too-many-locals
+        too-many-locals,
+        duplicate-code,
+        missing-class-docstring,
+        missing-function-docstring,
+        missing-module-docstring,
+        too-many-arguments,
+        invalid-name,
+        protected-access,
+        consider-using-f-string,
+        too-many-branches,
+        too-many-statements,
+        too-few-public-methods,
+        too-many-instance-attributes,
+        too-many-ancestors
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

etc/runner.json

@@ -9,8 +9,8 @@
         "lib.fireh_runner.setup_modules.python",
         "lib.fireh_runner.setup_modules.pybuildtool"
     ],
-    "python_version": "3.8",
-    "waf_version": "2.0.20",
+    "python_version": "3.9",
+    "waf_version": "2.0.22",
 
     "package_name": "PyBuildTool",
     "default_project": "pybuildtool",

pybuildtool/core/context.py

@@ -7,7 +7,7 @@ class WatchContext(Context.Context):
     variant = ''
 
     def __init__(self, **kw):
-        super(WatchContext, self).__init__(**kw)
+        super().__init__(**kw)
 
         self.top_dir = kw.get('top_dir', Context.top_dir)
         self.out_dir = kw.get('out_dir', Context.out_dir)

pybuildtool/core/rule.py

@@ -1,8 +1,7 @@
 from hashlib import md5
 import os
 import re
-
-from ..misc.collections_utils import make_list
+#-
 from ..misc.path import expand_resource
 
 
@@ -26,9 +25,8 @@ def __init__(self, group, config, file_in, file_out, depend_in, extra_out):
         for fs in (self.file_in, self.depend_in):
             self._expand_input_wilcards(fs)
 
-        # normalize `replace_patterns`, must be a list
-        self.conf['replace_patterns'] = make_list(
-                self.conf.get('replace_patterns'))
+        # normalize `_replace_patterns_`, must be a list of list
+        self.conf.setdefault('_replace_patterns_', [])
 
 
     def _expand_input_wilcards(self, items):
@@ -86,10 +84,8 @@ def files(self):
             if is_dir:
                 for fi in self.file_in:
                     fofi = fi
-                    replace_patterns = self.conf.get('replace_patterns', False)
-                    if replace_patterns:
-                        for (pat, rep) in replace_patterns:
-                            fofi = re.sub(pat, rep, fofi)
+                    for (pat, rep) in self.conf['_replace_patterns_']:
+                        fofi = re.sub(pat, rep, fofi)
                     basedir = self.conf.get('_source_basedir_', False)
                     if basedir:
                         basedir = expand_resource(self.group, basedir)
@@ -148,10 +144,8 @@ def rules(self):
                     continue
 
                 fofi = fi
-                replace_patterns = self.conf.get('replace_patterns', False)
-                if replace_patterns:
-                    for (pat, rep) in replace_patterns:
-                        fofi = re.sub(pat, rep, fofi)
+                for (pat, rep) in self.conf['_replace_patterns_']:
+                    fofi = re.sub(pat, rep, fofi)
                 # use basedir to produce file_out
                 basedir = self.conf.get('_source_basedir_', False)
                 if basedir:

pybuildtool/core/task.py

@@ -1,3 +1,41 @@
+"""
+Base class for pybuildtools tools.
+
+Options:
+
+    * _source_excluded_ : list, None
+                        : Pretend source files (*_in values) don't exist.
+
+    * _source_basedir_ : str, None
+                       : Create files in output dir, relative path to source
+                       : base directory.
+
+    * _source_grouped_ : bool, None
+                       : Don't create separate tasks for every input files, have
+                       : them as input files of a single task.
+                       : Actually I'm not so sure what this does, something like
+                       : have them all as arguments to shell command?
+
+    * _noop_retcodes_ : list, None
+                      : If Task.perform() returns these, pretend nothing
+                      : happened.
+
+    * _success_retcodes_ : list, None
+                         : If Task.perform() returns these, pretend as if it
+                         : returns 0 or a success.
+
+    * _replace_patterns_ : list, None
+                         : If the output is a directory, you can rename the
+                         : output files based on the source files.
+                         : This is a list of list.
+                         : The list elements consist of two items: python regex
+                         : and replacement.
+
+    * _no_io_ : bool, False
+              : This task doesn't need inputs or outputs.
+              : Only works if written in build.yml.
+
+"""
 import os
 from copy import deepcopy
 from time import time
@@ -23,7 +61,7 @@ class Task(BaseTask):
     _id = None
 
     def __init__(self, group, config, *args, **kwargs):
-        super(Task, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._id = uuid4().hex
 
         # Task's configuration can be declared higher in the build tree,
@@ -57,7 +95,7 @@ def prepare(self):
         pass
 
 
-    def prepare_args(self):
+    def prepare_args(self): # pylint:disable=no-self-use
         return []
 
 
@@ -90,26 +128,36 @@ def prepare_shadow_jutsu(self):
                 self.file_out.append(path)
 
 
-    def finalize_shadow_jutsu(self):
+    def finalize_shadow_jutsu(self, create_only=False):
         for filename in self.token_out:
+            if create_only and os.path.exists(filename):
+                continue
             try:
                 os.makedirs(os.path.dirname(filename))
             except OSError:
                 pass
-            with open(filename, 'w') as f:
+            with open(filename, 'w', encoding='utf-8') as f:
                 f.write(str(time()))
 
 
     def run(self):
         self.prepare_shadow_jutsu()
         self.prepare()
         ret = self.perform()
-        if ret == 0:
-            self.finalize_shadow_jutsu()
+
+        create_only = False
+        if ret in make_list(self.conf.get('_noop_retcodes_')):
+            create_only = True
+            ret = None
+        elif ret in make_list(self.conf.get('_success_retcodes_')):
+            ret = None
+        if not ret:
+            self.finalize_shadow_jutsu(create_only)
         return ret
 
 
-    def is_production(self):
+    @staticmethod
+    def is_production():
         return os.environ.get('PROJECT_VARIANT_IS_PRODUCTION') == '1'
 
 

pybuildtool/misc/collections_utils.py

@@ -44,7 +44,7 @@ def data_merge(a, b):
     except TypeError as e:
         raise Exception(
                 'TypeError "%s" in key "%s" when merging "%s" into "%s"' %\
-                (e, key, b, a))
+                (e, key, b, a)) from e
 
     return a
 

pybuildtool/misc/python.py

@@ -1,15 +1,8 @@
-try:
-    from importlib.machinery import SourceFileLoader
-    from importlib.util import module_from_spec, spec_from_loader
-except ImportError:
-    import imp
-
+from importlib.machinery import SourceFileLoader
+from importlib.util import module_from_spec, spec_from_loader
 
 def load_module_from_filename(filename, name):
-    try:
-        spec = spec_from_loader(name, SourceFileLoader(name, filename))
-        mod = module_from_spec(spec)
-        spec.loader.exec_module(mod)
-    except NameError:
-        mod = imp.load_source(name, filename)
+    spec = spec_from_loader(name, SourceFileLoader(name, filename))
+    mod = module_from_spec(spec)
+    spec.loader.exec_module(mod)
     return mod

pybuildtool/misc/resource.py

@@ -1,8 +1,20 @@
+import hashlib
+import mmap
 import os
 import re
 from ..core.group import Group
 from .collections_utils import make_list
 
+def get_filehash(filename):
+    if not os.path.exists(filename):
+        return None
+    hasher = hashlib.sha1()
+    with open(filename, 'rb') as f:
+        with mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ) as mm:
+            hasher.update(mm)
+    return hasher.digest()
+
+
 def get_source_files(conf, bld):
     """Collect raw file inputs."""
     groups = {}
@@ -11,7 +23,8 @@ def get_source_files(conf, bld):
     def parse_group(group_name, config, level):
         groups['_%s' % level] = group_name
 
-        if group_is_leaf(config):
+        options = config.pop('options', {})
+        if group_is_leaf(config, options):
             group_files = make_list(config.get('raw_file_in')) +\
                     make_list(config.get('raw_depend_in'))
 
@@ -25,9 +38,6 @@ def parse_group(group_name, config, level):
             return
 
         for subgroup in config:
-            if subgroup == 'options':
-                continue
-
             for f in parse_group(subgroup, config[subgroup], level + 1):
                 yield f
 
@@ -39,12 +49,13 @@ def parse_group(group_name, config, level):
             yield f
 
 
-def group_is_leaf(group):
+def group_is_leaf(group, options):
     """The lowests in the group tree are the tools."""
 
     return any(x in group for x in ('file_in', 'raw_file_in', 'file_out',
             'raw_file_out', 'depend_in', 'raw_depend_in', 'extra_out',
-            'raw_extra_out', 'rule_in'))
+            'raw_extra_out', 'rule_in'))\
+            or options.get('_no_io_', False)
 
 
 def prepare_targets(conf, bld):
@@ -100,7 +111,7 @@ def parse_group(group_name, config, level, parent_group):
         groups[g.get_name()] = g
         pattern = g.get_patterns()
 
-        if group_is_leaf(config):
+        if group_is_leaf(config, options):
             original_file_in = make_list(config.get('file_in'))
             file_in = list(_parse_input_listing(original_file_in, pattern))
             _add_raw_files(make_list(config.get('raw_file_in')), file_in,
@@ -129,12 +140,16 @@ def parse_group(group_name, config, level, parent_group):
                     token_names = bld._token_names[rule_in]
                     for f in token_names:
                         depend_in.append(f)
-                except (KeyError, AttributeError) as e:
+                except (KeyError, AttributeError):
                     print((parent_group.get_name(), group_name, dict(config),
                             level))
 
-                    bld.fatal('rule "%s" not found in: %s' % (rule_in,
-                            ', '.join(bld._token_names.keys())))
+                    if hasattr(bld, '_token_names'):
+                        bld.fatal("rule '%s' not found in: %s" % (rule_in,
+                                ', '.join(bld._token_names.keys())))
+                    else:
+                        bld.fatal("rule '%s' not found" % rule_in +\
+                                ", does it have *_in or *_out?")
 
             g(file_in=file_in, file_out=file_out, depend_in=depend_in,
                     extra_out=extra_out)

pybuildtool/misc/yaml_utils.py

@@ -20,10 +20,10 @@ class OrderedDictYAMLLoader(yaml.Loader):
     def __init__(self, *args, **kwargs):
         yaml.Loader.__init__(self, *args, **kwargs)
 
-        self.add_constructor(u'tag:yaml.org,2002:map',
+        self.add_constructor('tag:yaml.org,2002:map',
                 type(self).construct_yaml_map)
 
-        self.add_constructor(u'tag:yaml.org,2002:omap',
+        self.add_constructor('tag:yaml.org,2002:omap',
                 type(self).construct_yaml_map)
 
     def construct_yaml_map(self, node):

pybuildtool/tools/ansibleplay.py

@@ -31,10 +31,11 @@
 import os
 import sys
 import six
+from waflib import Logs # pylint:disable=import-error
 from yaml import safe_load as yaml_load
+#-
 from pybuildtool import BaseTask, expand_resource, make_list
 from pybuildtool.misc.python import load_module_from_filename
-from waflib import Logs # pylint:disable=import-error
 
 tool_name = __name__
 
@@ -45,7 +46,7 @@ class Task(BaseTask):
     items = None
 
     def __init__(self, *args, **kwargs):
-        super(Task, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.context = {}
         self.items = []
 
@@ -108,7 +109,7 @@ def prepare(self):
             if yaml_file is None:
                 self.bld.fatal('"context_yaml" for %s has invalid value' %\
                         tool_name.capitalize())
-            with open(yaml_file, 'r') as f:
+            with open(yaml_file, 'r', encoding='utf-8') as f:
                 self.context.update(yaml_load(f))
 
         # Python context

pybuildtool/tools/clean-css.py

@@ -39,7 +39,7 @@
 class Task(BaseTask):
 
     conf = {
-        'replace_patterns': ((r'\.css$', '.min.css'),),
+        '_replace_patterns_': ((r'\.css$', '.min.css'),),
     }
     name = tool_name