Populate options

bin/ecbundle-populate

@@ -17,8 +17,26 @@ import sys
 from argparse import SUPPRESS, ArgumentParser, RawTextHelpFormatter
 
 sys.path.insert(0, os.path.realpath(os.path.dirname(os.path.realpath(__file__))+'/..'))
-from ecbundle import CacheCreator, Timer
+from ecbundle.bundle import Bundle
+from ecbundle.cache import CacheCreator
 from ecbundle.logging import DEBUG, colors, error, logger, success
+from ecbundle.util import Timer, fullpath
+
+
+def get_src_dir():
+    src_dir = "source"
+    for i, arg in enumerate(sys.argv):
+        if arg.startswith("--src-dir="):
+            src_dir = arg.split("=")[1]
+        elif arg.startswith("--src-dir"):
+            src_dir = sys.argv[i + 1]
+    src_dir = fullpath(src_dir)
+    return src_dir
+
+
+def get_populate_options():
+    bundle = Bundle(get_src_dir() + '/bundle.yml')
+    return bundle.populate_options()
 
 
 def main():
@@ -54,6 +72,21 @@ def main():
     parser.add_argument('--artifacts-dir',
                         help='Location where downloaded data will be stored (default={{src-dir}}/artifacts')
 
+    try:
+        typemap = {'str': str, 'int': int}
+        options = get_populate_options()
+        for opt in options:
+            arg_opts = {}
+            if opt.type():
+                if opt.type() in typemap:
+                    arg_opts['type'] = typemap[opt.type()]
+            else:
+                arg_opts['action'] = 'store_true'
+            parser.add_argument('--' + opt.name(), help=opt.help(), **arg_opts)
+
+    except IOError:
+        pass
+
     # --------------------------------------------------------------------------
 
     # Close parser and populate variable args

ecbundle/bundle.py

@@ -75,6 +75,14 @@ def options(self):
         # Python 2:
         # return [Option(name=opt.items()[0][0], **opt.items()[0][1]) for opt in optconf]
 
+    def populate_options(self):
+        optconf = self.get("populate_options", [])
+        return [
+            Option(name=key, **dict(val.items()))
+            for opt in optconf
+            for (key, val) in opt.items()
+        ]
+
     def file(self):
         return self.filepath
 

ecbundle/cache.py

@@ -10,15 +10,28 @@
 from pathlib import Path
 
 from .bundle import Bundle
+from .logging import debug, info
 from .util import execute, fullpath
 
 __all__ = ["CacheCreator"]
 
 
+def get_populate_options(bundle, config):
+    options = bundle.populate_options()
+    return dict(
+        (pop.split("=", 1)[0], pop.split("=", 1)[1])
+        for opt in options
+        for pop in opt.populate(config.get(opt.key(), None))
+        if opt.key() in config
+    )
+
+
 class CacheCreator(object):
     def __init__(self, **kwargs):
         self.config = kwargs
         self.dryrun = self.config["dryrun"]
+        self.capture_output = self.config.get("capture_output", False)
+        self.log = debug if self.config.get("silent", False) else info
 
     def bundle(self):
         return Bundle(self.src_dir() + "/bundle.yml", env=False)
@@ -35,12 +48,26 @@ def artifacts_dir(self):
 
     def create(self):
         bundle = self.bundle()
+        options = get_populate_options(bundle, self.config)
+
         for project in bundle.projects():
+
+            if project.populate():
+                for pop in project.populate():
+                    k, v = pop.split("=", 1)
+                    opt = options.get(k, None)
+                    if not opt or opt.replace('"', "") == "None":
+                        options.update({k: v})
+
             create_cache = Path(self.src_dir()) / project.name() / "populate"
             if create_cache.is_file():
-                execute(
+                # We also might want to capture the output of the populate script
+                _output = execute(
                     str(create_cache),
                     dryrun=self.dryrun,
-                    env=dict(os.environ, ARTIFACTS_DIR=self.artifacts_dir()),
+                    env=dict(os.environ, ARTIFACTS_DIR=self.artifacts_dir(), **options),
+                    capture_output=self.capture_output,
                 )
+                if self.capture_output:
+                    self.log(_output)
         return 0

ecbundle/option.py

@@ -41,13 +41,26 @@ def cmake(self, value=None):
         else:
             return None
 
+    def populate(self, value=None):
+        if self.get("populate"):
+            populate_str = self.get("populate")
+            if "{{value}}" in populate_str:
+                populate_str = populate_str.replace("{{value}}", '"' + str(value) + '"')
+            return splitted(populate_str)
+        else:
+            return None
+
     def type(self):
         if self.get("type"):
             return self.get("type")
         if self.get("cmake"):
             cmake_str = self.get("cmake")
             if "{{value}}" in cmake_str:
                 return "str"
+        if self.get("populate"):
+            populate_str = self.get("populate")
+            if "{{value}}" in populate_str:
+                return "str"
         return None
 
     def __str__(self):

ecbundle/project.py

@@ -52,6 +52,13 @@ def cmake(self):
         else:
             return None
 
+    def populate(self):
+        if self.get("populate"):
+            split = [c.strip() for c in splitted(self.get("populate"))]
+            return [c for c in split if c]
+        else:
+            return None
+
     def dir(self):
         return self.get("dir")
 

tests/bundle_populate/bundle.yml

@@ -0,0 +1,28 @@
+---
+### Bundle
+
+name    : test-bundle-populate
+
+projects :
+
+    - project1 :
+        git     : ${BITBUCKET}/user/project1
+        version : 0.0.1
+        bundle  : false
+        populate: >
+                POPULATE_OPTION_1=0
+                POPULATE_OPTION_2=A
+
+populate_options:
+
+    - non-zero-option-1 :
+       help : Specify a non-default option 1
+       populate : POPULATE_OPTION_1=1
+
+    - option-2 :
+       help : Specify a non-default option 2
+       populate : POPULATE_OPTION_2={{value}}
+
+    - set-both-options :
+       help : Specify both options with single flag
+       populate : POPULATE_OPTION_1=1 POPULATE_OPTION_2={{value}}

tests/bundle_populate/test_populate.py

@@ -0,0 +1,110 @@
+# (C) Copyright 2020- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+
+
+import shutil
+from os import chmod
+from pathlib import Path
+
+import pytest
+from conftest import Watcher
+
+from ecbundle import CacheCreator, logger
+
+
+@pytest.fixture
+def watcher():
+    return Watcher(logger=logger)
+
+
+@pytest.fixture
+def here():
+    return Path(__file__).parent.resolve()
+
+
+@pytest.fixture
+def populate(here):
+    """
+    Add a dummy populate script.
+    """
+
+    src_dir = here / "source"
+    if not src_dir.exists():
+        src_dir.mkdir()
+
+    project_dir = src_dir / "project1"
+    if not project_dir.exists():
+        project_dir.mkdir()
+
+    populate_script = """#!/usr/bin/env bash
+echo "ARTIFACTS_DIR=$ARTIFACTS_DIR"
+echo "POPULATE_OPTION_1=$POPULATE_OPTION_1"
+echo "POPULATE_OPTION_2=$POPULATE_OPTION_2"
+"""
+
+    script_path = project_dir / "populate"
+    if not script_path.exists():
+        with open(script_path, "w") as file:
+            file.write(populate_script)
+        chmod(script_path, 0o755)
+
+    yield src_dir, project_dir
+
+    # Clean up after ourselves
+    if src_dir.exists():
+        shutil.rmtree(src_dir)
+
+
+@pytest.fixture
+def args(here):
+    return {
+        "src_dir": "%s" % (here / "source"),
+        "dryrun": False,
+        "capture_output": True,
+    }
+
+
+def test_populate(here, args, populate, watcher):
+
+    src_dir, project_dir = populate
+    shutil.copy(here / "bundle.yml", src_dir / "bundle.yml")
+
+    with watcher:
+        CacheCreator(**args).create()
+
+    assert f"{project_dir}/populate" in watcher.output
+    assert f"ARTIFACTS_DIR={src_dir}/artifacts" in watcher.output
+    assert "POPULATE_OPTION_1=0" in watcher.output
+    assert "POPULATE_OPTION_2=A" in watcher.output
+
+
+@pytest.mark.parametrize(
+    "populate_option",
+    [{"non_zero_option_1": True}, {"option_2": "B"}, {"set_both_options": "B"}],
+)
+def test_populate_options(here, args, populate, watcher, populate_option):
+
+    src_dir, project_dir = populate
+    shutil.copy(here / "bundle.yml", src_dir / "bundle.yml")
+
+    args.update(populate_option)
+    with watcher:
+        CacheCreator(**args).create()
+
+    assert f"{project_dir}/populate" in watcher.output
+    assert f"ARTIFACTS_DIR={src_dir}/artifacts" in watcher.output
+
+    if any(opt in populate_option for opt in ["non_zero_option_1", "set_both_options"]):
+        assert "POPULATE_OPTION_1=1" in watcher.output
+    else:
+        assert "POPULATE_OPTION_1=0" in watcher.output
+
+    if any(opt in populate_option for opt in ["option_2", "set_both_options"]):
+        assert 'POPULATE_OPTION_2="B"' in watcher.output
+    else:
+        assert "POPULATE_OPTION_2=A" in watcher.output