WIP: migrate tue-get to python

.gitignore

@@ -2,3 +2,6 @@ env
 .rsettings
 bin
 user/
+
+# Python install files
+*.egg-info

installer/parse_install_yaml.py

@@ -1,6 +1,7 @@
 #! /usr/bin/env python3
 
-from typing import List, Mapping, Optional
+from typing import Any, List, Mapping, Optional
+from functools import partial
 from os import environ
 import sys
 import yaml
@@ -254,5 +255,143 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type:
     return {"system_packages": system_packages, "commands": commands}
 
 
+def installyaml_parser2(installer: Any, path: str, now: bool = False) -> Mapping:
+    with open(path) as f:
+        try:
+            install_items = yaml.load(f, yaml.CSafeLoader)
+        except AttributeError:
+            install_items = yaml.load(f, yaml.SafeLoader)
+        except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e:
+            raise ValueError(f"Invalid yaml syntax: {e}")
+
+    if not isinstance(install_items, list):
+        raise ValueError("Root of install.yaml file should be a YAML sequence")
+
+    system_packages = []
+
+    commands = []
+
+    def get_distro_item(item: Mapping, key: str, release_version: str, release_type: str) -> Optional[str]:
+        if key in item:
+            value = item[key]
+            if value is None:
+                raise ValueError(f"'{key}' is defined, but has no value")
+        elif len(item) < 3 or "default" not in item:
+            raise ValueError(f"At least one distro and 'default' should be specified or none in install.yaml")
+        else:
+            for version in [release_version, "default"]:
+                if version in item:
+                    value = item[version][key]
+                    break
+
+        return value
+
+    # Combine now calls
+    now_cache = {
+        "system-now": [],
+        "pip-now": [],
+        "pip3-now": [],
+        "ppa-now": [],
+        "snap-now": [],
+        "gem-now": [],
+    }
+
+    for install_item in install_items:
+        command = None
+
+        try:
+            install_type = install_item["type"]  # type: str
+
+            if install_type == "empty":
+                return {"system_packages": system_packages, "commands": commands}
+
+            elif install_type == "ros":
+                try:
+                    source = get_distro_item(install_item, "source", ros_release, "ROS")
+                except ValueError as e:
+                    raise ValueError(f"[{install_type}]: {e.args[0]}")
+
+                # Both release and default are allowed to be None
+                if source is None:
+                    continue
+
+                source_type = source["type"]
+                if source_type == "git":
+                    command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source)))
+                elif source_type == "system":
+                    system_packages.append(f"ros-{ros_release}-{source['name']}")
+                    command = partial(getattr(installer, "tue_install_ros", "system", catkin_git(source)))
+                else:
+                    raise ValueError(f"Unknown ROS install type: '{source_type}'")
+
+            # Non ros targets that are packaged to be built with catkin
+            elif install_type == "catkin":
+                source = install_item["source"]
+
+                if not source["type"] == "git":
+                    raise ValueError(f"Unknown catkin install type: '{source['type']}'")
+                command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source)))
+
+            elif install_type == "git":
+                command = partial(getattr(installer, "tue_install_git", "git", type_git(install_item)))
+
+            elif install_type in [
+                "target",
+                "system",
+                "pip",
+                "pip3",
+                "ppa",
+                "snap",
+                "gem",
+                "dpkg",
+                "target-now",
+                "system-now",
+                "pip-now",
+                "pip3-now",
+                "ppa-now",
+                "snap-now",
+                "gem-now",
+            ]:
+                install_type = install_type.replace("-", "_")
+                if now and "now" not in install_type:
+                    install_type += "_now"
+
+                try:
+                    pkg_name = get_distro_item(install_item, "name", ubuntu_release, "Ubuntu")
+                except ValueError as e:
+                    raise ValueError(f"[{install_type}]: {e.args[0]}")
+
+                # Both release and default are allowed to be None
+                if pkg_name is None:
+                    continue
+
+                if "system" in install_type:
+                    system_packages.append(pkg_name)
+
+                # Cache install types which accept multiple pkgs at once
+                if install_type in now_cache:
+                    now_cache[install_type].append(pkg_name)
+                    continue
+                command = partial(getattr(installer, f"tue_install_{install_type}", pkg_name))
+
+            else:
+                raise ValueError(f"Unknown install type: '{install_type}'")
+
+        except KeyError as e:
+            raise KeyError(f"Invalid install.yaml file: Key '{e}' could not be found.")
+
+        if not command:
+            raise ValueError("Invalid install.yaml file")
+
+        commands.append(command)
+
+    for install_type, pkg_list in now_cache.items():
+        if pkg_list:
+            command = partial(getattr(installer, f"tue_install_{install_type}", pkg_list))
+            commands.append(command)
+
+    return {"system_packages": system_packages, "commands": commands}
+
+
 if __name__ == "__main__":
     sys.exit(main())

installer/tue-get-dep.bash

@@ -8,7 +8,7 @@ hash xmlstarlet 2> /dev/null || sudo apt-get install --assume-yes -qq xmlstarlet
 
 function _show_dep
 {
-    [[ "$1" == "ros" ]] && return 0
+    [[ "$1" == "ros" ]] || [[ "$1" == "ros1" ]] || [[ "$1" == "ros2" ]] && return 0
 
     [[ "$ROS_ONLY" = "true" && "$1" != ros-* ]] && return 0
 

setup.py

@@ -0,0 +1,43 @@
+import os.path
+
+from setuptools import find_packages
+from setuptools import setup
+
+with open(os.path.join(os.path.dirname(__file__), "VERSION"), "r") as f:
+    version = f.readline().strip()
+
+print(find_packages(where="src"))
+
+setup(
+    name="tue_env",
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    package_data={"tue_get": ["src/tue_get/resources/*"]},
+    include_package_data=True,
+    version=version,
+    author="Matthijs van der Burgh",
+    author_email="[email protected]",
+    maintainer="Matthijs van der Burgh",
+    maintainer_email="[email protected]",
+    url="https://github.com/tue-robotics/tue-env",
+    keywords=["catkin"],
+    classifiers=[
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Programming Language :: Python",
+    ],
+    description="Plugin for catkin_tools to enable building workspace documentation.",
+    # entry_points={
+    #     "catkin_tools.commands.catkin.verbs": [
+    #         "document = catkin_tools_document:description",
+    #     ],
+    #     "catkin_tools.spaces": [
+    #         "docs = catkin_tools_document.spaces.docs:description",
+    #     ],
+    # },
+    python_version=">=3.8",
+    install_requires=[
+        "pyyaml",
+        "termcolor",
+    ],
+)

src/tue_get/__init__.py

@@ -0,0 +1,4 @@
+from . import catkin_package_parser
+from . import install_yaml_parser
+from . import installer_impl
+from . import util

src/tue_get/catkin_package_parser.py

@@ -0,0 +1,47 @@
+from typing import List, Mapping, Optional, Tuple
+
+from catkin_pkg.package import Dependency, Package, parse_package
+import os
+
+
+def catkin_package_parser(
+    path: str,
+    skip_normal_deps: bool = False,
+    test_deps: bool = False,
+    doc_deps: bool = False,
+    warnings: Optional[List[str]] = None,
+    context: Optional[Mapping] = None,
+) -> Tuple[Package, List[Dependency]]:
+    if context is None:
+        context = os.environ
+    dep_types = []
+    if not skip_normal_deps:
+        dep_types.extend(
+            [
+                "build_depends",
+                "buildtool_depends",
+                "build_export_depends",
+                "buildtool_export_depends",
+                "exec_depends",
+            ]
+        )
+
+    if test_deps:
+        dep_types.append("test_depends")
+
+    if doc_deps:
+        dep_types.append("doc_depends")
+
+    pkg = parse_package(path, warnings=warnings)
+    pkg.evaluate_conditions(context)
+
+    deps = []
+    for dep_type in dep_types:
+        dep_list = getattr(pkg, dep_type)
+        for dep in dep_list:  # type: Dependency
+            if dep.evaluated_condition is None:
+                raise RuntimeError("Dependency condition is None")
+            if dep.evaluated_condition:
+                deps.append(dep)
+
+    return pkg, deps