From ccce545c647375d19409694c9c8103f8f8255638 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 3 Jan 2023 23:38:32 +0100 Subject: [PATCH 01/47] WIP: migrate tue-get to python --- installer/parse_install_yaml.py | 141 ++++++++++++++++- installer/tue_install_impl.py | 273 ++++++++++++++++++++++++++++++++ 2 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 installer/tue_install_impl.py diff --git a/installer/parse_install_yaml.py b/installer/parse_install_yaml.py index 10e14dbb7..6c07b9c51 100755 --- a/installer/parse_install_yaml.py +++ b/installer/parse_install_yaml.py @@ -1,6 +1,7 @@ #! /usr/bin/env python3 -from typing import List, Mapping, Optional +from typing import Any, List, Mapping, Optional +from functools import partial from os import environ import sys import yaml @@ -254,5 +255,143 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type: return {"system_packages": system_packages, "commands": commands} +def installyaml_parser2(installer: Any, path: str, now: bool = False) -> Mapping: + with open(path) as f: + try: + install_items = yaml.load(f, yaml.CSafeLoader) + except AttributeError: + install_items = yaml.load(f, yaml.SafeLoader) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e: + raise ValueError(f"Invalid yaml syntax: {e}") + + if not isinstance(install_items, list): + raise ValueError("Root of install.yaml file should be a YAML sequence") + + system_packages = [] + + commands = [] + + def get_distro_item(item: Mapping, key: str, release_version: str, release_type: str) -> Optional[str]: + if key in item: + value = item[key] + if value is None: + raise ValueError(f"'{key}' is defined, but has no value") + elif len(item) < 3 or "default" not in item: + raise ValueError(f"At least one distro and 'default' should be specified or none in install.yaml") + else: + for version in [release_version, "default"]: + if version in item: + value = item[version][key] + break + + return value + + # Combine now calls + now_cache = { + "system-now": [], + "pip-now": [], + "pip3-now": [], + "ppa-now": [], + "snap-now": [], + "gem-now": [], + } + + for install_item in install_items: + command = None + + try: + install_type = install_item["type"] # type: str + + if install_type == "empty": + return {"system_packages": system_packages, "commands": commands} + + elif install_type == "ros": + try: + source = get_distro_item(install_item, "source", ros_release, "ROS") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if source is None: + continue + + source_type = source["type"] + if source_type == "git": + command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source))) + elif source_type == "system": + system_packages.append(f"ros-{ros_release}-{source['name']}") + command = partial(getattr(installer, "tue_install_ros", "system", catkin_git(source))) + else: + raise ValueError(f"Unknown ROS install type: '{source_type}'") + + # Non ros targets that are packaged to be built with catkin + elif install_type == "catkin": + source = install_item["source"] + + if not source["type"] == "git": + raise ValueError(f"Unknown catkin install type: '{source['type']}'") + command = partial(getattr(installer, "tue_install_ros", "git", catkin_git(source))) + + elif install_type == "git": + command = partial(getattr(installer, "tue_install_git", "git", type_git(install_item))) + + elif install_type in [ + "target", + "system", + "pip", + "pip3", + "ppa", + "snap", + "gem", + "dpkg", + "target-now", + "system-now", + "pip-now", + "pip3-now", + "ppa-now", + "snap-now", + "gem-now", + ]: + install_type = install_type.replace("-", "_") + if now and "now" not in install_type: + install_type += "_now" + + try: + pkg_name = get_distro_item(install_item, "name", ubuntu_release, "Ubuntu") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if pkg_name is None: + continue + + if "system" in install_type: + system_packages.append(pkg_name) + + # Cache install types which accept multiple pkgs at once + if install_type in now_cache: + now_cache[install_type].append(pkg_name) + continue + command = partial(getattr(installer, f"tue_install_{install_type}", pkg_name)) + + else: + raise ValueError(f"Unknown install type: '{install_type}'") + + except KeyError as e: + raise KeyError(f"Invalid install.yaml file: Key '{e}' could not be found.") + + if not command: + raise ValueError("Invalid install.yaml file") + + commands.append(command) + + for install_type, pkg_list in now_cache.items(): + if pkg_list: + command = partial(getattr(installer, f"tue_install_{install_type}", pkg_list)) + commands.append(command) + + return {"system_packages": system_packages, "commands": commands} + + if __name__ == "__main__": sys.exit(main()) diff --git a/installer/tue_install_impl.py b/installer/tue_install_impl.py new file mode 100644 index 000000000..40e906b3b --- /dev/null +++ b/installer/tue_install_impl.py @@ -0,0 +1,273 @@ +from contextlib import contextmanager +import datetime +import os +from pathlib import Path +import shlex +import subprocess +from termcolor import colored + +from .parse_install_yaml import installyaml_parser2 + +CI = None + + +def is_CI() -> bool: + global CI + if CI is None: + CI = os.environ.get("CI", False) + + return CI + + +def date_stamp() -> str: + return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + + +class InstallerImpl: + def __init__(self, debug: bool = False): + self._debug = debug + self._current_target = "main-loop" + self._current_target_dir = "" + stamp = date_stamp() + self._log_file = os.path.join("tmp", f"tue-get-details-{stamp}") + Path(self._log_file).touch(exist_ok=False) + + try: + self._tue_env_dir = os.environ["TUE_ENV_DIR"] + except KeyError: + print("'TUE_ENV_DIR' is not defined") + raise + + self._self._tue_apt_get_updated_file = os.path.join("tmp", "tue_get_apt_get_updated") + + self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") + self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") + self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") + + os.makedirs(self._dependencies_dir, exist_ok=True) + os.makedirs(self._dependencies_on_dir, exist_ok=True) + os.makedirs(self._installed_dir, exist_ok=True) + + try: + self._targets_dir = os.environ["TUE_ENV_TARGETS_DIR"] + except KeyError: + print("'TUE_ENV_TARGETS_DIR' is not defined") + raise + + if not os.path.isdir(self._targets_dir): + raise RuntimeError(f"TUE_INSTALL_TARGETS_DIR '{self._targets_dir}' does not exist as directory") + + general_state_dir = os.path.join("tmp", "tue-installer") + if not os.path.isdir(general_state_dir): + self.tue_install_debug(f"mkdir {general_state_dir}") + os.makedirs(general_state_dir, exist_ok=True) + self.tue_install_debug(f"chmod a+rwx {general_state_dir}") + os.chmod(general_state_dir, 777) + + self._state_dir = os.path.join(general_state_dir, stamp) + self.tue_install_debug(f"mkdir {self._state_dir}") + os.makedirs(self._state_dir) + + self._warn_logs = [] + self._info_logs = [] + + def __del__(self): + os.rmdir(self._state_dir) + + def _log_to_file(self, msg: str) -> None: + with open(self._log_file, "a") as f: + f.write(msg + "\n") + + def _write_sorted_deps(self, child: str) -> None: + self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) + self._write_sorted_dep_file(os.path.join(self._dependencies_on_dir, child), self._current_target) + + def _write_sorted_dep_file(self, file: str, target: str) -> None: + if not os.path.isfile(file): + Path(file).touch(exist_ok=False) + targets = [] + else: + with open(file, "r+") as f: + targets = f.readlines() + + print(f"{targets=}") + targets.append(target + "\n") + targets.sort() + with open(file, "w") as f: + f.writelines(targets) + + @contextmanager + def _set_target(self, target: str): + parent_target = self._current_target + parent_target_dir = self._current_target_dir + self._current_target = target + self._current_target_dir = os.path.join(self._targets_dir, target) + yield + self._current_target = parent_target + self._current_target_dir = parent_target_dir + + def tue_install_error(self, msg: str) -> None: + log_msg = f"Error while installing target '{self._current_target}':\n\n\t{msg}\n\nLogfile: {self._log_file}" + print(colored(log_msg, color="red")) + self._log_to_file(log_msg) + + def tue_install_warning(self, msg: str) -> None: + log_msg = f"[{self._current_target}] WARNING: {msg}" + colored_log = colored(log_msg, color="yellow", attrs=["bold"]) + self._warn_logs.append(colored_log) + self._log_to_file(log_msg) + + def tue_install_info(self, msg: str) -> None: + log_msg = f"[{self._current_target}] INFO: {msg}" + colored_log = colored(log_msg, color="cyan") + self._info_logs.append(colored_log) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_debug(self, msg: str) -> None: + log_msg = f"[{self._current_target}] DEBUG: {msg}" + colored_log = colored(log_msg, color="blue") + if self._debug: + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_echo(self, msg: str) -> None: + log_msg = f"[{self._current_target}]: {msg}" + colored_log = colored(log_msg, attrs=["bold"]) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_tee(self, msg: str) -> None: + print(msg) + self._log_to_file(msg) + + def tue_install_pipe(self): + # ToDo: This should be implemented for install.bash scripts, not needed anymore for system calls in this script + pass + + def tue_install_target_now(self, target: str) -> int: + self.tue_install_debug(f"tue-install-target-now {target}") + + self.tue_install_debug(f"calling: tue-install-target {target} true") + return self.tue_install_target(target, True) + + def tue_install_target(self, target: str, now: bool = False) -> int: + self.tue_install_debug(f"tue-install-target {target} {now}") + + self.tue_install_debug("Installing target: {target}") + + # Check if valid target received as input + if not os.path.isdir(os.path.join(self._targets_dir, target)): + self.tue_install_debug(f"Target '{target}' does not exist.") + return False + + # If the target has a parent target, add target as a dependency to the parent target + if self._current_target and self._current_target != "main-loop" and self._current_target != target: + self._write_sorted_deps(target) + + with self._set_target(target): + + state_file = os.path.join(self._state_dir, "target") + state_file_now = f"{state_file}-now" + + # Determine if this target needs to be executed + execution_needed = True + + if is_CI() and not os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore")): + self.tue_install_debug( + f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file_now): + self.tue_install_debug( + f"File {state_file_now} does exist, so installation has already been executed with 'now' option. No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file): + if now: + self.tue_install_debug( + f"File {state_file_now} doesn't exist, but file {state_file} does. So installation has been executed yet, but not with the 'now' option. Going to execute it with 'now' option." + ) + else: + self.tue_install_debug( + f"File {state_file_now} does exist. 'now' is not enabled, so no execution needed." + ) + execution_needed = False + else: + if now: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute with 'now' option." + ) + else: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute without 'now' option." + ) + + if execution_needed: + self.tue_install_debug("Starting installation") + install_file = os.path.join(self._current_target_dir, "install") + + # Empty the target's dependency file + dep_file = os.path.join(self._dependencies_dir, target) + self.tue_install_debug(f"Emptying {dep_file}") + if os.path.isfile(dep_file): + with open(dep_file, "a") as f: + f.truncate(0) + + target_processed = False + + install_yaml_file = install_file + ".yaml" + if os.path.isfile(install_yaml_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_yaml")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_yaml file, so skipping install.yaml" + ) + target_processed = True + else: + self.tue_install_debug(f"Parsing {install_yaml_file}") + + cmds = installyaml_parser2(self, install_yaml_file, now)["commands"] + if not cmds: + self.tue_install_error(f"Invalid install.yaml: {cmds}") + + for cmd in cmds: + self.tue_install_debug(str(cmd)) + cmd() or self.tue_install_error( + f"Error while running: {cmd}" + ) # ToDo: Fix exception/return value handling + + target_processed = True + + install_bash_file = install_file + ".bash" + if os.path.isfile(install_bash_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_bash")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_bash file, so skipping install.bash" + ) + else: + self.tue_install_debug(f"Sourcing {install_bash_file}") + # ToDo: source install.bash + + target_processed = True + + if not target_processed: + self.tue_install_warning("Target does not contain a valid install.yaml/bash file") + + if now: + state_file_to_touch = state_file_now + else: + state_file_to_touch = state_file + + Path(state_file_to_touch).touch() + + self.tue_install_debug(f"Finished installing {target}") + + +if __name__ == "__main__": + cmd = shlex.split("git -C /home/amigo/.tue status") + p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return_value = p1.wait() + print(f"{return_value=}") + print(f"{p1.stdout.readlines()=}") + print(f"{p1.stderr.readlines()=}") + print(date_stamp()) From 956e4b8ab9eed9e01deaa8f39071133b36a7c58a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 3 Jan 2023 23:38:32 +0100 Subject: [PATCH 02/47] WIP: migrate tue-get to python --- src/tue_get/__init__.py | 3 + src/tue_get/install_yaml_parser.py | 219 +++++++++++++++++++ src/tue_get/installer_impl.py | 332 +++++++++++++++++++++++++++++ src/tue_get/util.py | 45 ++++ 4 files changed, 599 insertions(+) create mode 100644 src/tue_get/__init__.py create mode 100644 src/tue_get/install_yaml_parser.py create mode 100644 src/tue_get/installer_impl.py create mode 100644 src/tue_get/util.py diff --git a/src/tue_get/__init__.py b/src/tue_get/__init__.py new file mode 100644 index 000000000..ccd84a56a --- /dev/null +++ b/src/tue_get/__init__.py @@ -0,0 +1,3 @@ +from . import install_yaml_parser +from . import installer_impl +from . import util diff --git a/src/tue_get/install_yaml_parser.py b/src/tue_get/install_yaml_parser.py new file mode 100644 index 000000000..4c18e816c --- /dev/null +++ b/src/tue_get/install_yaml_parser.py @@ -0,0 +1,219 @@ +from typing import Any, List, Mapping, Optional + +from os import environ +from functools import partial +import yaml + +from lsb_release import get_distro_information + +ros_release = environ["TUE_ROS_DISTRO"] +ubuntu_release = get_distro_information()["CODENAME"] + + +def type_git(install_item: Mapping, allowed_keys: Optional[List[str]] = None) -> Mapping: + """ + Function to check the parsed yaml for install type git and generate the command string. + + The structure of a git install type is: + - type: git + url: + path: [LOCAL_CLONE_PATH] + version: [BRANCH_COMMIT_TAG] + + :param install_item: Extracted yaml component corresponding to install type git + :param allowed_keys: Additional keys to allow apart from the keys defined in install type git + :return: Mapping string containing repository url and optional keyword arguments target-dir and version + """ + + assert install_item["type"] == "git", f"Invalid install type '{install_item['type']}' for type 'git'" + + keys = {"type", "url", "path", "version"} + if allowed_keys: + keys |= set(allowed_keys) + + invalid_keys = [k for k in install_item.keys() if k not in keys] + + if invalid_keys: + raise KeyError(f"The following keys are invalid for install type 'git': {invalid_keys}") + + url = install_item.get("url") + target_dir = install_item.get("path") + version = install_item.get("version") + + if not url: + raise KeyError("'url' is a mandatory key for install type 'git'") + + return {"url": url, "target_dir": target_dir, "version": version} + + +def catkin_git(source: Mapping) -> Mapping: + """ + Function to generate installation command for catkin git targets from the extracted yaml + + The structure of a catkin git install type is: + - type: catkin/ros + source: + type: git + url: + path: [LOCAL_CLONE_PATH] + version: [BRANCH_COMMIT_TAG] + sub-dir: [PACKAGE_SUB_DIRECTORY] + + :param source: Extracted yaml component for the key 'source' corresponding to install type catkin/ros + :return: Mapping containing the keyword arguments to the primary catkin target installation command + """ + + args = type_git(source, allowed_keys=["sub-dir"]) + args["source_type"] = "git" + + args["sub_dir"] = source.get("sub-dir") + + return args + + +def installyaml_parser(installer: Any, path: str, now: bool = False) -> Mapping: + with open(path) as f: + try: + install_items = yaml.load(f, yaml.CSafeLoader) + except AttributeError: + install_items = yaml.load(f, yaml.SafeLoader) + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as e: + raise ValueError(f"Invalid yaml syntax: {e}") + + if not isinstance(install_items, list): + raise ValueError("Root of install.yaml file should be a YAML sequence") + + system_packages = [] + commands = [] + + # Combine now calls + now_cache = { + "system_now": [], + "pip_now": [], + "pip3_now": [], + "ppa_now": [], + "snap_now": [], + "gem_now": [], + } + + def get_distro_item(item: Mapping, key: str, release_version: str, release_type: str) -> Optional[str]: + if key in item: + value = item[key] + if value is None: + raise ValueError(f"'{key}' is defined, but has no value") + elif len(item) < 3 or "default" not in item: + raise ValueError(f"At least one distro and 'default' should be specified or none in install.yaml") + else: + for version in [release_version, "default"]: + if version in item: + value = item[version][key] + break + + return value + + for install_item in install_items: + command = None + + try: + install_type = install_item["type"] # type: str + install_type = install_type.replace("-", "_") # Need after switching to python impl of tue-get + + if install_type == "empty": + return {"system_packages": system_packages, "commands": commands} + + elif install_type == "ros": + try: + source = get_distro_item(install_item, "source", ros_release, "ROS") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if source is None: + continue + + source_type = source["type"] + if source_type == "git": + command = partial(getattr(installer, "tue_install_ros"), source_type="git", **catkin_git(source)) + elif source_type == "system": + system_packages.append(f"ros-{ros_release}-{source['name']}") + command = partial(getattr(installer, "tue_install_ros"), source_type="system", **catkin_git(source)) + else: + raise ValueError(f"Unknown ROS install type: '{source_type}'") + + # Non ros targets that are packaged to be built with catkin + elif install_type == "catkin": + source = install_item["source"] + + if not source["type"] == "git": + raise ValueError(f"Unknown catkin install type: '{source['type']}'") + command = partial(getattr(installer, "tue_install_ros"), source_type="git", **catkin_git(source)) + + elif install_type == "git": + command = partial(getattr(installer, "tue_install_git"), source_type="git", **type_git(install_item)) + + elif install_type in [ + "target", + "system", + "pip", + "pip3", + "ppa", + "snap", + "gem", + "dpkg", + "target_now", + "system_now", + "pip_now", + "pip3_now", + "ppa_now", + "snap_now", + "gem_now", + ]: + if now and "now" not in install_type: + install_type += "_now" + + try: + pkg_name = get_distro_item(install_item, "name", ubuntu_release, "Ubuntu") + except ValueError as e: + raise ValueError(f"[{install_type}]: {e.args[0]}") + + # Both release and default are allowed to be None + if pkg_name is None: + continue + + if "system" in install_type: + system_packages.append(pkg_name) + + # Cache install types which accept multiple pkgs at once + if install_type in now_cache: + now_cache[install_type].append(pkg_name) + continue + command = partial(getattr(installer, f"tue_install_{install_type}"), pkg_name) + + else: + raise ValueError(f"Unknown install type: '{install_type}'") + + except KeyError as e: + raise KeyError(f"Invalid install.yaml file: Key '{e}' could not be found.") + + if not command: + raise ValueError("Invalid install.yaml file") + + commands.append(command) + + for install_type, pkg_list in now_cache.items(): + if pkg_list: + command = partial(getattr(installer, f"tue_install_{install_type}"), pkg_list) + commands.append(command) + + return {"system_packages": system_packages, "commands": commands} + + +if __name__ == "__main__": + import sys + from tue_get.installer_impl import InstallerImpl + if len(sys.argv) < 2: + print("Provide yaml file to parse") + exit(1) + + impl = InstallerImpl(True) + print(installyaml_parser(impl, sys.argv[1], False)) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py new file mode 100644 index 000000000..7379439fd --- /dev/null +++ b/src/tue_get/installer_impl.py @@ -0,0 +1,332 @@ +from typing import List, Optional + +from contextlib import contextmanager +import datetime +import os +from pathlib import Path +import shlex +import subprocess +from termcolor import colored + +from .install_yaml_parser import installyaml_parser + +CI = None + + +def is_CI() -> bool: + global CI + if CI is None: + CI = os.environ.get("CI", False) + + return CI + + +def date_stamp() -> str: + return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + + +class InstallerImpl: + def __init__(self, debug: bool = False): + self._debug = debug + self._current_target = "main-loop" + self._current_target_dir = "" + stamp = date_stamp() + self._log_file = os.path.join(os.sep, "tmp", f"tue-get-details-{stamp}") + Path(self._log_file).touch(exist_ok=False) + + try: + self._tue_env_dir = os.environ["TUE_ENV_DIR"] + except KeyError: + print("'TUE_ENV_DIR' is not defined") + raise + + self._tue_apt_get_updated_file = os.path.join(os.sep, "tmp", "tue_get_apt_get_updated") + + self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") + self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") + self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") + + os.makedirs(self._dependencies_dir, exist_ok=True) + os.makedirs(self._dependencies_on_dir, exist_ok=True) + os.makedirs(self._installed_dir, exist_ok=True) + + try: + self._targets_dir = os.environ["TUE_ENV_TARGETS_DIR"] + except KeyError: + print("'TUE_ENV_TARGETS_DIR' is not defined") + raise + + if not os.path.isdir(self._targets_dir): + raise RuntimeError(f"TUE_INSTALL_TARGETS_DIR '{self._targets_dir}' does not exist as directory") + + general_state_dir = os.path.join(os.sep, "tmp", "tue-installer") + if not os.path.isdir(general_state_dir): + self.tue_install_debug(f"mkdir {general_state_dir}") + os.makedirs(general_state_dir, mode=0o700, exist_ok=True) + # self.tue_install_debug(f"chmod a+rwx {general_state_dir}") + # os.chmod(general_state_dir, 0o700) + + self._state_dir = os.path.join(general_state_dir, stamp) + self.tue_install_debug(f"mkdir {self._state_dir}") + os.makedirs(self._state_dir, mode=0o700) + + self._warn_logs = [] + self._info_logs = [] + + def __del__(self): + os.rmdir(self._state_dir) + + def _log_to_file(self, msg: str) -> None: + with open(self._log_file, "a") as f: + f.write(msg + "\n") + + def _write_sorted_deps(self, child: str) -> None: + self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) + self._write_sorted_dep_file(os.path.join(self._dependencies_on_dir, child), self._current_target) + + def _write_sorted_dep_file(self, file: str, target: str) -> None: + if not os.path.isfile(file): + Path(file).touch(exist_ok=False) + targets = [] + else: + with open(file, "r+") as f: + targets = f.readlines() + + print(f"{targets=}") + targets.append(target + "\n") + targets.sort() + with open(file, "w") as f: + f.writelines(targets) + + @contextmanager + def _set_target(self, target: str): + parent_target = self._current_target + parent_target_dir = self._current_target_dir + self._current_target = target + self._current_target_dir = os.path.join(self._targets_dir, target) + yield + self._current_target = parent_target + self._current_target_dir = parent_target_dir + + def tue_install_error(self, msg: str) -> None: + log_msg = f"Error while installing target '{self._current_target}':\n\n\t{msg}\n\nLogfile: {self._log_file}" + print(colored(log_msg, color="red")) + self._log_to_file(log_msg) + + def tue_install_warning(self, msg: str) -> None: + log_msg = f"[{self._current_target}] WARNING: {msg}" + colored_log = colored(log_msg, color="yellow", attrs=["bold"]) + self._warn_logs.append(colored_log) + self._log_to_file(log_msg) + + def tue_install_info(self, msg: str) -> None: + log_msg = f"[{self._current_target}] INFO: {msg}" + colored_log = colored(log_msg, color="cyan") + self._info_logs.append(colored_log) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_debug(self, msg: str) -> None: + log_msg = f"[{self._current_target}] DEBUG: {msg}" + colored_log = colored(log_msg, color="blue") + if self._debug: + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_echo(self, msg: str) -> None: + log_msg = f"[{self._current_target}]: {msg}" + colored_log = colored(log_msg, attrs=["bold"]) + print(colored_log) + self._log_to_file(log_msg) + + def tue_install_tee(self, msg: str) -> None: + print(msg) + self._log_to_file(msg) + + def tue_install_pipe(self): + # ToDo: This should be implemented for install.bash scripts, not needed anymore for system calls in this script + pass + + def tue_install_target_now(self, target: str) -> int: + self.tue_install_debug(f"tue-install-target-now {target}") + + self.tue_install_debug(f"calling: tue-install-target {target} true") + return self.tue_install_target(target, True) + + def tue_install_target(self, target: str, now: bool = False) -> int: + self.tue_install_debug(f"tue-install-target {target} {now}") + + self.tue_install_debug("Installing target: {target}") + + # Check if valid target received as input + if not os.path.isdir(os.path.join(self._targets_dir, target)): + self.tue_install_debug(f"Target '{target}' does not exist.") + return False + + # If the target has a parent target, add target as a dependency to the parent target + if self._current_target and self._current_target != "main-loop" and self._current_target != target: + self._write_sorted_deps(target) + + with self._set_target(target): + + state_file = os.path.join(self._state_dir, "target") + state_file_now = f"{state_file}-now" + + # Determine if this target needs to be executed + execution_needed = True + + if is_CI() and not os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore")): + self.tue_install_debug( + f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file_now): + self.tue_install_debug( + f"File {state_file_now} does exist, so installation has already been executed with 'now' option. No execution is needed." + ) + execution_needed = False + elif os.path.isfile(state_file): + if now: + self.tue_install_debug( + f"File {state_file_now} doesn't exist, but file {state_file} does. So installation has been executed yet, but not with the 'now' option. Going to execute it with 'now' option." + ) + else: + self.tue_install_debug( + f"File {state_file_now} does exist. 'now' is not enabled, so no execution needed." + ) + execution_needed = False + else: + if now: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute with 'now' option." + ) + else: + self.tue_install_debug( + f"Files {state_file_now} and {state_file} don't exist. Going to execute without 'now' option." + ) + + if execution_needed: + self.tue_install_debug("Starting installation") + install_file = os.path.join(self._current_target_dir, "install") + + # Empty the target's dependency file + dep_file = os.path.join(self._dependencies_dir, target) + self.tue_install_debug(f"Emptying {dep_file}") + if os.path.isfile(dep_file): + with open(dep_file, "a") as f: + f.truncate(0) + + target_processed = False + + install_yaml_file = install_file + ".yaml" + if os.path.isfile(install_yaml_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_yaml")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_yaml file, so skipping install.yaml" + ) + target_processed = True + else: + self.tue_install_debug(f"Parsing {install_yaml_file}") + + cmds = installyaml_parser(self, install_yaml_file, now)["commands"] + if not cmds: + self.tue_install_error(f"Invalid install.yaml: {cmds}") + + for cmd in cmds: + self.tue_install_debug(str(cmd)) + cmd() or self.tue_install_error( + f"Error while running: {cmd}" + ) # ToDo: Fix exception/return value handling + + target_processed = True + + install_bash_file = install_file + ".bash" + if os.path.isfile(install_bash_file): + if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_bash")): + self.tue_install_debug( + "Running in CI mode and found .ci_ignore_bash file, so skipping install.bash" + ) + else: + self.tue_install_debug(f"Sourcing {install_bash_file}") + # ToDo: source install.bash + + target_processed = True + + if not target_processed: + self.tue_install_warning("Target does not contain a valid install.yaml/bash file") + + if now: + state_file_to_touch = state_file_now + else: + state_file_to_touch = state_file + + Path(state_file_to_touch).touch() + + self.tue_install_debug(f"Finished installing {target}") + + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None): + pass + + def tue_install_apply_patch(self, patch_file: str, target_dir: str): + pass + + def tue_install_cp(self, source_file: str, target_file: str): + pass + + def tue_install_add_text(self, source_file: str, target_file: str): + pass + + def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None): + pass + + def tue_install_system(self, pkgs: List[str]): + pass + + def tue_install_system_now(self, pkgs: List[str]): + pass + + def tue_install_apt_get_update(self): + pass + + def tue_install_ppa(self, ppa: List[str]): + pass + + def tue_install_ppa_now(self, ppa: List[str]): + pass + + def tue_install_pip(self, pkgs: List[str]): + pass + + def tue_install_pip_now(self, pkgs: List[str]): + pass + + def tue_install_snap(self, pkgs: List[str]): + pass + + def tue_install_snap_now(self, pkgs: List[str]): + pass + + def tue_install_gem(self, pkgs: List[str]): + pass + + def tue_install_gem_now(self, pkgs: List[str]): + pass + + def tue_install_dpkg(self, pkg_file: str): + pass + + def tue_install_dpkg_now(self, pkg_file: str): + pass + + def tue_install_ros(self, source_type: str, **kwargs): + pass + + +if __name__ == "__main__": + cmd = shlex.split("git -C /home/amigo/.tue status") + p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return_value = p1.wait() + print(f"{return_value=}") + print(f"{p1.stdout.readlines()=}") + print(f"{p1.stderr.readlines()=}") + print(date_stamp()) diff --git a/src/tue_get/util.py b/src/tue_get/util.py new file mode 100644 index 000000000..25dea7671 --- /dev/null +++ b/src/tue_get/util.py @@ -0,0 +1,45 @@ +from typing import Callable, IO, Optional +import subprocess +from threading import Thread + + +class BackgroundPopen(subprocess.Popen): + """ + Inspired by https://stackoverflow.com/a/42352331 + """ + + @staticmethod + def _proxy_lines(pipe: IO, handler: Callable): + with pipe: + for line in pipe: + handler(line) + + def __init__(self, out_handler: Optional[Callable] = None, err_handler: Optional[Callable] = None, *args, **kwargs): + if out_handler is not None: + kwargs["stdout"] = subprocess.PIPE + if err_handler is not None: + kwargs["stderr"] = subprocess.PIPE + super().__init__(*args, **kwargs) + if out_handler is not None: + Thread(target=self._proxy_lines, args=[self.stdout, out_handler]).start() + if err_handler is not None: + Thread(target=self._proxy_lines, args=[self.stderr, err_handler]).start() + + +if __name__ == "__main__": + import shlex + from termcolor import cprint + + cmd = """ + bash -c ' + for i in {1..100} + do + echo -e "${i}" + done' + """ + + def cyan_handler(line): + cprint(line, color="cyan", end="") + + bla = BackgroundPopen(out_handler=cyan_handler, args=shlex.split(cmd), text=True) + bla.wait() From e7e53aedada6db5736fdb9b39a038161fc910780 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 4 Jan 2023 20:35:51 +0100 Subject: [PATCH 03/47] Add *.egg-info to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 16c86f18b..553b141f4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ env .rsettings bin user/ + +# Python install files +*.egg-info From 8fcab63dd79a3ded6d06f70c9c0b507e938a0b58 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 4 Jan 2023 20:36:10 +0100 Subject: [PATCH 04/47] Make this a python module --- setup.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..2923f1c22 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +import os.path + +from setuptools import find_packages +from setuptools import setup + +with open(os.path.join(os.path.dirname(__file__), "VERSION"), "r") as f: + version = f.readline().strip() + +print(find_packages(where="src")) + +setup( + name="tue_env", + packages=find_packages(where="src"), + package_dir={"": "src"}, + version=version, + author="Matthijs van der Burgh", + author_email="MatthijsBurgh@outlook.com", + maintainer="Matthijs van der Burgh", + maintainer_email="MatthijsBurgh@outlook.com", + url="https://github.com/tue-robotics/tue-env", + keywords=["catkin"], + classifiers=[ + "Environment :: Console", + "Intended Audience :: Developers", + "Programming Language :: Python", + ], + description="Plugin for catkin_tools to enable building workspace documentation.", + # entry_points={ + # "catkin_tools.commands.catkin.verbs": [ + # "document = catkin_tools_document:description", + # ], + # "catkin_tools.spaces": [ + # "docs = catkin_tools_document.spaces.docs:description", + # ], + # }, + python_version=">=3.8", + install_requires=[ + "termcolor", + "pyyaml", + ], +) From a22c95fd37425db6021336614054e8a45db55a14 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Wed, 4 Jan 2023 20:36:30 +0100 Subject: [PATCH 05/47] Add first version of pytests --- test/test_tue_get/__init__.py | 0 test/test_tue_get/test_install_yaml_parser.py | 118 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 test/test_tue_get/__init__.py create mode 100644 test/test_tue_get/test_install_yaml_parser.py diff --git a/test/test_tue_get/__init__.py b/test/test_tue_get/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/test_tue_get/test_install_yaml_parser.py b/test/test_tue_get/test_install_yaml_parser.py new file mode 100644 index 000000000..3826f23d4 --- /dev/null +++ b/test/test_tue_get/test_install_yaml_parser.py @@ -0,0 +1,118 @@ +import os +from typing import List, Optional + +import tempfile +import yaml + +from tue_get import install_yaml_parser + + +class MockInstaller: + def __init__(self): + pass + + def tue_install_target_now(self, target: str): + pass + + def tue_install_target(self, target: str, now: bool = False): + pass + + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None): + pass + + def tue_install_apply_patch(self, patch_file: str, target_dir: str): + pass + + def tue_install_cp(self, source_file: str, target_file: str): + pass + + def tue_install_add_text(self, source_file: str, target_file: str): + pass + + def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None): + pass + + def tue_install_system(self, pkgs: List[str]): + pass + + def tue_install_system_now(self, pkgs: List[str]): + pass + + def tue_install_apt_get_update(self): + pass + + def tue_install_ppa(self, ppa: List[str]): + pass + + def tue_install_ppa_now(self, ppa: List[str]): + pass + + def tue_install_pip(self, pkgs: List[str]): + pass + + def tue_install_pip_now(self, pkgs: List[str]): + pass + + def tue_install_snap(self, pkgs: List[str]): + pass + + def tue_install_snap_now(self, pkgs: List[str]): + pass + + def tue_install_gem(self, pkgs: List[str]): + pass + + def tue_install_gem_now(self, pkgs: List[str]): + pass + + def tue_install_dpkg(self, pkg_file: str): + pass + + def tue_install_dpkg_now(self, pkg_file: str): + pass + + def tue_install_ros(self, source_type: str, **kwargs): + pass + + +def parse_target(yaml_seq: List, now: bool = False): + install_file = tempfile.NamedTemporaryFile(mode="w", delete=False) + try: + dumper = yaml.CSafeDumper + except AttributeError: + dumper = yaml.SafeDumper + + yaml.dump(yaml_seq, install_file, dumper) + + install_file.close() + + installer = MockInstaller() + try: + cmds = install_yaml_parser.installyaml_parser(installer, install_file.name, now)["commands"] + except Exception: + raise + finally: + os.unlink(install_file.name) + + return cmds + + +def test_empty_target(): + target = [{"type": "empty"}] + cmds = parse_target(target, False) + assert not cmds + cmds = parse_target(target, True) + assert not cmds + +def test_system_target(): + target = [{"type": "system", "name": "pkg_name"}] + cmds = parse_target(target, False) + assert cmds + assert len(cmds) == 1 + for cmd in cmds: + cmd() + # assert isinstance(cmds[0].func, MockInstaller.tue_install_system) + cmds = parse_target(target, True) + assert cmds + assert len(cmds) == 1 + # assert isinstance(cmds[0].func, MockInstaller.tue_install_system_now) From 94ca54aa4a0f4034c388cc9e85b367dfc7c259fc Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 01:04:05 +0100 Subject: [PATCH 06/47] Most bash functions should be connected to python --- setup.py | 2 + src/tue_get/installer_impl.py | 405 +++++++++++++++++++--- src/tue_get/resources/installer_impl.bash | 217 ++++++++++++ src/tue_get/util.py | 9 +- 4 files changed, 580 insertions(+), 53 deletions(-) create mode 100644 src/tue_get/resources/installer_impl.bash diff --git a/setup.py b/setup.py index 2923f1c22..3683e1a6f 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,8 @@ name="tue_env", packages=find_packages(where="src"), package_dir={"": "src"}, + package_data={"tue_get": ["src/tue_get/resources/*"]}, + include_package_data=True, version=version, author="Matthijs van der Burgh", author_email="MatthijsBurgh@outlook.com", diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 7379439fd..8e3babc66 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1,14 +1,19 @@ -from typing import List, Optional +import glob +import subprocess +from typing import List, Optional, Union from contextlib import contextmanager import datetime +import filecmp +import getpass import os from pathlib import Path import shlex -import subprocess +import shutil from termcolor import colored -from .install_yaml_parser import installyaml_parser +from tue_get.install_yaml_parser import installyaml_parser +from tue_get.util import BackgroundPopen CI = None @@ -25,7 +30,15 @@ def date_stamp() -> str: return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") +def _which_split_cmd(cmd: str) -> List[str]: + cmds = shlex.split(cmd) + cmds[0] = shutil.which(cmds[0]) + return cmds + + class InstallerImpl: + _apt_get_update_file = os.path.join(os.pathsep, "tmp", "tue_get_apt_get_updated") + def __init__(self, debug: bool = False): self._debug = debug self._current_target = "main-loop" @@ -40,8 +53,6 @@ def __init__(self, debug: bool = False): print("'TUE_ENV_DIR' is not defined") raise - self._tue_apt_get_updated_file = os.path.join(os.sep, "tmp", "tue_get_apt_get_updated") - self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") @@ -73,12 +84,21 @@ def __init__(self, debug: bool = False): self._warn_logs = [] self._info_logs = [] + # ToDo: we might want to use sets here + self._systems = [] + self._ppas = [] + self._pips = [] + self._snaps = [] + self._gems = [] + + self._sudo_password = None + def __del__(self): - os.rmdir(self._state_dir) + shutil.rmtree(self._state_dir) - def _log_to_file(self, msg: str) -> None: + def _log_to_file(self, msg: str, end="\n") -> None: with open(self._log_file, "a") as f: - f.write(msg + "\n") + f.write(msg + end) def _write_sorted_deps(self, child: str) -> None: self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) @@ -108,10 +128,125 @@ def _set_target(self, target: str): self._current_target = parent_target self._current_target_dir = parent_target_dir + def _out_handler(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + def _write_stdin(msg) -> None: + sub.stdin.write(f"{msg}\n") + sub.stdin.flush() + + line = line.strip() + # ToDo: use regex to get attributes of installer + if line.startswith("[sudo] password for"): + if self._sudo_password is None: + self._sudo_password = getpass.getpass(line) + _write_stdin(self._sudo_password) + elif line.startswith("tue-install-error: "): + self.tue_install_error(line[19:]) + _write_stdin(1) + elif line.startswith("tue-install-warning: "): + self.tue_install_warning(line[21:]) + _write_stdin(0) + elif line.startswith("tue-install-info: "): + self.tue_install_info(line[18:]) + _write_stdin(0) + elif line.startswith("tue-install-debug: "): + self.tue_install_debug(line[19:]) + _write_stdin(0) + elif line.startswith("tue-install-echo: "): + self.tue_install_echo(line[18:]) + _write_stdin(0) + elif line.startswith("tue-install-tee: "): + self.tue_install_tee(line[17:]) + _write_stdin(0) + elif line.startswith("tue-install-pipe: "): + output = line[18:].split("^^^") + self.tue_install_pipe(*output) + _write_stdin(0) + elif line.startswith("tue-install-target-now: "): + self.tue_install_target_now(line[24:]) + _write_stdin(0) + elif line.startswith("tue-install-target: "): + success = self.tue_install_target(line[20:]) + _write_stdin(not success) + elif line.startswith("tue-install-git: "): + success = self.tue_install_git(*line[17:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-apply-patch: "): + success = self.tue_install_apply_patch(*line[25:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-cp: "): + success = self.tue_install_cp(*line[15:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-add-text: "): + success = self.tue_install_add_text(*line[21:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-get-releases: "): + success = self.tue_install_get_releases(*line[26:].split()) + _write_stdin(not success) + elif line.startswith("tue-install-system: "): + pkgs = line[20:].split() + self.tue_install_system(pkgs) + _write_stdin(0) + elif line.startswith("tue-install-system-now: "): + pkgs = line[24:].split() + success = self.tue_install_system_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-apt-get-update"): + self.tue_install_apt_get_update() + _write_stdin(0) + elif line.startswith("tue-install-ppa: "): + ppas = line[17:].split() + self.tue_install_ppa(ppas) + _write_stdin(0) + elif line.startswith("tue-install-ppa-now: "): + ppas = line[21:].split() + success = self.tue_install_ppa_now(ppas) + _write_stdin(not success) + elif line.startswith("tue-install-pip: "): + pkgs = line[17:].split() + self.tue_install_pip(pkgs) + _write_stdin(0) + elif line.startswith("tue-install-pip-now: "): + pkgs = line[20:].split() + success = self.tue_install_pip_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-snap: "): + pkgs = line[18:].split() + self.tue_install_snap(pkgs) + _write_stdin(0) + elif line.startswith("tue-install-snap-now: "): + pkgs = line[22:].split() + success = self.tue_install_snap_now(pkgs) + _write_stdin(not success) + elif line.startswith("tue-install-gem: "): + pkgs = line[17:].split(" ") + self.tue_install_gem(pkgs) + _write_stdin(0) + elif line.startswith("tue-install-gem-now: "): + pkgs = line[21:].split(" ") + success = self.tue_install_gem_now(pkgs) + _write_stdin(not success) + else: + self.tue_install_tee(line) + + def _out_handler_sudo_password(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + line = line.strip() + if line.startswith("[sudo] password for"): + if self._sudo_password is None: + self._sudo_password = getpass.getpass(line) + sub.stdin.write(f"{self._sudo_password}\n") + sub.stdin.flush() + else: + self.tue_install_tee(line) + def tue_install_error(self, msg: str) -> None: - log_msg = f"Error while installing target '{self._current_target}':\n\n\t{msg}\n\nLogfile: {self._log_file}" + # Make sure the entire msg is indented, not just the first line + lines = msg.splitlines() + lines = [f" {line}" for line in lines] + msg = "\n".join(lines) + log_msg = f"Error while installing target '{self._current_target}':\n\n{msg}\n\nLogfile: {self._log_file}" print(colored(log_msg, color="red")) self._log_to_file(log_msg) + # ToDo: We should stop after this def tue_install_warning(self, msg: str) -> None: log_msg = f"[{self._current_target}] WARNING: {msg}" @@ -139,22 +274,30 @@ def tue_install_echo(self, msg: str) -> None: print(colored_log) self._log_to_file(log_msg) - def tue_install_tee(self, msg: str) -> None: - print(msg) + def tue_install_tee(self, msg: str, color=None, on_color=None, attrs=None) -> None: + print(colored(msg, color=color, on_color=on_color, attrs=attrs)) self._log_to_file(msg) - def tue_install_pipe(self): - # ToDo: This should be implemented for install.bash scripts, not needed anymore for system calls in this script - pass + def tue_install_pipe(self, stdout: str, stderr: str) -> None: + """ + Only to be used for bash scripts + Prints stdout, stderr is printed in red + + :param stdout: + :param stderr: + :return: + """ + self.tue_install_tee(stdout) + self.tue_install_tee(stderr, color="red") - def tue_install_target_now(self, target: str) -> int: - self.tue_install_debug(f"tue-install-target-now {target}") + def tue_install_target_now(self, target: str) -> bool: + self.tue_install_debug(f"tue-install-target-now {target=}") - self.tue_install_debug(f"calling: tue-install-target {target} true") + self.tue_install_debug(f"calling: tue-install-target {target} True") return self.tue_install_target(target, True) - def tue_install_target(self, target: str, now: bool = False) -> int: - self.tue_install_debug(f"tue-install-target {target} {now}") + def tue_install_target(self, target: str, now: bool = False) -> bool: + self.tue_install_debug(f"tue-install-target {target=} {now=}") self.tue_install_debug("Installing target: {target}") @@ -235,7 +378,7 @@ def tue_install_target(self, target: str, now: bool = False) -> int: for cmd in cmds: self.tue_install_debug(str(cmd)) cmd() or self.tue_install_error( - f"Error while running: {cmd}" + f"Error while running:\n {cmd}" ) # ToDo: Fix exception/return value handling target_processed = True @@ -248,7 +391,25 @@ def tue_install_target(self, target: str, now: bool = False) -> int: ) else: self.tue_install_debug(f"Sourcing {install_bash_file}") - # ToDo: source install.bash + resource_file = os.path.join(os.path.dirname(__file__), "resources", "installer_impl.bash") + cmd = f"bash -c 'source {resource_file} && source {install_bash_file}'" + cmds = _which_split_cmd(cmd) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen( + args=cmds, + out_handler=self._out_handler, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error( + f"Error while running({sub.returncode}):\n {' '.join(cmds)}" f"\n\n{stderr}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False target_processed = True @@ -264,53 +425,206 @@ def tue_install_target(self, target: str, now: bool = False) -> int: self.tue_install_debug(f"Finished installing {target}") - def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None): + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None) -> bool: pass - def tue_install_apply_patch(self, patch_file: str, target_dir: str): - pass + def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: + self.tue_install_debug(f"tue-install-apply-patch {patch_file=} {target_dir=}") + + if not patch_file: + self.tue_install_error("Invalid tue-install-apply-patch call: needs patch file as argument") + + patch_file_path = os.path.join(self._current_target_dir, patch_file) + if not os.path.isfile(patch_file_path): + self.tue_install_error(f"Invalid tue-install-apply-patch call: patch file {patch_file_path} does not exist") + + if not target_dir: + self.tue_install_error("Invalid tue-install-apply-patch call: needs target directory as argument") + + if not os.path.isdir(target_dir): + self.tue_install_error( + f"Invalid tue-install-apply-patch call: target directory {target_dir} does not exist" + ) + + cmd = f"patch -s -N -r - -p0 -d {target_dir} < {patch_file_path}" + cmds = _which_split_cmd(cmd) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error(f"Error while running({sub.returncode}):\n{' '.join(cmds)}\n\n{stderr}") + # ToDo: This depends on behaviour of tue-install-error + return False - def tue_install_cp(self, source_file: str, target_file: str): - pass + return True + + def tue_install_cp(self, source: str, target: str) -> bool: + self.tue_install_debug(f"tue-install-cp {source=} {target=}") + + if not source: + self.tue_install_error("Invalid tue-install-cp call: needs source directory as argument") + + if not target: + self.tue_install_error("Invalid tue-install-cp call: needs target dir as argument") + + if os.path.isdir(target): + self.tue_install_debug(f"tue-install-cp: target {target} is a directory") + elif os.path.isfile(target): + self.tue_install_debug(f"tue-install-cp: target {target} is a file") + target = os.path.dirname(target) + else: + self.tue_install_error(f"tue-install-cp: target {target} does not exist") + + # Check if user is allowed to write on target destination + root_required = True + + if os.access(target, os.W_OK): + root_required = False + + source_path = os.path.join(self._current_target_dir, source) + source_files = glob.glob(source_path) + + for file in source_files: + if not os.path.isfile(file): + self.tue_install_error(f"tue-install-cp: source {file} is not a file") + + cp_target = os.path.join(target, os.path.basename(file)) - def tue_install_add_text(self, source_file: str, target_file: str): + if os.path.isfile(cp_target) and filecmp.cmp(file, cp_target): + self.tue_install_debug(f"tue-install-cp: {file} and {cp_target} are identical, skipping") + continue + + self.tue_install_debug(f"tue-install-cp: copying {file} to {cp_target}") + + if root_required: + sudo_cmd = "sudo -S " + self.tue_install_debug(f"Using elevated privileges (sudo) to copy ${file} to ${cp_target}") + else: + sudo_cmd = "" + + cmd = f"{sudo_cmd}python -c 'import os; os.makedirs(\"{target}\", exist_ok=True)'" + cmds = _which_split_cmd(cmd) + print(cmds) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen( + args=cmds, + out_handler=self._out_handler_sudo_password, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error( + f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}" f"\n\n{stderr}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + cmd = f'{sudo_cmd}python -c \'import shutil; shutil.copy2("{file}", "{cp_target}")\'' + cmds = _which_split_cmd(cmd) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen( + args=cmds, + out_handler=self._out_handler_sudo_password, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error(f"Error while copying({sub.returncode}):\n{' '.join(cmds)}\n\n{stderr}") + # ToDo: This depends on behaviour of tue-install-error + return False + + def tue_install_add_text(self, source_file: str, target_file: str) -> bool: pass - def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None): + def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None) -> bool: pass def tue_install_system(self, pkgs: List[str]): - pass + self._systems.extend(pkgs) - def tue_install_system_now(self, pkgs: List[str]): + def tue_install_system_now(self, pkgs: List[str]) -> bool: pass def tue_install_apt_get_update(self): - pass + self.tue_install_debug("tue-install-apt-get-update") + self.tue_install_debug("Requiring an update of apt-get before next 'apt-get install'") + if os.path.isfile(self._apt_get_updated_file): + os.remove(self._apt_get_updated_file) - def tue_install_ppa(self, ppa: List[str]): - pass + def tue_install_ppa(self, ppas: List[str]): + self._ppas.extend(ppas) - def tue_install_ppa_now(self, ppa: List[str]): + def tue_install_ppa_now(self, ppa: List[str]) -> bool: pass def tue_install_pip(self, pkgs: List[str]): - pass + self._pips.extend(pkgs) - def tue_install_pip_now(self, pkgs: List[str]): + def tue_install_pip_now(self, pkgs: List[str]) -> bool: pass def tue_install_snap(self, pkgs: List[str]): - pass + self._snaps.extend(pkgs) - def tue_install_snap_now(self, pkgs: List[str]): + def tue_install_snap_now(self, pkgs: List[str]) -> bool: pass def tue_install_gem(self, pkgs: List[str]): - pass + self._gems.extend(pkgs) + + def tue_install_gem_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-gem-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-gem-now call: got an empty list of packages as argument.") + + self.tue_install_system_now(["ruby", "ruby-dev", "rubygems-integration"]) + + cmd = "gem list" + cmds = _which_split_cmd(cmd) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error( + f"Error while getting installed gem packages({sub.returncode}):\n {' '.join(cmds)}\n\n{stderr}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False - def tue_install_gem_now(self, pkgs: List[str]): - pass + gems_installed = sub.stdout.read().splitlines() + gems_installed = [gem.split(" ")[0] for gem in gems_installed] + gems_to_install = [] + for pkg in pkgs: + if pkg not in gems_installed: + gems_to_install.append(pkg) + self.tue_install_debug(f"gem pkg: {pkg} is not yet installed") + else: + self.tue_install_debug(f"gem pkg: {pkg} is already installed") + + if gems_to_install: + cmd = f"gem install {' '.join(gems_to_install)}" + cmds = _which_split_cmd(cmd) + sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + stderr = sub.stderr.read() + self.tue_install_error( + f"An error occurred while installing gem packages({sub.returncode}):\n {' '.join(cmds)}" + f"\n\n{stderr}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + return True def tue_install_dpkg(self, pkg_file: str): pass @@ -323,10 +637,5 @@ def tue_install_ros(self, source_type: str, **kwargs): if __name__ == "__main__": - cmd = shlex.split("git -C /home/amigo/.tue status") - p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return_value = p1.wait() - print(f"{return_value=}") - print(f"{p1.stdout.readlines()=}") - print(f"{p1.stderr.readlines()=}") - print(date_stamp()) + bla = InstallerImpl(debug=True) + bla.tue_install_target("test") diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash new file mode 100644 index 000000000..d18c2d976 --- /dev/null +++ b/src/tue_get/resources/installer_impl.bash @@ -0,0 +1,217 @@ +#! /usr/bin/env bash + +function tue-install-error +{ + echo -e "tue-install-error: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-warning +{ + echo -e "tue-install-warning: $*" + local return_value + read -r return_value + echo -e "return_value: ${return_value}" + return $(("$return_value")) +} + +function tue-install-info +{ + echo -e "tue-install-info: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-debug +{ + echo -e "tue-install-info: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-echo +{ + echo -e "tue-install-echo: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-tee +{ + echo -e "tue-install-tee: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-pipe +{ + local pipefail_old return_code + pipefail_old=$(set -o | grep pipefail | awk '{printf $2}') + [ "$pipefail_old" != "on" ] && set -o pipefail # set pipefail if not yet set + tue-install-echo "$*" + # Executes the command (all arguments), catch stdout and stderr, red styled, print them directly and to file + { + IFS=$'\n' read -r -d '' CAPTURED_STDERR; + IFS=$'\n' read -r -d '' CAPTURED_STDOUT; + } < <((printf '\0%s\0' "$("$@")" 1>&2) 2>&1) + return_code=$? + [ "$pipefail_old" != "on" ] && set +o pipefail # restore old pipefail setting + echo -e "tue-install-pipe: ${CAPTURED_STDOUT}^^^${CAPTURED_STDERR}" + read -r return_value + if [ "$return_value" != "0" ] + then + return $(("$return_value")) + fi + + return $return_code +} + +function tue-install-target-now +{ + echo -e "tue-install-target-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-target +{ + echo -e "tue-install-target: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-git +{ + echo -e "tue-install-git: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-apply-patch +{ + echo -e "tue-install-apply-patch: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-cp +{ + echo -e "tue-install-cp: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-add-text +{ + echo -e "tue-install-add-text: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-get-releases +{ + echo -e "tue-install-get-releases: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-system +{ + echo -e "tue-install-system: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-system-now +{ + echo -e "tue-install-system-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-apt-get-update +{ + echo -e "tue-install-apt-get-update: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-ppa +{ + echo -e "tue-install-ppa: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-ppa-now +{ + echo -e "tue-install-ppa-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-pip +{ + echo -e "tue-install-pip: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-pip-now +{ + echo -e "tue-install-pip-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-snap +{ + echo -e "tue-install-snap: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-snap-now +{ + echo -e "tue-install-snap-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-gem +{ + echo -e "tue-install-gem: $*" + local return_value + read -r return_value + return $(("$return_value")) +} + +function tue-install-gem-now +{ + echo -e "tue-install-gem-now: $*" + local return_value + read -r return_value + return $(("$return_value")) +} diff --git a/src/tue_get/util.py b/src/tue_get/util.py index 25dea7671..340841eea 100644 --- a/src/tue_get/util.py +++ b/src/tue_get/util.py @@ -8,11 +8,10 @@ class BackgroundPopen(subprocess.Popen): Inspired by https://stackoverflow.com/a/42352331 """ - @staticmethod - def _proxy_lines(pipe: IO, handler: Callable): + def _proxy_lines(self, pipe: IO, handler: Callable): with pipe: for line in pipe: - handler(line) + handler(self, line) def __init__(self, out_handler: Optional[Callable] = None, err_handler: Optional[Callable] = None, *args, **kwargs): if out_handler is not None: @@ -38,8 +37,8 @@ def __init__(self, out_handler: Optional[Callable] = None, err_handler: Optional done' """ - def cyan_handler(line): - cprint(line, color="cyan", end="") + def cyan_handler(sub: BackgroundPopen, line: str): + cprint(line.strip(), color="cyan") bla = BackgroundPopen(out_handler=cyan_handler, args=shlex.split(cmd), text=True) bla.wait() From 872cbc7f32a4313933115873c859e28bd64ea624 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 08:52:47 +0100 Subject: [PATCH 07/47] Sudo password is on stderr --- src/tue_get/installer_impl.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 8e3babc66..28621b2d3 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -228,7 +228,7 @@ def _write_stdin(msg) -> None: else: self.tue_install_tee(line) - def _out_handler_sudo_password(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + def _err_handler_sudo_password(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: line = line.strip() if line.startswith("[sudo] password for"): if self._sudo_password is None: @@ -236,7 +236,7 @@ def _out_handler_sudo_password(self, sub: BackgroundPopen, line: Union[bytes, st sub.stdin.write(f"{self._sudo_password}\n") sub.stdin.flush() else: - self.tue_install_tee(line) + self.tue_install_tee(line, color="red") def tue_install_error(self, msg: str) -> None: # Make sure the entire msg is indented, not just the first line @@ -398,13 +398,13 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: sub = BackgroundPopen( args=cmds, out_handler=self._out_handler, - stderr=subprocess.PIPE, + err_handler=self._err_handler_sudo_password, stdin=subprocess.PIPE, text=True, ) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() + # stderr = sub.stderr.read() self.tue_install_error( f"Error while running({sub.returncode}):\n {' '.join(cmds)}" f"\n\n{stderr}" ) @@ -498,27 +498,26 @@ def tue_install_cp(self, source: str, target: str) -> bool: self.tue_install_debug(f"tue-install-cp: copying {file} to {cp_target}") if root_required: - sudo_cmd = "sudo -S " + sudo_cmd = "sudo " self.tue_install_debug(f"Using elevated privileges (sudo) to copy ${file} to ${cp_target}") else: sudo_cmd = "" cmd = f"{sudo_cmd}python -c 'import os; os.makedirs(\"{target}\", exist_ok=True)'" cmds = _which_split_cmd(cmd) - print(cmds) self.tue_install_echo(" ".join(cmds)) sub = BackgroundPopen( args=cmds, - out_handler=self._out_handler_sudo_password, + err_handler=self._err_handler_sudo_password, stderr=subprocess.PIPE, stdin=subprocess.PIPE, text=True, ) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() + # stderr = sub.stderr.read() self.tue_install_error( - f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}" f"\n\n{stderr}" + f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}" # f"\n\n{stderr}" ) # ToDo: This depends on behaviour of tue-install-error return False @@ -528,7 +527,7 @@ def tue_install_cp(self, source: str, target: str) -> bool: self.tue_install_echo(" ".join(cmds)) sub = BackgroundPopen( args=cmds, - out_handler=self._out_handler_sudo_password, + err_handler=self._err_handler_sudo_password, stderr=subprocess.PIPE, stdin=subprocess.PIPE, text=True, From 234710704a264c4e566d8ea6fc022a448b72bd4a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 16:56:39 +0100 Subject: [PATCH 08/47] remove tue_install_impl.py; should be moved --- installer/tue_install_impl.py | 273 ---------------------------------- 1 file changed, 273 deletions(-) delete mode 100644 installer/tue_install_impl.py diff --git a/installer/tue_install_impl.py b/installer/tue_install_impl.py deleted file mode 100644 index 40e906b3b..000000000 --- a/installer/tue_install_impl.py +++ /dev/null @@ -1,273 +0,0 @@ -from contextlib import contextmanager -import datetime -import os -from pathlib import Path -import shlex -import subprocess -from termcolor import colored - -from .parse_install_yaml import installyaml_parser2 - -CI = None - - -def is_CI() -> bool: - global CI - if CI is None: - CI = os.environ.get("CI", False) - - return CI - - -def date_stamp() -> str: - return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") - - -class InstallerImpl: - def __init__(self, debug: bool = False): - self._debug = debug - self._current_target = "main-loop" - self._current_target_dir = "" - stamp = date_stamp() - self._log_file = os.path.join("tmp", f"tue-get-details-{stamp}") - Path(self._log_file).touch(exist_ok=False) - - try: - self._tue_env_dir = os.environ["TUE_ENV_DIR"] - except KeyError: - print("'TUE_ENV_DIR' is not defined") - raise - - self._self._tue_apt_get_updated_file = os.path.join("tmp", "tue_get_apt_get_updated") - - self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") - self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") - self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") - - os.makedirs(self._dependencies_dir, exist_ok=True) - os.makedirs(self._dependencies_on_dir, exist_ok=True) - os.makedirs(self._installed_dir, exist_ok=True) - - try: - self._targets_dir = os.environ["TUE_ENV_TARGETS_DIR"] - except KeyError: - print("'TUE_ENV_TARGETS_DIR' is not defined") - raise - - if not os.path.isdir(self._targets_dir): - raise RuntimeError(f"TUE_INSTALL_TARGETS_DIR '{self._targets_dir}' does not exist as directory") - - general_state_dir = os.path.join("tmp", "tue-installer") - if not os.path.isdir(general_state_dir): - self.tue_install_debug(f"mkdir {general_state_dir}") - os.makedirs(general_state_dir, exist_ok=True) - self.tue_install_debug(f"chmod a+rwx {general_state_dir}") - os.chmod(general_state_dir, 777) - - self._state_dir = os.path.join(general_state_dir, stamp) - self.tue_install_debug(f"mkdir {self._state_dir}") - os.makedirs(self._state_dir) - - self._warn_logs = [] - self._info_logs = [] - - def __del__(self): - os.rmdir(self._state_dir) - - def _log_to_file(self, msg: str) -> None: - with open(self._log_file, "a") as f: - f.write(msg + "\n") - - def _write_sorted_deps(self, child: str) -> None: - self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) - self._write_sorted_dep_file(os.path.join(self._dependencies_on_dir, child), self._current_target) - - def _write_sorted_dep_file(self, file: str, target: str) -> None: - if not os.path.isfile(file): - Path(file).touch(exist_ok=False) - targets = [] - else: - with open(file, "r+") as f: - targets = f.readlines() - - print(f"{targets=}") - targets.append(target + "\n") - targets.sort() - with open(file, "w") as f: - f.writelines(targets) - - @contextmanager - def _set_target(self, target: str): - parent_target = self._current_target - parent_target_dir = self._current_target_dir - self._current_target = target - self._current_target_dir = os.path.join(self._targets_dir, target) - yield - self._current_target = parent_target - self._current_target_dir = parent_target_dir - - def tue_install_error(self, msg: str) -> None: - log_msg = f"Error while installing target '{self._current_target}':\n\n\t{msg}\n\nLogfile: {self._log_file}" - print(colored(log_msg, color="red")) - self._log_to_file(log_msg) - - def tue_install_warning(self, msg: str) -> None: - log_msg = f"[{self._current_target}] WARNING: {msg}" - colored_log = colored(log_msg, color="yellow", attrs=["bold"]) - self._warn_logs.append(colored_log) - self._log_to_file(log_msg) - - def tue_install_info(self, msg: str) -> None: - log_msg = f"[{self._current_target}] INFO: {msg}" - colored_log = colored(log_msg, color="cyan") - self._info_logs.append(colored_log) - print(colored_log) - self._log_to_file(log_msg) - - def tue_install_debug(self, msg: str) -> None: - log_msg = f"[{self._current_target}] DEBUG: {msg}" - colored_log = colored(log_msg, color="blue") - if self._debug: - print(colored_log) - self._log_to_file(log_msg) - - def tue_install_echo(self, msg: str) -> None: - log_msg = f"[{self._current_target}]: {msg}" - colored_log = colored(log_msg, attrs=["bold"]) - print(colored_log) - self._log_to_file(log_msg) - - def tue_install_tee(self, msg: str) -> None: - print(msg) - self._log_to_file(msg) - - def tue_install_pipe(self): - # ToDo: This should be implemented for install.bash scripts, not needed anymore for system calls in this script - pass - - def tue_install_target_now(self, target: str) -> int: - self.tue_install_debug(f"tue-install-target-now {target}") - - self.tue_install_debug(f"calling: tue-install-target {target} true") - return self.tue_install_target(target, True) - - def tue_install_target(self, target: str, now: bool = False) -> int: - self.tue_install_debug(f"tue-install-target {target} {now}") - - self.tue_install_debug("Installing target: {target}") - - # Check if valid target received as input - if not os.path.isdir(os.path.join(self._targets_dir, target)): - self.tue_install_debug(f"Target '{target}' does not exist.") - return False - - # If the target has a parent target, add target as a dependency to the parent target - if self._current_target and self._current_target != "main-loop" and self._current_target != target: - self._write_sorted_deps(target) - - with self._set_target(target): - - state_file = os.path.join(self._state_dir, "target") - state_file_now = f"{state_file}-now" - - # Determine if this target needs to be executed - execution_needed = True - - if is_CI() and not os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore")): - self.tue_install_debug( - f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. No execution is needed." - ) - execution_needed = False - elif os.path.isfile(state_file_now): - self.tue_install_debug( - f"File {state_file_now} does exist, so installation has already been executed with 'now' option. No execution is needed." - ) - execution_needed = False - elif os.path.isfile(state_file): - if now: - self.tue_install_debug( - f"File {state_file_now} doesn't exist, but file {state_file} does. So installation has been executed yet, but not with the 'now' option. Going to execute it with 'now' option." - ) - else: - self.tue_install_debug( - f"File {state_file_now} does exist. 'now' is not enabled, so no execution needed." - ) - execution_needed = False - else: - if now: - self.tue_install_debug( - f"Files {state_file_now} and {state_file} don't exist. Going to execute with 'now' option." - ) - else: - self.tue_install_debug( - f"Files {state_file_now} and {state_file} don't exist. Going to execute without 'now' option." - ) - - if execution_needed: - self.tue_install_debug("Starting installation") - install_file = os.path.join(self._current_target_dir, "install") - - # Empty the target's dependency file - dep_file = os.path.join(self._dependencies_dir, target) - self.tue_install_debug(f"Emptying {dep_file}") - if os.path.isfile(dep_file): - with open(dep_file, "a") as f: - f.truncate(0) - - target_processed = False - - install_yaml_file = install_file + ".yaml" - if os.path.isfile(install_yaml_file): - if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_yaml")): - self.tue_install_debug( - "Running in CI mode and found .ci_ignore_yaml file, so skipping install.yaml" - ) - target_processed = True - else: - self.tue_install_debug(f"Parsing {install_yaml_file}") - - cmds = installyaml_parser2(self, install_yaml_file, now)["commands"] - if not cmds: - self.tue_install_error(f"Invalid install.yaml: {cmds}") - - for cmd in cmds: - self.tue_install_debug(str(cmd)) - cmd() or self.tue_install_error( - f"Error while running: {cmd}" - ) # ToDo: Fix exception/return value handling - - target_processed = True - - install_bash_file = install_file + ".bash" - if os.path.isfile(install_bash_file): - if is_CI() and os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore_bash")): - self.tue_install_debug( - "Running in CI mode and found .ci_ignore_bash file, so skipping install.bash" - ) - else: - self.tue_install_debug(f"Sourcing {install_bash_file}") - # ToDo: source install.bash - - target_processed = True - - if not target_processed: - self.tue_install_warning("Target does not contain a valid install.yaml/bash file") - - if now: - state_file_to_touch = state_file_now - else: - state_file_to_touch = state_file - - Path(state_file_to_touch).touch() - - self.tue_install_debug(f"Finished installing {target}") - - -if __name__ == "__main__": - cmd = shlex.split("git -C /home/amigo/.tue status") - p1 = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return_value = p1.wait() - print(f"{return_value=}") - print(f"{p1.stdout.readlines()=}") - print(f"{p1.stderr.readlines()=}") - print(date_stamp()) From 4c55516dc60c3df452b4618499bda421bcc1f215 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 16:58:04 +0100 Subject: [PATCH 09/47] Fix tue-install-pipe; does require echo of stdout for pipe --- src/tue_get/installer_impl.py | 7 +++++-- src/tue_get/resources/installer_impl.bash | 12 +++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 28621b2d3..b22da2b3e 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -159,6 +159,7 @@ def _write_stdin(msg) -> None: _write_stdin(0) elif line.startswith("tue-install-pipe: "): output = line[18:].split("^^^") + output = map(lambda x: x.replace("^", "\n").strip(), output) self.tue_install_pipe(*output) _write_stdin(0) elif line.startswith("tue-install-target-now: "): @@ -287,8 +288,10 @@ def tue_install_pipe(self, stdout: str, stderr: str) -> None: :param stderr: :return: """ - self.tue_install_tee(stdout) - self.tue_install_tee(stderr, color="red") + if stdout: + self.tue_install_tee(stdout) + if stderr: + self.tue_install_tee(stderr, color="red") def tue_install_target_now(self, target: str) -> bool: self.tue_install_debug(f"tue-install-target-now {target=}") diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index d18c2d976..38b20d5a8 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -13,7 +13,6 @@ function tue-install-warning echo -e "tue-install-warning: $*" local return_value read -r return_value - echo -e "return_value: ${return_value}" return $(("$return_value")) } @@ -27,7 +26,7 @@ function tue-install-info function tue-install-debug { - echo -e "tue-install-info: $*" + echo -e "tue-install-debug: $*" local return_value read -r return_value return $(("$return_value")) @@ -51,6 +50,7 @@ function tue-install-tee function tue-install-pipe { + local return_code local pipefail_old return_code pipefail_old=$(set -o | grep pipefail | awk '{printf $2}') [ "$pipefail_old" != "on" ] && set -o pipefail # set pipefail if not yet set @@ -62,13 +62,19 @@ function tue-install-pipe } < <((printf '\0%s\0' "$("$@")" 1>&2) 2>&1) return_code=$? [ "$pipefail_old" != "on" ] && set +o pipefail # restore old pipefail setting + # shellcheck disable=SC2034 + TUE_INSTALL_PIPE_STDOUT=$CAPTURED_STDOUT + + CAPTURED_STDOUT=$(echo -e "$CAPTURED_STDOUT" | tr '\n' '^') + CAPTURED_STDERR=$(echo -e "$CAPTURED_STDERR" | tr '\n' '^') echo -e "tue-install-pipe: ${CAPTURED_STDOUT}^^^${CAPTURED_STDERR}" read -r return_value if [ "$return_value" != "0" ] then + tue-install-echo "return_value: $return_value" return $(("$return_value")) fi - + tue-install-echo "return_code: $return_code" return $return_code } From 7207cc94b5abd4a5dac594f91ed720edced9bea1 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 17:52:41 +0100 Subject: [PATCH 10/47] Add tue-install-add-text --- src/tue_get/install_yaml_parser.py | 1 + src/tue_get/installer_impl.py | 203 +++++++++++++----- test/test_tue_get/test_install_yaml_parser.py | 1 + 3 files changed, 152 insertions(+), 53 deletions(-) diff --git a/src/tue_get/install_yaml_parser.py b/src/tue_get/install_yaml_parser.py index 4c18e816c..845bf8f9c 100644 --- a/src/tue_get/install_yaml_parser.py +++ b/src/tue_get/install_yaml_parser.py @@ -211,6 +211,7 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type: if __name__ == "__main__": import sys from tue_get.installer_impl import InstallerImpl + if len(sys.argv) < 2: print("Provide yaml file to parse") exit(1) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index b22da2b3e..d5f2aa5fc 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -185,8 +185,8 @@ def _write_stdin(msg) -> None: _write_stdin(not success) elif line.startswith("tue-install-system: "): pkgs = line[20:].split() - self.tue_install_system(pkgs) - _write_stdin(0) + success = self.tue_install_system(pkgs) + _write_stdin(not success) elif line.startswith("tue-install-system-now: "): pkgs = line[24:].split() success = self.tue_install_system_now(pkgs) @@ -196,32 +196,32 @@ def _write_stdin(msg) -> None: _write_stdin(0) elif line.startswith("tue-install-ppa: "): ppas = line[17:].split() - self.tue_install_ppa(ppas) - _write_stdin(0) + success = self.tue_install_ppa(ppas) + _write_stdin(not success) elif line.startswith("tue-install-ppa-now: "): ppas = line[21:].split() success = self.tue_install_ppa_now(ppas) _write_stdin(not success) elif line.startswith("tue-install-pip: "): pkgs = line[17:].split() - self.tue_install_pip(pkgs) - _write_stdin(0) + success = self.tue_install_pip(pkgs) + _write_stdin(not success) elif line.startswith("tue-install-pip-now: "): pkgs = line[20:].split() success = self.tue_install_pip_now(pkgs) _write_stdin(not success) elif line.startswith("tue-install-snap: "): pkgs = line[18:].split() - self.tue_install_snap(pkgs) - _write_stdin(0) + success = self.tue_install_snap(pkgs) + _write_stdin(not success) elif line.startswith("tue-install-snap-now: "): pkgs = line[22:].split() success = self.tue_install_snap_now(pkgs) _write_stdin(not success) elif line.startswith("tue-install-gem: "): pkgs = line[17:].split(" ") - self.tue_install_gem(pkgs) - _write_stdin(0) + success = self.tue_install_gem(pkgs) + _write_stdin(not success) elif line.startswith("tue-install-gem-now: "): pkgs = line[21:].split(" ") success = self.tue_install_gem_now(pkgs) @@ -229,7 +229,7 @@ def _write_stdin(msg) -> None: else: self.tue_install_tee(line) - def _err_handler_sudo_password(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + def _err_handler(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: line = line.strip() if line.startswith("[sudo] password for"): if self._sudo_password is None: @@ -302,7 +302,7 @@ def tue_install_target_now(self, target: str) -> bool: def tue_install_target(self, target: str, now: bool = False) -> bool: self.tue_install_debug(f"tue-install-target {target=} {now=}") - self.tue_install_debug("Installing target: {target}") + self.tue_install_debug(f"Installing target: {target}") # Check if valid target received as input if not os.path.isdir(os.path.join(self._targets_dir, target)): @@ -352,7 +352,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: ) if execution_needed: - self.tue_install_debug("Starting installation") + self.tue_install_debug(f"Starting installation of target: {target}") install_file = os.path.join(self._current_target_dir, "install") # Empty the target's dependency file @@ -401,16 +401,14 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: sub = BackgroundPopen( args=cmds, out_handler=self._out_handler, - err_handler=self._err_handler_sudo_password, + err_handler=self._err_handler, stdin=subprocess.PIPE, text=True, ) sub.wait() if sub.returncode != 0: # stderr = sub.stderr.read() - self.tue_install_error( - f"Error while running({sub.returncode}):\n {' '.join(cmds)}" f"\n\n{stderr}" - ) + self.tue_install_error(f"Error while running({sub.returncode}):\n {' '.join(cmds)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -452,11 +450,10 @@ def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: cmd = f"patch -s -N -r - -p0 -d {target_dir} < {patch_file_path}" cmds = _which_split_cmd(cmd) self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub = BackgroundPopen(args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, text=True) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() - self.tue_install_error(f"Error while running({sub.returncode}):\n{' '.join(cmds)}\n\n{stderr}") + self.tue_install_error(f"Error while running({sub.returncode}):\n{' '.join(cmds)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -511,17 +508,14 @@ def tue_install_cp(self, source: str, target: str) -> bool: self.tue_install_echo(" ".join(cmds)) sub = BackgroundPopen( args=cmds, - err_handler=self._err_handler_sudo_password, - stderr=subprocess.PIPE, + out_handler=self._out_handler, + err_handler=self._err_handler, stdin=subprocess.PIPE, text=True, ) sub.wait() if sub.returncode != 0: - # stderr = sub.stderr.read() - self.tue_install_error( - f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}" # f"\n\n{stderr}" - ) + self.tue_install_error(f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -530,29 +524,131 @@ def tue_install_cp(self, source: str, target: str) -> bool: self.tue_install_echo(" ".join(cmds)) sub = BackgroundPopen( args=cmds, - err_handler=self._err_handler_sudo_password, - stderr=subprocess.PIPE, + out_handler=self._out_handler, + err_handler=self._err_handler, stdin=subprocess.PIPE, text=True, ) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() - self.tue_install_error(f"Error while copying({sub.returncode}):\n{' '.join(cmds)}\n\n{stderr}") + self.tue_install_error(f"Error while copying({sub.returncode}):\n{' '.join(cmds)}") # ToDo: This depends on behaviour of tue-install-error return False def tue_install_add_text(self, source_file: str, target_file: str) -> bool: - pass + self.tue_install_debug(f"tue-install-add-text {source_file=} {target_file=}") + + if not source_file: + self.tue_install_error("Invalid tue-install-add-text call: needs source file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_file: + self.tue_install_error("Invalid tue-install-add-text call: needs target file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + if source_file.startswith("/") or source_file.startswith("~"): + self.tue_install_error( + "Invalid tue-install-add-text call: source file must be relative to target directory" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_file.startswith("/") and not target_file.startswith("~"): + self.tue_install_error( + "Invalid tue-install-add-text call: target file must be absolute or " "relative to the home directory" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + root_required = True + if os.access(target_file, os.W_OK): + root_required = False + self.tue_install_debug("tue-install-add-text: NO root required") + else: + self.tue_install_debug("tue-install-add-text: root required") + + if root_required: + sudo_cmd = "sudo " + self.tue_install_debug("Using elevated privileges (sudo) to add text") + else: + sudo_cmd = "" + + source_file_path = os.path.join(self._current_target_dir, source_file) + if not os.path.isfile(source_file_path): + self.tue_install_error(f"tue-install-add-text: source file {source_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + target_file_path = os.path.expanduser(target_file) + if not os.path.isfile(target_file_path): + self.tue_install_error(f"tue-install-add-text: target file {target_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + with open(source_file_path, "r") as f: + source_text = f.read().splitlines() + + begin_tag = source_text[0] + end_tag = source_text[-1] + source_body = source_text[1:-1] + source_text = "\n".join(source_text) + + self.tue_install_debug(f"tue-install-add-text: {begin_tag=}, {end_tag=}\n{source_body=}") + + with open(target_file_path, "r") as f: + target_text = f.read().splitlines() + + if not begin_tag in target_text: + self.tue_install_debug( + f"tue-install-add-text: {begin_tag=} not found in {target_file_path=}, " + "appending to {target_file_path}" + ) + cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path} 1>/dev/null\"" + else: + self.tue_install_debug( + f"tue-install-add-text: {begin_tag=} found in {target_file_path=}, " + "so comparing the files for changed lines" + ) + + if filecmp.cmp(source_file_path, target_file_path): + self.tue_install_debug( + f"tue-install-add-text: {source_file_path=} and {target_file_path=} are " "identical, skipping" + ) + return True + + begin_index = target_text.index(begin_tag) + end_index = target_text.index(end_tag) + target_text = target_text[:begin_index] + source_text.splitlines() + target_text[end_index + 1 :] + target_text = "\n".join(target_text) + + cmd = f"bash -c \"echo -e '{target_text}' | {sudo_cmd}tee {target_file_path} 1>/dev/null\"" + + cmds = _which_split_cmd(cmd) + self.tue_install_echo(" ".join(cmds)) + sub = BackgroundPopen( + args=cmds, + out_handler=self._out_handler, + err_handler=self._err_handler, + stdin=subprocess.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while adding text({sub.returncode}):\n{' '.join(cmds)}") + # ToDo: This depends on behaviour of tue-install-error + return False def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None) -> bool: pass - def tue_install_system(self, pkgs: List[str]): + def tue_install_system(self, pkgs: List[str]) -> bool: self._systems.extend(pkgs) + return True def tue_install_system_now(self, pkgs: List[str]) -> bool: - pass + return True def tue_install_apt_get_update(self): self.tue_install_debug("tue-install-apt-get-update") @@ -560,26 +656,30 @@ def tue_install_apt_get_update(self): if os.path.isfile(self._apt_get_updated_file): os.remove(self._apt_get_updated_file) - def tue_install_ppa(self, ppas: List[str]): + def tue_install_ppa(self, ppas: List[str]) -> bool: self._ppas.extend(ppas) + return True def tue_install_ppa_now(self, ppa: List[str]) -> bool: - pass + return True - def tue_install_pip(self, pkgs: List[str]): + def tue_install_pip(self, pkgs: List[str]) -> bool: self._pips.extend(pkgs) + return True def tue_install_pip_now(self, pkgs: List[str]) -> bool: - pass + return True - def tue_install_snap(self, pkgs: List[str]): + def tue_install_snap(self, pkgs: List[str]) -> bool: self._snaps.extend(pkgs) + return True def tue_install_snap_now(self, pkgs: List[str]) -> bool: - pass + return True - def tue_install_gem(self, pkgs: List[str]): + def tue_install_gem(self, pkgs: List[str]) -> bool: self._gems.extend(pkgs) + return True def tue_install_gem_now(self, pkgs: List[str]) -> bool: self.tue_install_debug(f"tue-install-gem-now {pkgs=}") @@ -592,12 +692,11 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: cmd = "gem list" cmds = _which_split_cmd(cmd) self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=subprocess.PIPE, text=True) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() self.tue_install_error( - f"Error while getting installed gem packages({sub.returncode}):\n {' '.join(cmds)}\n\n{stderr}" + f"Error while getting installed gem packages({sub.returncode}):\n {' '.join(cmds)}" ) # ToDo: This depends on behaviour of tue-install-error return False @@ -615,29 +714,27 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: if gems_to_install: cmd = f"gem install {' '.join(gems_to_install)}" cmds = _which_split_cmd(cmd) - sub = BackgroundPopen(args=cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sub = BackgroundPopen(args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, text=True) sub.wait() if sub.returncode != 0: - stderr = sub.stderr.read() self.tue_install_error( f"An error occurred while installing gem packages({sub.returncode}):\n {' '.join(cmds)}" - f"\n\n{stderr}" ) # ToDo: This depends on behaviour of tue-install-error return False return True - def tue_install_dpkg(self, pkg_file: str): - pass + def tue_install_dpkg(self, pkg_file: str) -> bool: + return True - def tue_install_dpkg_now(self, pkg_file: str): - pass + def tue_install_dpkg_now(self, pkg_file: str) -> bool: + return True - def tue_install_ros(self, source_type: str, **kwargs): - pass + def tue_install_ros(self, source_type: str, **kwargs) -> bool: + return True if __name__ == "__main__": bla = InstallerImpl(debug=True) - bla.tue_install_target("test") + bla.tue_install_target("networking") diff --git a/test/test_tue_get/test_install_yaml_parser.py b/test/test_tue_get/test_install_yaml_parser.py index 3826f23d4..11a7abc03 100644 --- a/test/test_tue_get/test_install_yaml_parser.py +++ b/test/test_tue_get/test_install_yaml_parser.py @@ -104,6 +104,7 @@ def test_empty_target(): cmds = parse_target(target, True) assert not cmds + def test_system_target(): target = [{"type": "system", "name": "pkg_name"}] cmds = parse_target(target, False) From 2a0ca0aec940ac791cbfaf14068fc8a3228275ad Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 11:32:55 +0100 Subject: [PATCH 11/47] Reorder deps --- setup.py | 2 +- src/tue_get/{util.py => util/BackgroundPopen.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/tue_get/{util.py => util/BackgroundPopen.py} (100%) diff --git a/setup.py b/setup.py index 3683e1a6f..3cb955da4 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ # }, python_version=">=3.8", install_requires=[ - "termcolor", "pyyaml", + "termcolor", ], ) diff --git a/src/tue_get/util.py b/src/tue_get/util/BackgroundPopen.py similarity index 100% rename from src/tue_get/util.py rename to src/tue_get/util/BackgroundPopen.py From 98672ccb49e312a1976ffeea8e0cfba33d7d376d Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 11:35:13 +0100 Subject: [PATCH 12/47] Workin tue-install-system-now --- src/tue_get/installer_impl.py | 263 ++++++++++++++++++++++++---------- 1 file changed, 187 insertions(+), 76 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index d5f2aa5fc..a690df16b 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1,19 +1,20 @@ -import glob -import subprocess -from typing import List, Optional, Union +from typing import List, Optional, Union, Tuple from contextlib import contextmanager import datetime import filecmp import getpass +import glob import os from pathlib import Path import shlex import shutil +import subprocess as sp from termcolor import colored +from time import sleep from tue_get.install_yaml_parser import installyaml_parser -from tue_get.util import BackgroundPopen +from tue_get.util.BackgroundPopen import BackgroundPopen CI = None @@ -30,14 +31,14 @@ def date_stamp() -> str: return datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S") -def _which_split_cmd(cmd: str) -> List[str]: +def _which_split_cmd(cmd: str) -> Tuple[str, List[str]]: cmds = shlex.split(cmd) cmds[0] = shutil.which(cmds[0]) - return cmds + return " ".join(cmds), cmds class InstallerImpl: - _apt_get_update_file = os.path.join(os.pathsep, "tmp", "tue_get_apt_get_updated") + _apt_get_updated_file = os.path.join(os.sep, "tmp", "tue_get_apt_get_updated") def __init__(self, debug: bool = False): self._debug = debug @@ -142,6 +143,7 @@ def _write_stdin(msg) -> None: elif line.startswith("tue-install-error: "): self.tue_install_error(line[19:]) _write_stdin(1) + # ToDo: or should we call sub.kill() here? elif line.startswith("tue-install-warning: "): self.tue_install_warning(line[21:]) _write_stdin(0) @@ -239,6 +241,15 @@ def _err_handler(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: else: self.tue_install_tee(line, color="red") + def _default_background_popen(self, cmd: str) -> BackgroundPopen: + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_echo(repr(cmd)) + sub = BackgroundPopen( + args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, stdin=sp.PIPE, text=True + ) + sub.wait() + return sub + def tue_install_error(self, msg: str) -> None: # Make sure the entire msg is indented, not just the first line lines = msg.splitlines() @@ -323,18 +334,22 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: if is_CI() and not os.path.isfile(os.path.join(self._current_target_dir, ".ci_ignore")): self.tue_install_debug( - f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. No execution is needed." + f"Running installer in CI mode and file {self._current_target_dir}/.ci_ignore exists. " + "No execution is needed." ) execution_needed = False elif os.path.isfile(state_file_now): self.tue_install_debug( - f"File {state_file_now} does exist, so installation has already been executed with 'now' option. No execution is needed." + f"File {state_file_now} does exist, so installation has already been executed with 'now' option. " + "No execution is needed." ) execution_needed = False elif os.path.isfile(state_file): if now: self.tue_install_debug( - f"File {state_file_now} doesn't exist, but file {state_file} does. So installation has been executed yet, but not with the 'now' option. Going to execute it with 'now' option." + f"File {state_file_now} doesn't exist, but file {state_file} does. " + "So installation has been executed yet, but not with the 'now' option. " + "Going to execute it with 'now' option." ) else: self.tue_install_debug( @@ -377,6 +392,8 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: cmds = installyaml_parser(self, install_yaml_file, now)["commands"] if not cmds: self.tue_install_error(f"Invalid install.yaml: {cmds}") + # ToDo: This depends on behaviour of tue-install-error + return False for cmd in cmds: self.tue_install_debug(str(cmd)) @@ -395,20 +412,10 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: else: self.tue_install_debug(f"Sourcing {install_bash_file}") resource_file = os.path.join(os.path.dirname(__file__), "resources", "installer_impl.bash") - cmd = f"bash -c 'source {resource_file} && source {install_bash_file}'" - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen( - args=cmds, - out_handler=self._out_handler, - err_handler=self._err_handler, - stdin=subprocess.PIPE, - text=True, - ) - sub.wait() + cmd = f"bash -c \"source {resource_file} && source {install_bash_file}\"" + sub = self._default_background_popen(cmd) if sub.returncode != 0: - # stderr = sub.stderr.read() - self.tue_install_error(f"Error while running({sub.returncode}):\n {' '.join(cmds)}") + self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -434,26 +441,31 @@ def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: if not patch_file: self.tue_install_error("Invalid tue-install-apply-patch call: needs patch file as argument") + # ToDo: This depends on behaviour of tue-install-error + return False patch_file_path = os.path.join(self._current_target_dir, patch_file) if not os.path.isfile(patch_file_path): self.tue_install_error(f"Invalid tue-install-apply-patch call: patch file {patch_file_path} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False if not target_dir: self.tue_install_error("Invalid tue-install-apply-patch call: needs target directory as argument") + # ToDo: This depends on behaviour of tue-install-error + return False if not os.path.isdir(target_dir): self.tue_install_error( f"Invalid tue-install-apply-patch call: target directory {target_dir} does not exist" ) + # ToDo: This depends on behaviour of tue-install-error + return False cmd = f"patch -s -N -r - -p0 -d {target_dir} < {patch_file_path}" - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen(args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, text=True) - sub.wait() + sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"Error while running({sub.returncode}):\n{' '.join(cmds)}") + self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -464,9 +476,13 @@ def tue_install_cp(self, source: str, target: str) -> bool: if not source: self.tue_install_error("Invalid tue-install-cp call: needs source directory as argument") + # ToDo: This depends on behaviour of tue-install-error + return False if not target: self.tue_install_error("Invalid tue-install-cp call: needs target dir as argument") + # ToDo: This depends on behaviour of tue-install-error + return False if os.path.isdir(target): self.tue_install_debug(f"tue-install-cp: target {target} is a directory") @@ -475,6 +491,8 @@ def tue_install_cp(self, source: str, target: str) -> bool: target = os.path.dirname(target) else: self.tue_install_error(f"tue-install-cp: target {target} does not exist") + # ToDo: This depends on behaviour of tue-install-error + return False # Check if user is allowed to write on target destination root_required = True @@ -488,6 +506,8 @@ def tue_install_cp(self, source: str, target: str) -> bool: for file in source_files: if not os.path.isfile(file): self.tue_install_error(f"tue-install-cp: source {file} is not a file") + # ToDo: This depends on behaviour of tue-install-error + return False cp_target = os.path.join(target, os.path.basename(file)) @@ -504,34 +524,16 @@ def tue_install_cp(self, source: str, target: str) -> bool: sudo_cmd = "" cmd = f"{sudo_cmd}python -c 'import os; os.makedirs(\"{target}\", exist_ok=True)'" - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen( - args=cmds, - out_handler=self._out_handler, - err_handler=self._err_handler, - stdin=subprocess.PIPE, - text=True, - ) - sub.wait() + sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"Error while creating the directory({sub.returncode}):\n{' '.join(cmds)}") + self.tue_install_error(f"Error while creating the directory({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False cmd = f'{sudo_cmd}python -c \'import shutil; shutil.copy2("{file}", "{cp_target}")\'' - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen( - args=cmds, - out_handler=self._out_handler, - err_handler=self._err_handler, - stdin=subprocess.PIPE, - text=True, - ) - sub.wait() + sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"Error while copying({sub.returncode}):\n{' '.join(cmds)}") + self.tue_install_error(f"Error while copying({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -593,7 +595,6 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: begin_tag = source_text[0] end_tag = source_text[-1] source_body = source_text[1:-1] - source_text = "\n".join(source_text) self.tue_install_debug(f"tue-install-add-text: {begin_tag=}, {end_tag=}\n{source_body=}") @@ -605,49 +606,137 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: f"tue-install-add-text: {begin_tag=} not found in {target_file_path=}, " "appending to {target_file_path}" ) - cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path} 1>/dev/null\"" + source_text = "\n".join(source_text) + cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path}\"" else: self.tue_install_debug( f"tue-install-add-text: {begin_tag=} found in {target_file_path=}, " "so comparing the files for changed lines" ) - if filecmp.cmp(source_file_path, target_file_path): - self.tue_install_debug( - f"tue-install-add-text: {source_file_path=} and {target_file_path=} are " "identical, skipping" - ) - return True - begin_index = target_text.index(begin_tag) end_index = target_text.index(end_tag) - target_text = target_text[:begin_index] + source_text.splitlines() + target_text[end_index + 1 :] + target_body = target_text[begin_index + 1 : end_index] + self.tue_install_debug(f"tue-install-add-text:\n{target_body=}") + + if source_body == target_body: + self.tue_install_debug("tue-install-add-text: Lines have not changed, so not copying") + return True + + self.tue_install_debug("tue-install-add-text: Lines have changed, so copying") + + target_text = target_text[: begin_index + 1] + source_body + target_text[end_index:] + print(target_text) target_text = "\n".join(target_text) - cmd = f"bash -c \"echo -e '{target_text}' | {sudo_cmd}tee {target_file_path} 1>/dev/null\"" + cmd = f"bash -c \"echo -e \'{target_text}\' | {sudo_cmd}tee {target_file_path}\"" - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_echo(repr(cmd)) sub = BackgroundPopen( args=cmds, - out_handler=self._out_handler, err_handler=self._err_handler, - stdin=subprocess.PIPE, + stdout=sp.DEVNULL, + stdin=sp.PIPE, text=True, ) sub.wait() if sub.returncode != 0: - self.tue_install_error(f"Error while adding text({sub.returncode}):\n{' '.join(cmds)}") + self.tue_install_error(f"Error while adding text({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None) -> bool: - pass + return True def tue_install_system(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-system {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-system call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + self._systems.extend(pkgs) return True def tue_install_system_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-system-now {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-system-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + installed_pkgs = [] + + def _out_handler_installed_pkgs(sub: BackgroundPopen, line: str) -> None: + installed_pkgs.append(line.strip()) + + # Check if pkg is not already installed dpkg -S does not cover previously removed packages + # Based on https://stackoverflow.com/questions/1298066 + cmd = "dpkg-query -W -f '${package} ${status}\n'" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_echo(repr(cmd)) + sub = BackgroundPopen( + args=cmds, + out_handler=_out_handler_installed_pkgs, # Needed to prevent buffer to get full + err_handler=self._err_handler, + stdin=sp.PIPE, + text=True, + ) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while getting installed packages({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + installed_pkgs = [pkg[:-21] for pkg in installed_pkgs if pkg[-20:] == "install ok installed"] + pkgs_to_install = [] + for pkg in pkgs: + if pkg not in installed_pkgs: + self.tue_install_debug(f"Package {pkg} is not installed") + pkgs_to_install.append(pkg) + else: + self.tue_install_debug(f"Package {pkg} is already installed") + + if not pkgs_to_install: + return True + + # Install packages + apt_get_cmd = f"sudo apt-get install --assume-yes -q {' '.join(pkgs_to_install)}" + self.tue_install_echo(f"Going to run the following command:\n{apt_get_cmd}") + + def _wait_for_dpkg_lock(): + i = 0 + rotate_list = ["-", "\\", "|", "/"] + cmd = "sudo fuser /var/lib/dpkg/lock" + cmd, cmds = _which_split_cmd(cmd) + while sp.run(cmds, stdout=sp.DEVNULL, stderr=sp.DEVNULL).returncode == 0: + print(f"[{rotate_list[i % len(rotate_list)]}] Waiting for dpkg lock", end="\r") + i += 1 + sleep(0.4) + return + + _wait_for_dpkg_lock() + + if not os.path.isfile(self._apt_get_updated_file): + cmd = "sudo apt-get update" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Error while updating apt-get({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + Path(self._apt_get_updated_file).touch() + + sub = self._default_background_popen(apt_get_cmd) + if sub.returncode != 0: + self.tue_install_error( + f"Error while installing system packages({sub.returncode}):" f"\n {repr(apt_get_cmd)}" + ) + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"Installed {pkgs} ({sub.returncode})") + return True def tue_install_apt_get_update(self): @@ -657,6 +746,12 @@ def tue_install_apt_get_update(self): os.remove(self._apt_get_updated_file) def tue_install_ppa(self, ppas: List[str]) -> bool: + self.tue_install_debug(f"tue-install-ppa {ppas=}") + if not ppas: + self.tue_install_error("Invalid tue-install-ppa call: needs ppa as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + self._ppas.extend(ppas) return True @@ -664,6 +759,12 @@ def tue_install_ppa_now(self, ppa: List[str]) -> bool: return True def tue_install_pip(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-pip {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-pip call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + self._pips.extend(pkgs) return True @@ -671,6 +772,12 @@ def tue_install_pip_now(self, pkgs: List[str]) -> bool: return True def tue_install_snap(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-snap {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-snap call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + self._snaps.extend(pkgs) return True @@ -678,6 +785,12 @@ def tue_install_snap_now(self, pkgs: List[str]) -> bool: return True def tue_install_gem(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-gem {pkgs=}") + if not pkgs: + self.tue_install_error("Invalid tue-install-gem call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + self._gems.extend(pkgs) return True @@ -686,18 +799,18 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: if not pkgs: self.tue_install_error("Invalid tue-install-gem-now call: got an empty list of packages as argument.") + # ToDo: This depends on behaviour of tue-install-error + return False self.tue_install_system_now(["ruby", "ruby-dev", "rubygems-integration"]) cmd = "gem list" - cmds = _which_split_cmd(cmd) - self.tue_install_echo(" ".join(cmds)) - sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=subprocess.PIPE, text=True) + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_echo(repr(cmd)) + sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) sub.wait() if sub.returncode != 0: - self.tue_install_error( - f"Error while getting installed gem packages({sub.returncode}):\n {' '.join(cmds)}" - ) + self.tue_install_error(f"Error while getting installed gem packages({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False @@ -713,12 +826,10 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: if gems_to_install: cmd = f"gem install {' '.join(gems_to_install)}" - cmds = _which_split_cmd(cmd) - sub = BackgroundPopen(args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, text=True) - sub.wait() + sub = self._default_background_popen(cmd) if sub.returncode != 0: self.tue_install_error( - f"An error occurred while installing gem packages({sub.returncode}):\n {' '.join(cmds)}" + f"An error occurred while installing gem packages({sub.returncode}):\n {repr(cmd)}" ) # ToDo: This depends on behaviour of tue-install-error return False From 4e8f187d9a69a05f18a1a46c54ef6bc88242171a Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:20:15 +0100 Subject: [PATCH 13/47] Various improvements to tue-get --- src/tue_get/installer_impl.py | 46 ++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index a690df16b..c9475da19 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union, Tuple +from typing import List, Optional, Tuple from contextlib import contextmanager import datetime @@ -7,6 +7,7 @@ import glob import os from pathlib import Path +import re import shlex import shutil import subprocess as sp @@ -15,6 +16,7 @@ from tue_get.install_yaml_parser import installyaml_parser from tue_get.util.BackgroundPopen import BackgroundPopen +from tue_get.util.grep import grep_directory, grep_file CI = None @@ -37,8 +39,22 @@ def _which_split_cmd(cmd: str) -> Tuple[str, List[str]]: return " ".join(cmds), cmds +def _wait_for_dpkg_lock(): + i = 0 + rotate_list = ["-", "\\", "|", "/"] + cmd = "sudo fuser /var/lib/dpkg/lock" + cmd, cmds = _which_split_cmd(cmd) + while sp.run(cmds, stdout=sp.DEVNULL, stderr=sp.DEVNULL).returncode == 0: + print(f"[{rotate_list[i % len(rotate_list)]}] Waiting for dpkg lock", end="\r") + i += 1 + sleep(0.4) + return + + class InstallerImpl: _apt_get_updated_file = os.path.join(os.sep, "tmp", "tue_get_apt_get_updated") + _sources_list = os.path.join(os.sep, "etc", "apt", "sources.list") + _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") def __init__(self, debug: bool = False): self._debug = debug @@ -105,7 +121,8 @@ def _write_sorted_deps(self, child: str) -> None: self._write_sorted_dep_file(os.path.join(self._dependencies_dir, self._current_target), child) self._write_sorted_dep_file(os.path.join(self._dependencies_on_dir, child), self._current_target) - def _write_sorted_dep_file(self, file: str, target: str) -> None: + @staticmethod + def _write_sorted_dep_file(file: str, target: str) -> None: if not os.path.isfile(file): Path(file).touch(exist_ok=False) targets = [] @@ -129,7 +146,7 @@ def _set_target(self, target: str): self._current_target = parent_target self._current_target_dir = parent_target_dir - def _out_handler(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + def _out_handler(self, sub: BackgroundPopen, line: str) -> None: def _write_stdin(msg) -> None: sub.stdin.write(f"{msg}\n") sub.stdin.flush() @@ -198,10 +215,12 @@ def _write_stdin(msg) -> None: _write_stdin(0) elif line.startswith("tue-install-ppa: "): ppas = line[17:].split() + ppas = [ppa.replace("^", " ").strip() for ppa in ppas] success = self.tue_install_ppa(ppas) _write_stdin(not success) elif line.startswith("tue-install-ppa-now: "): ppas = line[21:].split() + ppas = [ppa.replace("^", " ").strip() for ppa in ppas] success = self.tue_install_ppa_now(ppas) _write_stdin(not success) elif line.startswith("tue-install-pip: "): @@ -231,7 +250,7 @@ def _write_stdin(msg) -> None: else: self.tue_install_tee(line) - def _err_handler(self, sub: BackgroundPopen, line: Union[bytes, str]) -> None: + def _err_handler(self, sub: BackgroundPopen, line: str) -> None: line = line.strip() if line.startswith("[sudo] password for"): if self._sudo_password is None: @@ -412,7 +431,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: else: self.tue_install_debug(f"Sourcing {install_bash_file}") resource_file = os.path.join(os.path.dirname(__file__), "resources", "installer_impl.bash") - cmd = f"bash -c \"source {resource_file} && source {install_bash_file}\"" + cmd = f'bash -c \"source {resource_file} && source {install_bash_file}\"' sub = self._default_background_popen(cmd) if sub.returncode != 0: self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") @@ -601,13 +620,13 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: with open(target_file_path, "r") as f: target_text = f.read().splitlines() - if not begin_tag in target_text: + if begin_tag not in target_text: self.tue_install_debug( f"tue-install-add-text: {begin_tag=} not found in {target_file_path=}, " "appending to {target_file_path}" ) source_text = "\n".join(source_text) - cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path}\"" + cmd = f"bash -c \"echo - e \'{source_text}\' | {sudo_cmd}tee -a {target_file_path}\"" else: self.tue_install_debug( f"tue-install-add-text: {begin_tag=} found in {target_file_path=}, " @@ -705,17 +724,6 @@ def _out_handler_installed_pkgs(sub: BackgroundPopen, line: str) -> None: apt_get_cmd = f"sudo apt-get install --assume-yes -q {' '.join(pkgs_to_install)}" self.tue_install_echo(f"Going to run the following command:\n{apt_get_cmd}") - def _wait_for_dpkg_lock(): - i = 0 - rotate_list = ["-", "\\", "|", "/"] - cmd = "sudo fuser /var/lib/dpkg/lock" - cmd, cmds = _which_split_cmd(cmd) - while sp.run(cmds, stdout=sp.DEVNULL, stderr=sp.DEVNULL).returncode == 0: - print(f"[{rotate_list[i % len(rotate_list)]}] Waiting for dpkg lock", end="\r") - i += 1 - sleep(0.4) - return - _wait_for_dpkg_lock() if not os.path.isfile(self._apt_get_updated_file): @@ -748,7 +756,7 @@ def tue_install_apt_get_update(self): def tue_install_ppa(self, ppas: List[str]) -> bool: self.tue_install_debug(f"tue-install-ppa {ppas=}") if not ppas: - self.tue_install_error("Invalid tue-install-ppa call: needs ppa as argument") + self.tue_install_error("Invalid tue-install-ppa call: needs ppas as argument") # ToDo: This depends on behaviour of tue-install-error return False From da1d3e8de06885566a88f1fbaa9a9702c44249ca Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:20:38 +0100 Subject: [PATCH 14/47] Add working tue-install-ppa-now --- src/tue_get/installer_impl.py | 55 ++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index c9475da19..f8bff20ea 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -431,7 +431,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: else: self.tue_install_debug(f"Sourcing {install_bash_file}") resource_file = os.path.join(os.path.dirname(__file__), "resources", "installer_impl.bash") - cmd = f'bash -c \"source {resource_file} && source {install_bash_file}\"' + cmd = f'bash -c "source {resource_file} && source {install_bash_file}"' sub = self._default_background_popen(cmd) if sub.returncode != 0: self.tue_install_error(f"Error while running({sub.returncode}):\n {repr(cmd)}") @@ -626,7 +626,7 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: "appending to {target_file_path}" ) source_text = "\n".join(source_text) - cmd = f"bash -c \"echo - e \'{source_text}\' | {sudo_cmd}tee -a {target_file_path}\"" + cmd = f"bash -c \"echo - e '{source_text}' | {sudo_cmd}tee -a {target_file_path}\"" else: self.tue_install_debug( f"tue-install-add-text: {begin_tag=} found in {target_file_path=}, " @@ -648,7 +648,7 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: print(target_text) target_text = "\n".join(target_text) - cmd = f"bash -c \"echo -e \'{target_text}\' | {sudo_cmd}tee {target_file_path}\"" + cmd = f"bash -c \"echo -e '{target_text}' | {sudo_cmd}tee {target_file_path}\"" cmd, cmds = _which_split_cmd(cmd) self.tue_install_echo(repr(cmd)) @@ -763,7 +763,54 @@ def tue_install_ppa(self, ppas: List[str]) -> bool: self._ppas.extend(ppas) return True - def tue_install_ppa_now(self, ppa: List[str]) -> bool: + def tue_install_ppa_now(self, ppas: List[str]) -> bool: + self.tue_install_debug(f"tue-install-ppa-now {ppas=}") + + if not ppas: + self.tue_install_error("Invalid tue-install-ppa-now call: needs ppas as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + ppa_added = False + for ppa in ppas: + if not ppa.startswith("ppa:") and not ppa.startswith("deb "): + self.tue_install_error(f"Invalid ppa: needs to start with 'ppa:' or 'deb ' ({ppa=})") + # ToDo: This depends on behaviour of tue-install-error + return False + + if ppa.startswith("ppa:"): + ppa_pattern = f"^deb.*{ppa[4:]}".replace("[", "\[").replace("]", "\]").replace("/", "\/") + ppa_pattern = re.compile(ppa_pattern) + if grep_directory(self._sources_list_dir, ppa_pattern): + self.tue_install_debug(f"PPA '{ppa}' is already added previously") + continue + + elif ppa.startswith("deb "): + ppa_pattern = f"^{ppa}$".replace("[", "\[").replace("]", "\]").replace("/", "\/") + ppa_pattern = re.compile(ppa_pattern) + if grep_file(self._sources_list, ppa_pattern): + self.tue_install_debug(f"PPA '{ppa}' is already added previously") + continue + + self.tue_install_system_now(["software-properties-common"]) + + self.tue_install_info(f"Adding ppa: {ppa}") + + # Wait for apt-lock first (https://askubuntu.com/a/375031) + _wait_for_dpkg_lock() + + cmd = f"sudo add-apt-repository --no-update --yes '{ppa}'" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while adding ppa: {ppa}") + # ToDo: This depends on behaviour of tue-install-error + return False + + ppa_added = True + + if ppa_added: + self.tue_install_apt_get_update() + return True def tue_install_pip(self, pkgs: List[str]) -> bool: From 7bdea751d33a75d32182a1eff39b94c15b9e4a30 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:21:09 +0100 Subject: [PATCH 15/47] Add grep funcs --- src/tue_get/util/__init__.py | 2 + src/tue_get/util/grep.py | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 src/tue_get/util/__init__.py create mode 100644 src/tue_get/util/grep.py diff --git a/src/tue_get/util/__init__.py b/src/tue_get/util/__init__.py new file mode 100644 index 000000000..a384ea7ad --- /dev/null +++ b/src/tue_get/util/__init__.py @@ -0,0 +1,2 @@ +from . import BackgroundPopen +from . import grep diff --git a/src/tue_get/util/grep.py b/src/tue_get/util/grep.py new file mode 100644 index 000000000..42a9a8bae --- /dev/null +++ b/src/tue_get/util/grep.py @@ -0,0 +1,71 @@ +from typing import List +import os +import re + +text_characters = b"".join(map(lambda x: bytes((x,)), range(32, 127))) + b"\n\r\t\f\b" +_null_trans = bytes.maketrans(b"", b"") + + +def istextfile(filename: str, blocksize: int = 512) -> bool: + return istext(open(filename, "rb").read(blocksize)) + + +def istext(b: bytes) -> bool: + if b"\0" in b: + return False + + if not b: # Empty files are considered text + return True + + # Get the non-text characters (maps a character to itself then + # use the 'remove' option to get rid of the text characters.) + t = b.translate(_null_trans, text_characters) + + # If more than 30% non-text characters, then + # this is considered a binary file + if len(t) / len(b) > 0.30: + return False + return True + + +def grep_directory( + directory: str, pattern: re.Pattern, recursive: bool = False, include_binary: bool = False +) -> List[str]: + """ + Searches for a regex in a directory. + + :param directory: The directory to search in. + :param pattern: The regex to search with. + :param recursive: (optional) Whether to search recursively. Defaults to False. + :param include_binary: (optional) Whether to include binary files. Defaults to False. + + :return: A list of files that match the regex. + """ + files = [] + for root, dirs, filenames in os.walk(directory): + for filename in filenames: + full_path = os.path.join(root, filename) + if not include_binary and not istextfile(full_path): + continue + if grep_file(full_path, pattern): + files.append(full_path) + if not recursive: + break + return files + + +def grep_file(file: str, pattern: re.Pattern) -> bool: + """ + Searches for a regex in a file. + + :param file: The file to search in. + :param pattern: The regex to search with. + + :return: Whether the regex was found in the file. + """ + with open(file, "r") as f: + lines = f.readlines() + for line in lines: + if pattern.search(line): + return True + return False From d13e0e6676b8db40a62e4bf2532ab7c1892e99b7 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:43:28 +0100 Subject: [PATCH 16/47] Add working tue-install-snap-now --- src/tue_get/installer_impl.py | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index f8bff20ea..4972c8a41 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -837,6 +837,48 @@ def tue_install_snap(self, pkgs: List[str]) -> bool: return True def tue_install_snap_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-snap-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-snap-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_system_now(["snapd"]) + + cmd = "snap list" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_echo(repr(cmd)) + sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) + sub.wait() + if sub.returncode != 0: + self.tue_install_error(f"Error while getting snap list({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + snaps_installed = sub.stdout.read().splitlines()[1:] + snaps_installed = [snap.split()[0] for snap in snaps_installed] + snaps_to_install = [] + for pkg in pkgs: + if pkg not in snaps_installed: + snaps_to_install.append(pkg) + self.tue_install_debug(f"Snap '{pkg}' is not installed yet") + else: + self.tue_install_debug(f"Snap '{pkg}' is already installed") + + if not snaps_to_install: + self.tue_install_debug("No snaps to install") + return True + + for pkg in snaps_to_install: + cmd = f"sudo snap install --classic {pkg}" + sub = self._default_background_popen(cmd) + breakpoint() + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while installing snap: {pkg}") + # ToDo: This depends on behaviour of tue-install-error + return False + return True def tue_install_gem(self, pkgs: List[str]) -> bool: From 2498ace8c17f72cc4c458f514653102962236600 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:48:57 +0100 Subject: [PATCH 17/47] Add ?working? tue-install-dpkg-now --- src/tue_get/installer_impl.py | 37 ++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 4972c8a41..60106fc35 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -933,12 +933,43 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: return True - def tue_install_dpkg(self, pkg_file: str) -> bool: - return True - def tue_install_dpkg_now(self, pkg_file: str) -> bool: + self.tue_install_debug(f"tue-install-dpkg-now {pkg_file=}") + + if not pkg_file: + self.tue_install_error("Invalid tue-install-dpkg-now call: got an empty package file as argument.") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isabs(pkg_file): + pkg_file = os.path.join(self._current_target_dir, pkg_file) + + if not os.path.isfile(pkg_file): + self.tue_install_error(f"Invalid tue-install-dpkg-now call: {pkg_file} is not a file.") + # ToDo: This depends on behaviour of tue-install-error + return False + + cmd = f"sudo dpkg -i {pkg_file}" + sub = self._default_background_popen(cmd) + # ToDo: Should we check the return code? + # if sub.returncode != 0: + # self.tue_install_error(f"An error occurred while installing dpkg package({sub.returncode}):" + # f"\n {repr(cmd)}") + # # ToDo: This depends on behaviour of tue-install-error + # return False + + cmd = f"sudo apt-get install -f" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"An error occurred while fixing dpkg isntall({sub.returncode}):" + f"\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + return True + tue_install_dpkg = tue_install_dpkg_now + def tue_install_ros(self, source_type: str, **kwargs) -> bool: return True From c028f01ea06cd5fd900def3404aa86b07e1769c9 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 20:51:01 +0100 Subject: [PATCH 18/47] Add inspired by reference for istext funcs --- src/tue_get/util/grep.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tue_get/util/grep.py b/src/tue_get/util/grep.py index 42a9a8bae..34d7aee84 100644 --- a/src/tue_get/util/grep.py +++ b/src/tue_get/util/grep.py @@ -2,6 +2,10 @@ import os import re +""" +Text functions inspired by https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python +""" + text_characters = b"".join(map(lambda x: bytes((x,)), range(32, 127))) + b"\n\r\t\f\b" _null_trans = bytes.maketrans(b"", b"") From 3664f65c5e0c085602ce6fea4e3c79984d1cdc8c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 21:35:29 +0100 Subject: [PATCH 19/47] Add working tue-install-pip-now --- src/tue_get/installer_impl.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 60106fc35..bfb207384 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -7,6 +7,8 @@ import glob import os from pathlib import Path +from pip import main as pip_main +from pip._internal.req.constructors import install_req_from_line as pip_install_req_from_line import re import shlex import shutil @@ -824,6 +826,43 @@ def tue_install_pip(self, pkgs: List[str]) -> bool: return True def tue_install_pip_now(self, pkgs: List[str]) -> bool: + self.tue_install_debug(f"tue-install-pip-now {pkgs=}") + + if not pkgs: + self.tue_install_error("Invalid tue-install-pip-now call: needs packages as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + pips_to_install = [] + git_pips_to_install = [] + for pkg in pkgs: + if pkg.startswith("git+"): + git_pips_to_install.append(pkg) + continue + + req = pip_install_req_from_line(pkg) + req.check_if_exists(use_user_site=True) + + if req.satisfied_by: + self.tue_install_debug(f"{pkg} is already installed, {req.satisfied_by}") + else: + pips_to_install.append(pkg) + + if pips_to_install: + returncode = pip_main(["install", "--user", *pips_to_install]) + if returncode != 0: + self.tue_install_error(f"An error occurred while installing pip packages: {pips_to_install}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if git_pips_to_install: + for pkg in git_pips_to_install: + returncode = pip_main(["install", "--user", pkg]) + if returncode != 0: + self.tue_install_error(f"An error occurred while installing pip packages: {pkg}") + # ToDo: This depends on behaviour of tue-install-error + return False + return True def tue_install_snap(self, pkgs: List[str]) -> bool: From e8d5cfc06d12e6ac0ef642a400cce11c8e074240 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 21:35:48 +0100 Subject: [PATCH 20/47] Apply black --- src/tue_get/installer_impl.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index bfb207384..f3b5bf6c4 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1000,8 +1000,7 @@ def tue_install_dpkg_now(self, pkg_file: str) -> bool: cmd = f"sudo apt-get install -f" sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"An error occurred while fixing dpkg isntall({sub.returncode}):" - f"\n {repr(cmd)}") + self.tue_install_error(f"An error occurred while fixing dpkg isntall({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False From 29821bdafb5988143f0139a206099c5ba8e32c12 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 22:31:12 +0100 Subject: [PATCH 21/47] Fix yaml parser --- src/tue_get/install_yaml_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tue_get/install_yaml_parser.py b/src/tue_get/install_yaml_parser.py index 845bf8f9c..61678a822 100644 --- a/src/tue_get/install_yaml_parser.py +++ b/src/tue_get/install_yaml_parser.py @@ -133,10 +133,10 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type: source_type = source["type"] if source_type == "git": - command = partial(getattr(installer, "tue_install_ros"), source_type="git", **catkin_git(source)) + command = partial(getattr(installer, "tue_install_ros"), **catkin_git(source)) elif source_type == "system": system_packages.append(f"ros-{ros_release}-{source['name']}") - command = partial(getattr(installer, "tue_install_ros"), source_type="system", **catkin_git(source)) + command = partial(getattr(installer, "tue_install_ros"), source_type="system", name=source["name"]) else: raise ValueError(f"Unknown ROS install type: '{source_type}'") From 1608c3fb289c719c1d528ed35f587863d8560110 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 22:31:40 +0100 Subject: [PATCH 22/47] WIP tue-install-ros --- src/tue_get/installer_impl.py | 160 +++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index f3b5bf6c4..bddb6ba93 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1009,7 +1009,165 @@ def tue_install_dpkg_now(self, pkg_file: str) -> bool: tue_install_dpkg = tue_install_dpkg_now def tue_install_ros(self, source_type: str, **kwargs) -> bool: - return True + self.tue_install_debug(f"tue-install-ros {source_type=} {kwargs=}") + + if not source_type: + self.tue_install_error("Invalid tue-install-ros call: needs source type as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + TUE_ROS_DISTRO = os.getenv("TUE_ROS_DISTRO", None) + if not TUE_ROS_DISTRO: + self.tue_install_error("TUE_ROS_DISTRO is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + TUE_ROS_VERSION = os.getenv("TUE_ROS_VERSION", None) + if not TUE_ROS_VERSION: + self.tue_install_error("TUE_ROS_VERSION is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + + ros_pkg_name = self._current_target.lstrip("ros-") + if "-" in ros_pkg_name: + correct_ros_pkg_name = ros_pkg_name.replace("-", "_") + self.tue_install_error(f"A ROS package cannot contain dashes ({ros_pkg_name}), " + f"make sure the package is named '{correct_ros_pkg_name}' and rename the " + f"target to 'ros-{correct_ros_pkg_name}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + # First of all, make sure ROS itself is installed + if not self.tue_install_target(f"ros{TUE_ROS_VERSION}"): + self.tue_install_error(f"Failed to install ros{TUE_ROS_VERSION}") + + if source_type == "system": + """ + Install a ROS package from the system repository + kwargs: + - name: The name of the package to install + """ + name = kwargs["name"] + if name is None: + self.tue_install_error("Invalid tue-install-ros call(system): needs src as argument") + self.tue_install_debug(f"tue-install-system ros-{TUE_ROS_DISTRO}-{src}") + if not self.tue_install_system(["ros-{TUE_ROS_DISTRO}-{src}"]): + self.tue_install_error(f"Failed to append ros-{TUE_ROS_DISTRO}-{src}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + + if source_type != "git": + self.tue_install_error(f"Unknown ROS source type: {source_type}") + # ToDo: This depends on behaviour of tue-install-error + return False + + """ + Install a ROS package from a git repository + kwargs: + - url: The git repository to clone + - sub_dir (optional): The subdirectory of the repository to install + - version (optional): The version of the package to install + - target_dir (optional): The directory to install the package to + """ + + url = kwargs["url"] + if url is None: + self.tue_install_error("Invalid tue-install-ros call(git): needs url as argument") + + sub_dir = kwargs["sub_dir"] + version = kwargs["version"] + target_dir = kwargs["target_dir"] + + TUE_SYSTEM_DIR = os.getenv("TUE_SYSTEM_DIR", None) + if not TUE_SYSTEM_DIR: + self.tue_install_error("ROS_PACKAGE_INSTALL_DIR is not set") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Make sure the ROS package install dir exists + ROS_PACKAGE_INSTALL_DIR = os.path.join(TUE_SYSTEM_DIR, "src") + if not os.path.isdir(ROS_PACKAGE_INSTALL_DIR): + self.tue_install_error(f"ROS_PACKAGE_INSTALL_DIR is not a directory: {ROS_PACKAGE_INSTALL_DIR}") + # ToDo: This depends on behaviour of tue-install-error + return False + + + + # If repos_dir is not set, try generating the default path from git url + if target_dir is None: + # ToDo: convert _git_url_to_repos_dir to python + target_dir = sp.check_output(f"bash -c '_git_url_to_repos_dir {url}'", text=True).strip() + if not target_dir: + self.tue_install_error(f"Could not create target_dir path from the git url: '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"{target_dir=}") + + ros_pkg_dir = os.path.join(ROS_PACKAGE_INSTALL_DIR, ros_pkg_name) + + if not self.tue_install_git(url, target_dir, version): + self.tue_install_error(f"Failed to clone ros package '{ros_pkg_name}' from '{url}'") + + if not os.path.isdir(target_dir): + if not os.path.isdir(os.path.join(target_dir, sub_dir)): + self.tue_install_error(f"Subdirectory '{sub_dir}' does not exist for url '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Test if the current symbolic link points to the same repository dir. If not, give a warning + # because it means the source URL has changed + if os.path.islink(ros_pkg_dir): + if os.path.realpath(ros_pkg_dir) != os.path.realpath(os.path.join(target_dir, sub_dir)): + self.tue_install_info(f"url has changed to {url}/{sub_dir}") + os.remove(ros_pkg_dir) + os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + elif os.path.isdir(ros_pkg_dir): + self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") + # ToDo: This depends on behaviour of tue-install-error + return False + elif not os.path.exists(ros_pkg_dir): + os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + else: + self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " + "Aany other option is incorrect") + + +if [["$TUE_INSTALL_SKIP_ROS_DEPS" != "all"]] +then +local +pkg_xml = "$ros_pkg_dir" / package.xml +if [-f "$pkg_xml"] +then +# Catkin +tue - install - debug +"Parsing $pkg_xml" +local +deps +deps =$("$TUE_INSTALL_SCRIPTS_DIR" / parse_package_xml.py "$pkg_xml") +tue - install - debug +"Parsed package.xml\n$deps" + +for dep in $deps +do +# Preference given to target name starting with ros- +tue-install-target ros-"$dep" | | tue-install-target "$dep" | | \ + tue-install-error "Targets 'ros-$dep' and '$dep' don't exist" +done + +else +tue-install-warning "Does not contain a valid ROS package.xml" +fi +else +tue-install-debug "No need to parse package.xml for dependencies" +fi + +else +tue-install-error "Checking out $src was not successful." +fi + +TUE_INSTALL_PKG_DIR=$ros_pkg_dir if __name__ == "__main__": From b473db09466377fc970093e2bc3a250f13f97b6c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sun, 8 Jan 2023 22:31:55 +0100 Subject: [PATCH 23/47] Revert back to test target --- src/tue_get/installer_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index bddb6ba93..40cce0a97 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1172,4 +1172,4 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: if __name__ == "__main__": bla = InstallerImpl(debug=True) - bla.tue_install_target("networking") + bla.tue_install_target("test", True) From 928cba21816bda231569f4e858a6d412c0f2f036 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 12:18:42 +0100 Subject: [PATCH 24/47] Working tue-install-ros --- src/tue_get/__init__.py | 1 + src/tue_get/catkin_package_parser.py | 48 ++++++++ src/tue_get/installer_impl.py | 164 +++++++++++++++------------ 3 files changed, 139 insertions(+), 74 deletions(-) create mode 100644 src/tue_get/catkin_package_parser.py diff --git a/src/tue_get/__init__.py b/src/tue_get/__init__.py index ccd84a56a..1998250a4 100644 --- a/src/tue_get/__init__.py +++ b/src/tue_get/__init__.py @@ -1,3 +1,4 @@ +from . import catkin_package_parser from . import install_yaml_parser from . import installer_impl from . import util diff --git a/src/tue_get/catkin_package_parser.py b/src/tue_get/catkin_package_parser.py new file mode 100644 index 000000000..0723d96c0 --- /dev/null +++ b/src/tue_get/catkin_package_parser.py @@ -0,0 +1,48 @@ +from typing import List, Mapping, Optional, Tuple + +from catkin_pkg.package import Dependency, Package, parse_package +import os + + +def catkin_package_parser( + path: str, + skip_normal_deps: bool = False, + test_deps: bool = False, + doc_deps: bool = False, + warnings: Optional[List[str]] = None, + context: Optional[Mapping] = None, +) -> Tuple[Package, List[Dependency]]: + + if context is None: + context = os.environ + dep_types = [] + if not skip_normal_deps: + dep_types.extend( + [ + "build_depends", + "buildtool_depends", + "build_export_depends", + "buildtool_export_depends", + "exec_depends", + ] + ) + + if test_deps: + dep_types.append("test_depends") + + if doc_deps: + dep_types.append("doc_depends") + + pkg = parse_package(path, warnings=warnings) + pkg.evaluate_conditions(context) + + deps = [] + for dep_type in dep_types: + dep_list = getattr(pkg, dep_type) + for dep in dep_list: # type: Dependency + if dep.evaluated_condition is None: + raise RuntimeError("Dependency condition is None") + if dep.evaluated_condition: + deps.append(dep) + + return pkg, deps diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 40cce0a97..a1e392903 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1,5 +1,6 @@ from typing import List, Optional, Tuple +from catkin_pkg.package import PACKAGE_MANIFEST_FILENAME, InvalidPackage from contextlib import contextmanager import datetime import filecmp @@ -16,6 +17,7 @@ from termcolor import colored from time import sleep +from tue_get.catkin_package_parser import catkin_package_parser from tue_get.install_yaml_parser import installyaml_parser from tue_get.util.BackgroundPopen import BackgroundPopen from tue_get.util.grep import grep_directory, grep_file @@ -58,7 +60,10 @@ class InstallerImpl: _sources_list = os.path.join(os.sep, "etc", "apt", "sources.list") _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") - def __init__(self, debug: bool = False): + def __init__(self, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False): + self._ros_test_deps = ros_test_deps + self._ros_doc_deps = ros_doc_deps + self._skip_ros_deps = skip_ros_deps self._debug = debug self._current_target = "main-loop" self._current_target_dir = "" @@ -1016,14 +1021,14 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - TUE_ROS_DISTRO = os.getenv("TUE_ROS_DISTRO", None) - if not TUE_ROS_DISTRO: - self.tue_install_error("TUE_ROS_DISTRO is not set") + tue_ros_distro = os.getenv("tue_ros_distro", None) + if not tue_ros_distro: + self.tue_install_error("tue_ros_distro is not set") # ToDo: This depends on behaviour of tue-install-error return False - TUE_ROS_VERSION = os.getenv("TUE_ROS_VERSION", None) - if not TUE_ROS_VERSION: - self.tue_install_error("TUE_ROS_VERSION is not set") + tue_ros_version = os.getenv("tue_ros_version", None) + if not tue_ros_version: + self.tue_install_error("tue_ros_version is not set") # ToDo: This depends on behaviour of tue-install-error return False @@ -1037,8 +1042,8 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: return False # First of all, make sure ROS itself is installed - if not self.tue_install_target(f"ros{TUE_ROS_VERSION}"): - self.tue_install_error(f"Failed to install ros{TUE_ROS_VERSION}") + if not self.tue_install_target(f"ros{tue_ros_version}"): + self.tue_install_error(f"Failed to install ros{tue_ros_version}") if source_type == "system": """ @@ -1048,10 +1053,13 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: """ name = kwargs["name"] if name is None: - self.tue_install_error("Invalid tue-install-ros call(system): needs src as argument") - self.tue_install_debug(f"tue-install-system ros-{TUE_ROS_DISTRO}-{src}") - if not self.tue_install_system(["ros-{TUE_ROS_DISTRO}-{src}"]): - self.tue_install_error(f"Failed to append ros-{TUE_ROS_DISTRO}-{src}") + self.tue_install_error("Invalid tue-install-ros call(system): needs 'name' as argument") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_debug(f"tue-install-system ros-{tue_ros_distro}-{name}") + if not self.tue_install_system(["ros-{tue_ros_distro}-{src}"]): + self.tue_install_error(f"Failed to append ros-{tue_ros_distro}-{name}") # ToDo: This depends on behaviour of tue-install-error return False @@ -1079,16 +1087,16 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: version = kwargs["version"] target_dir = kwargs["target_dir"] - TUE_SYSTEM_DIR = os.getenv("TUE_SYSTEM_DIR", None) - if not TUE_SYSTEM_DIR: - self.tue_install_error("ROS_PACKAGE_INSTALL_DIR is not set") + tue_system_dir = os.getenv("tue_system_dir", None) + if not tue_system_dir: + self.tue_install_error("ros_package_install_dir is not set") # ToDo: This depends on behaviour of tue-install-error return False # Make sure the ROS package install dir exists - ROS_PACKAGE_INSTALL_DIR = os.path.join(TUE_SYSTEM_DIR, "src") - if not os.path.isdir(ROS_PACKAGE_INSTALL_DIR): - self.tue_install_error(f"ROS_PACKAGE_INSTALL_DIR is not a directory: {ROS_PACKAGE_INSTALL_DIR}") + ros_package_install_dir = os.path.join(tue_system_dir, "src") + if not os.path.isdir(ros_package_install_dir): + self.tue_install_error(f"ros_package_install_dir is not a directory: {ros_package_install_dir}") # ToDo: This depends on behaviour of tue-install-error return False @@ -1105,69 +1113,77 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: self.tue_install_debug(f"{target_dir=}") - ros_pkg_dir = os.path.join(ROS_PACKAGE_INSTALL_DIR, ros_pkg_name) + ros_pkg_dir = os.path.join(ros_package_install_dir, ros_pkg_name) if not self.tue_install_git(url, target_dir, version): self.tue_install_error(f"Failed to clone ros package '{ros_pkg_name}' from '{url}'") if not os.path.isdir(target_dir): - if not os.path.isdir(os.path.join(target_dir, sub_dir)): - self.tue_install_error(f"Subdirectory '{sub_dir}' does not exist for url '{url}'") + self.tue_install_error(f"ROS package '{ros_pkg_name}' from '{url}' was not cloned to '{target_dir}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isdir(os.path.join(target_dir, sub_dir)): + self.tue_install_error(f"Subdirectory '{sub_dir}' does not exist for url '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + # Test if the current symbolic link points to the same repository dir. If not, give a warning + # because it means the source URL has changed + if os.path.islink(ros_pkg_dir): + if os.path.realpath(ros_pkg_dir) != os.path.realpath(os.path.join(target_dir, sub_dir)): + self.tue_install_info(f"url has changed to {url}/{sub_dir}") + os.remove(ros_pkg_dir) + os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + elif os.path.isdir(ros_pkg_dir): + self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") # ToDo: This depends on behaviour of tue-install-error return False + elif not os.path.exists(ros_pkg_dir): + os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + else: + self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " + "Any other option is incorrect") - # Test if the current symbolic link points to the same repository dir. If not, give a warning - # because it means the source URL has changed - if os.path.islink(ros_pkg_dir): - if os.path.realpath(ros_pkg_dir) != os.path.realpath(os.path.join(target_dir, sub_dir)): - self.tue_install_info(f"url has changed to {url}/{sub_dir}") - os.remove(ros_pkg_dir) - os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) - elif os.path.isdir(ros_pkg_dir): - self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") - # ToDo: This depends on behaviour of tue-install-error - return False - elif not os.path.exists(ros_pkg_dir): - os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) - else: - self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " - "Aany other option is incorrect") - - -if [["$TUE_INSTALL_SKIP_ROS_DEPS" != "all"]] -then -local -pkg_xml = "$ros_pkg_dir" / package.xml -if [-f "$pkg_xml"] -then -# Catkin -tue - install - debug -"Parsing $pkg_xml" -local -deps -deps =$("$TUE_INSTALL_SCRIPTS_DIR" / parse_package_xml.py "$pkg_xml") -tue - install - debug -"Parsed package.xml\n$deps" - -for dep in $deps -do -# Preference given to target name starting with ros- -tue-install-target ros-"$dep" | | tue-install-target "$dep" | | \ - tue-install-error "Targets 'ros-$dep' and '$dep' don't exist" -done - -else -tue-install-warning "Does not contain a valid ROS package.xml" -fi -else -tue-install-debug "No need to parse package.xml for dependencies" -fi - -else -tue-install-error "Checking out $src was not successful." -fi - -TUE_INSTALL_PKG_DIR=$ros_pkg_dir + if self._skip_ros_deps and not self._ros_test_deps and not self._ros_doc_deps: + self.tue_install_debug("Skipping resolving of ROS dependencies") + return True + + # Resolve ROS dependencies + pkg_xml = os.path.join(ros_pkg_dir, PACKAGE_MANIFEST_FILENAME) + if os.path.isfile(pkg_xml): + self.tue_install_warning(f"Does not contain a valid ROS {PACKAGE_MANIFEST_FILENAME}") + return True + + self.tue_install_debug(f"Parsing {pkg_xml}") + try: + _, deps = catkin_package_parser(pkg_xml, self._skip_ros_deps, self._ros_test_deps, self._ros_doc_deps) + except IOError as e: + self.tue_install_error(f"Could not parse {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except InvalidPackage as e: + self.tue_install_error(f"Invalid {PACKAGE_MANIFEST_FILENAME}:\n{e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except ValueError as e: + self.tue_install_error(f"Could not evaluate conditions in {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + except RuntimeError as e: + self.tue_install_error(f"Unevaluated condition found in {pkg_xml}: {e}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for dep in deps: + success = self.tue_install_target(f"ros-{dep.name}") or self.tue_install_target(dep.name) + if not success: + self.tue_install_error(f"Targets 'ros-{dep.name}' and '{dep}' don't exist") + # ToDo: This depends on behaviour of tue-install-error + return False + + # ToDO: TUE_INSTALL_PKG_DIR was set ros_pkg_dir which was then use in tue-install-apply-patch; we are not doing that not (yet) in python + return True if __name__ == "__main__": From 7aa35b7962956a6957ae455d41692f31effbadd2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 16:16:29 +0100 Subject: [PATCH 25/47] Remove temp prints --- src/tue_get/installer_impl.py | 34 +++++++++++++---------- src/tue_get/resources/installer_impl.bash | 2 -- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index a1e392903..79a493b7d 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1021,14 +1021,14 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - tue_ros_distro = os.getenv("tue_ros_distro", None) + tue_ros_distro = os.getenv("TUE_ROS_DISTRO", None) if not tue_ros_distro: - self.tue_install_error("tue_ros_distro is not set") + self.tue_install_error("TUE_ROS_DISTRO is not set") # ToDo: This depends on behaviour of tue-install-error return False - tue_ros_version = os.getenv("tue_ros_version", None) + tue_ros_version = os.getenv("TUE_ROS_VERSION", None) if not tue_ros_version: - self.tue_install_error("tue_ros_version is not set") + self.tue_install_error("TUE_ROS_VERSION is not set") # ToDo: This depends on behaviour of tue-install-error return False @@ -1084,10 +1084,12 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: self.tue_install_error("Invalid tue-install-ros call(git): needs url as argument") sub_dir = kwargs["sub_dir"] + if sub_dir is None: + sub_dir = "" version = kwargs["version"] target_dir = kwargs["target_dir"] - tue_system_dir = os.getenv("tue_system_dir", None) + tue_system_dir = os.getenv("TUE_SYSTEM_DIR", None) if not tue_system_dir: self.tue_install_error("ros_package_install_dir is not set") # ToDo: This depends on behaviour of tue-install-error @@ -1105,7 +1107,9 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # If repos_dir is not set, try generating the default path from git url if target_dir is None: # ToDo: convert _git_url_to_repos_dir to python - target_dir = sp.check_output(f"bash -c '_git_url_to_repos_dir {url}'", text=True).strip() + cmd = f"bash -c '_git_url_to_repos_dir {url}'" + cmd, cmds = _which_split_cmd(cmd) + target_dir = sp.check_output(cmds, text=True).strip() if not target_dir: self.tue_install_error(f"Could not create target_dir path from the git url: '{url}'") # ToDo: This depends on behaviour of tue-install-error @@ -1135,15 +1139,15 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: self.tue_install_info(f"url has changed to {url}/{sub_dir}") os.remove(ros_pkg_dir) os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) - elif os.path.isdir(ros_pkg_dir): - self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") - # ToDo: This depends on behaviour of tue-install-error - return False - elif not os.path.exists(ros_pkg_dir): - os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) - else: - self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " - "Any other option is incorrect") + elif os.path.isdir(ros_pkg_dir): + self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") + # ToDo: This depends on behaviour of tue-install-error + return False + elif not os.path.exists(ros_pkg_dir): + os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + else: + self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " + "Any other option is incorrect") if self._skip_ros_deps and not self._ros_test_deps and not self._ros_doc_deps: self.tue_install_debug("Skipping resolving of ROS dependencies") diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index 38b20d5a8..972adac52 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -71,10 +71,8 @@ function tue-install-pipe read -r return_value if [ "$return_value" != "0" ] then - tue-install-echo "return_value: $return_value" return $(("$return_value")) fi - tue-install-echo "return_code: $return_code" return $return_code } From 8baee29c105f272cf6381ca10bf0b8be5a4883e5 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 16:21:18 +0100 Subject: [PATCH 26/47] Add tue-install-pip3(-now) for bash scripts For backward compatibility --- src/tue_get/resources/installer_impl.bash | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index 972adac52..948580736 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -180,6 +180,13 @@ function tue-install-pip return $(("$return_value")) } +# TEMP for backward compatibility +function tue-install-pip3 +{ + tue-install-pip "$*" + return $? +} + function tue-install-pip-now { echo -e "tue-install-pip-now: $*" @@ -188,6 +195,13 @@ function tue-install-pip-now return $(("$return_value")) } +# TEMP for backward compatibility +function tue-install-pip3-now +{ + tue-install-pip-now "$*" + return $? +} + function tue-install-snap { echo -e "tue-install-snap: $*" From 2aa7977dd46a10016aae4e6601c37a4bf3350c43 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:04:26 +0100 Subject: [PATCH 27/47] Handle multiline bash logs correctly --- src/tue_get/installer_impl.py | 17 ++++++++++------- src/tue_get/resources/installer_impl.bash | 12 ++++++------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 79a493b7d..d22d255e4 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -158,6 +158,9 @@ def _write_stdin(msg) -> None: sub.stdin.write(f"{msg}\n") sub.stdin.flush() + def _clean_output(msg: str) -> str: + return msg.replace("^", "\n").strip() + line = line.strip() # ToDo: use regex to get attributes of installer if line.startswith("[sudo] password for"): @@ -165,27 +168,27 @@ def _write_stdin(msg) -> None: self._sudo_password = getpass.getpass(line) _write_stdin(self._sudo_password) elif line.startswith("tue-install-error: "): - self.tue_install_error(line[19:]) + self.tue_install_error(_clean_output(line[19:])) _write_stdin(1) # ToDo: or should we call sub.kill() here? elif line.startswith("tue-install-warning: "): - self.tue_install_warning(line[21:]) + self.tue_install_warning(_clean_output(line[21:])) _write_stdin(0) elif line.startswith("tue-install-info: "): - self.tue_install_info(line[18:]) + self.tue_install_info(_clean_output(line[18:])) _write_stdin(0) elif line.startswith("tue-install-debug: "): - self.tue_install_debug(line[19:]) + self.tue_install_debug(_clean_output(line[19:])) _write_stdin(0) elif line.startswith("tue-install-echo: "): - self.tue_install_echo(line[18:]) + self.tue_install_echo(_clean_output(line[18:])) _write_stdin(0) elif line.startswith("tue-install-tee: "): - self.tue_install_tee(line[17:]) + self.tue_install_tee(_clean_output(line[17:])) _write_stdin(0) elif line.startswith("tue-install-pipe: "): output = line[18:].split("^^^") - output = map(lambda x: x.replace("^", "\n").strip(), output) + output = map(_clean_output, output) self.tue_install_pipe(*output) _write_stdin(0) elif line.startswith("tue-install-target-now: "): diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index 948580736..b1dec96c9 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -2,7 +2,7 @@ function tue-install-error { - echo -e "tue-install-error: $*" + echo -e "tue-install-error: $(echo -e "$*" | tr '\n' '^')" local return_value read -r return_value return $(("$return_value")) @@ -10,7 +10,7 @@ function tue-install-error function tue-install-warning { - echo -e "tue-install-warning: $*" + echo -e "tue-install-warning: $(echo -e "$*" | tr '\n' '^')" local return_value read -r return_value return $(("$return_value")) @@ -18,7 +18,7 @@ function tue-install-warning function tue-install-info { - echo -e "tue-install-info: $*" + echo -e "tue-install-info: $(echo -e "$*" | tr '\n' '^')" local return_value read -r return_value return $(("$return_value")) @@ -26,7 +26,7 @@ function tue-install-info function tue-install-debug { - echo -e "tue-install-debug: $*" + echo -e "tue-install-debug: $(echo -e "$*" | tr '\n' '^')" local return_value read -r return_value return $(("$return_value")) @@ -34,7 +34,7 @@ function tue-install-debug function tue-install-echo { - echo -e "tue-install-echo: $*" + echo -e "tue-install-echo: $(echo -e "$*" | tr '\n' '^')" local return_value read -r return_value return $(("$return_value")) @@ -42,7 +42,7 @@ function tue-install-echo function tue-install-tee { - echo -e "tue-install-tee: $*" + echo -e "tue-install-tee: $(echo -e "$*" | tr '\n' '^')}" local return_value read -r return_value return $(("$return_value")) From 779d8e63bd26cc7d91f0947d652e19391999fe1b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:19:34 +0100 Subject: [PATCH 28/47] Various small bug fixes --- src/tue_get/installer_impl.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index d22d255e4..849959c7e 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -132,16 +132,15 @@ def _write_sorted_deps(self, child: str) -> None: def _write_sorted_dep_file(file: str, target: str) -> None: if not os.path.isfile(file): Path(file).touch(exist_ok=False) - targets = [] + targets = set() else: - with open(file, "r+") as f: - targets = f.readlines() + with open(file, "r") as f: + targets = set(f.read().splitlines()) - print(f"{targets=}") - targets.append(target + "\n") - targets.sort() + targets.add(target) with open(file, "w") as f: - f.writelines(targets) + for target in sorted(targets): + f.write(f"{target}\n") @contextmanager def _set_target(self, target: str): @@ -354,8 +353,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: self._write_sorted_deps(target) with self._set_target(target): - - state_file = os.path.join(self._state_dir, "target") + state_file = os.path.join(self._state_dir, target) state_file_now = f"{state_file}-now" # Determine if this target needs to be executed @@ -461,6 +459,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: Path(state_file_to_touch).touch() self.tue_install_debug(f"Finished installing {target}") + return True def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None) -> bool: pass @@ -566,6 +565,8 @@ def tue_install_cp(self, source: str, target: str) -> bool: # ToDo: This depends on behaviour of tue-install-error return False + return True + def tue_install_add_text(self, source_file: str, target_file: str) -> bool: self.tue_install_debug(f"tue-install-add-text {source_file=} {target_file=}") @@ -697,7 +698,7 @@ def tue_install_system_now(self, pkgs: List[str]) -> bool: installed_pkgs = [] - def _out_handler_installed_pkgs(sub: BackgroundPopen, line: str) -> None: + def _out_handler_installed_pkgs(_: BackgroundPopen, line: str) -> None: installed_pkgs.append(line.strip()) # Check if pkg is not already installed dpkg -S does not cover previously removed packages @@ -997,7 +998,7 @@ def tue_install_dpkg_now(self, pkg_file: str) -> bool: return False cmd = f"sudo dpkg -i {pkg_file}" - sub = self._default_background_popen(cmd) + _ = self._default_background_popen(cmd) # ToDo: Should we check the return code? # if sub.returncode != 0: # self.tue_install_error(f"An error occurred while installing dpkg package({sub.returncode}):" @@ -1008,7 +1009,7 @@ def tue_install_dpkg_now(self, pkg_file: str) -> bool: cmd = f"sudo apt-get install -f" sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"An error occurred while fixing dpkg isntall({sub.returncode}):\n {repr(cmd)}") + self.tue_install_error(f"An error occurred while fixing dpkg install({sub.returncode}):\n {repr(cmd)}") # ToDo: This depends on behaviour of tue-install-error return False From de5e76fa56bbab8bae82fbac0322f21db217646b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:20:44 +0100 Subject: [PATCH 29/47] Temp prints as dummy funcs --- src/tue_get/installer_impl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 849959c7e..90976f16b 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -462,7 +462,8 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: return True def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None) -> bool: - pass + self.tue_install_debug(f"tue-install-git {url=} {target_dir=} {version=}") + return True def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: self.tue_install_debug(f"tue-install-apply-patch {patch_file=} {target_dir=}") @@ -677,6 +678,7 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: return False def tue_install_get_releases(self, url: str, filename: str, output_dir: str, tag: Optional[str] = None) -> bool: + self.tue_install_debug(f"tue-install-get-releases {url=} {filename=} {output_dir=} {tag=}") return True def tue_install_system(self, pkgs: List[str]) -> bool: From 269f9db70f09cf0a92cb36bbf32e8f45045713fc Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:21:31 +0100 Subject: [PATCH 30/47] Let tue-install-ppa do the string magic --- src/tue_get/resources/installer_impl.bash | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index b1dec96c9..709504b8b 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -158,7 +158,12 @@ function tue-install-apt-get-update function tue-install-ppa { - echo -e "tue-install-ppa: $*" + local ppa_list + for ppa in "$@" + do + ppa_list="${ppa_list:+${ppa_list} }${ppa// /^}" + done + echo -e "tue-install-ppa: ${ppa_list}" local return_value read -r return_value return $(("$return_value")) @@ -166,7 +171,12 @@ function tue-install-ppa function tue-install-ppa-now { - echo -e "tue-install-ppa-now: $*" + local ppa_list + for ppa in "$@" + do + ppa_list="${ppa_list:+${ppa_list} }${ppa// /^}" + done + echo -e "tue-install-ppa-now: $ppa_list" local return_value read -r return_value return $(("$return_value")) From 0a9aaa444344e7f6981fb253e0df7ca9d18a2157 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:28:49 +0100 Subject: [PATCH 31/47] Apply black --- src/tue_get/installer_impl.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 90976f16b..d222350d4 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -60,7 +60,9 @@ class InstallerImpl: _sources_list = os.path.join(os.sep, "etc", "apt", "sources.list") _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") - def __init__(self, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False): + def __init__( + self, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False + ): self._ros_test_deps = ros_test_deps self._ros_doc_deps = ros_doc_deps self._skip_ros_deps = skip_ros_deps @@ -1041,9 +1043,11 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: ros_pkg_name = self._current_target.lstrip("ros-") if "-" in ros_pkg_name: correct_ros_pkg_name = ros_pkg_name.replace("-", "_") - self.tue_install_error(f"A ROS package cannot contain dashes ({ros_pkg_name}), " - f"make sure the package is named '{correct_ros_pkg_name}' and rename the " - f"target to 'ros-{correct_ros_pkg_name}'") + self.tue_install_error( + f"A ROS package cannot contain dashes ({ros_pkg_name}), " + f"make sure the package is named '{correct_ros_pkg_name}' and rename the " + f"target to 'ros-{correct_ros_pkg_name}'" + ) # ToDo: This depends on behaviour of tue-install-error return False @@ -1108,8 +1112,6 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - - # If repos_dir is not set, try generating the default path from git url if target_dir is None: # ToDo: convert _git_url_to_repos_dir to python @@ -1152,8 +1154,7 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: elif not os.path.exists(ros_pkg_dir): os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) else: - self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. " - "Any other option is incorrect") + self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. Any other option is incorrect") if self._skip_ros_deps and not self._ros_test_deps and not self._ros_doc_deps: self.tue_install_debug("Skipping resolving of ROS dependencies") From fb8421f050f590c50cc2d32e348d3ceadbf7bad2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:29:02 +0100 Subject: [PATCH 32/47] Fix tue-install-cp --- src/tue_get/installer_impl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index d222350d4..fb3f4c453 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -517,9 +517,10 @@ def tue_install_cp(self, source: str, target: str) -> bool: if os.path.isdir(target): self.tue_install_debug(f"tue-install-cp: target {target} is a directory") + target_dir = target elif os.path.isfile(target): self.tue_install_debug(f"tue-install-cp: target {target} is a file") - target = os.path.dirname(target) + target_dir = os.path.dirname(target) else: self.tue_install_error(f"tue-install-cp: target {target} does not exist") # ToDo: This depends on behaviour of tue-install-error @@ -540,7 +541,10 @@ def tue_install_cp(self, source: str, target: str) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - cp_target = os.path.join(target, os.path.basename(file)) + if os.path.isdir(target): + cp_target = os.path.join(target_dir, os.path.basename(file)) + else: + cp_target = target if os.path.isfile(cp_target) and filecmp.cmp(file, cp_target): self.tue_install_debug(f"tue-install-cp: {file} and {cp_target} are identical, skipping") From f9b68d371b9e9c1698129d48f0a533798c54138c Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 17:29:29 +0100 Subject: [PATCH 33/47] Don't write to stdin of closed process --- src/tue_get/installer_impl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index fb3f4c453..8b960a285 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -155,7 +155,12 @@ def _set_target(self, target: str): self._current_target_dir = parent_target_dir def _out_handler(self, sub: BackgroundPopen, line: str) -> None: - def _write_stdin(msg) -> None: + def _write_stdin(msg: str) -> None: + if sub.returncode is not None: + self.tue_install_error( + f"Cannot write to stdin of process {sub.pid} as it has already terminated {line=}" + ) + return sub.stdin.write(f"{msg}\n") sub.stdin.flush() From 4806537f3399705ce267c1f4e68e240870b814ca Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 18:31:06 +0100 Subject: [PATCH 34/47] Fix package.xml check --- src/tue_get/installer_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 8b960a285..3e0622d01 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1171,7 +1171,7 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # Resolve ROS dependencies pkg_xml = os.path.join(ros_pkg_dir, PACKAGE_MANIFEST_FILENAME) - if os.path.isfile(pkg_xml): + if not os.path.isfile(pkg_xml): self.tue_install_warning(f"Does not contain a valid ROS {PACKAGE_MANIFEST_FILENAME}") return True From 7b1a8321da636c698c193b6de4cdb51b111a12d4 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 18:31:36 +0100 Subject: [PATCH 35/47] Fix args generation of tue-install-target(-now) --- src/tue_get/install_yaml_parser.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tue_get/install_yaml_parser.py b/src/tue_get/install_yaml_parser.py index 61678a822..d2fa4ec26 100644 --- a/src/tue_get/install_yaml_parser.py +++ b/src/tue_get/install_yaml_parser.py @@ -187,7 +187,10 @@ def get_distro_item(item: Mapping, key: str, release_version: str, release_type: if install_type in now_cache: now_cache[install_type].append(pkg_name) continue - command = partial(getattr(installer, f"tue_install_{install_type}"), pkg_name) + command = partial( + getattr(installer, f"tue_install_{install_type}"), + pkg_name if "target" in install_type else [pkg_name], + ) else: raise ValueError(f"Unknown install type: '{install_type}'") From 43a958b7538a57a488ce8fe8dee50605d4816038 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 18:32:08 +0100 Subject: [PATCH 36/47] Fix tue-install-warning not printing --- src/tue_get/installer_impl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 3e0622d01..b4a4261b5 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -299,6 +299,7 @@ def tue_install_warning(self, msg: str) -> None: log_msg = f"[{self._current_target}] WARNING: {msg}" colored_log = colored(log_msg, color="yellow", attrs=["bold"]) self._warn_logs.append(colored_log) + print(colored_log) self._log_to_file(log_msg) def tue_install_info(self, msg: str) -> None: From 2d95076b7b8eed3c062a1cb4401f6ab09e4ba65b Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Mon, 9 Jan 2023 18:32:47 +0100 Subject: [PATCH 37/47] Fix ros system pkg name --- src/tue_get/installer_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index b4a4261b5..1c78bc500 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1078,7 +1078,7 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: return False self.tue_install_debug(f"tue-install-system ros-{tue_ros_distro}-{name}") - if not self.tue_install_system(["ros-{tue_ros_distro}-{src}"]): + if not self.tue_install_system([f"ros-{tue_ros_distro}-{name}"]): self.tue_install_error(f"Failed to append ros-{tue_ros_distro}-{name}") # ToDo: This depends on behaviour of tue-install-error return False From 4ef22d5699ecfe92067bceae2ab4b6fac3bdf766 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 00:41:08 +0100 Subject: [PATCH 38/47] (tue-get dep) adapt to ros1/ros2 target --- installer/tue-get-dep.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/tue-get-dep.bash b/installer/tue-get-dep.bash index aa00f521a..583fe4841 100755 --- a/installer/tue-get-dep.bash +++ b/installer/tue-get-dep.bash @@ -8,7 +8,7 @@ hash xmlstarlet 2> /dev/null || sudo apt-get install --assume-yes -qq xmlstarlet function _show_dep { - [[ "$1" == "ros" ]] && return 0 + [[ "$1" == "ros" ]] || [[ "$1" == "ros1" ]] || [[ "$1" == "ros2" ]] && return 0 [[ "$ROS_ONLY" = "true" && "$1" != ros-* ]] && return 0 From 5a7b5e83ac00bc492375f18ae90f0136f261cc41 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 00:41:57 +0100 Subject: [PATCH 39/47] Add working tue-install-git --- src/tue_get/installer_impl.py | 163 +++++++++++++++++++++- src/tue_get/resources/installer_impl.bash | 17 ++- 2 files changed, 178 insertions(+), 2 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 1c78bc500..0c3de60c4 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -61,8 +61,9 @@ class InstallerImpl: _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") def __init__( - self, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False + self, branch: Optional[str] = None, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False ): + self._branch = branch self._ros_test_deps = ros_test_deps self._ros_doc_deps = ros_doc_deps self._skip_ros_deps = skip_ros_deps @@ -82,6 +83,7 @@ def __init__( self._dependencies_dir = os.path.join(self._tue_env_dir, ".env", "dependencies") self._dependencies_on_dir = os.path.join(self._tue_env_dir, ".env", "dependencies-on") self._installed_dir = os.path.join(self._tue_env_dir, ".env", "installed") + self._version_cache_dir = os.path.join(self._tue_env_dir, ".env", "version_cache") os.makedirs(self._dependencies_dir, exist_ok=True) os.makedirs(self._dependencies_on_dir, exist_ok=True) @@ -117,6 +119,8 @@ def __init__( self._snaps = [] self._gems = [] + self._git_pull_queue = set() + self._sudo_password = None def __del__(self): @@ -469,8 +473,165 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: self.tue_install_debug(f"Finished installing {target}") return True + def _show_update_msg(self, repo, msg: str = None): + # shellcheck disable=SC2086,SC2116 + if msg: + print_msg = "\n" + print_msg += f" {colored(repo, attrs=['bold'])}" + print_msg += f"\n--------------------------------------------------" + print_msg += f"\n{msg}" + print_msg += f"\n--------------------------------------------------" + print_msg += f"\n" + self.tue_install_tee(print_msg) + else: + self.tue_install_tee(f"{colored(repo, attrs=['bold'])}: up-tp-date") + + def _try_branch_git(self, target_dir, version): + self.tue_install_debug(f"_try_branch_git {target_dir=} {version=}") + + cmd = f"git -C {target_dir} checkout {version} --" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + try_branch_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{try_branch_res=}") + + cmd = f"git -C {target_dir} submodule sync --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_sync_results = sp.run(cmds, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + submodule_sync_res = submodule_sync_results.stdout.strip() + self.tue_install_debug(f"{submodule_sync_res=}") + + cmd = f"git -C {target_dir} submodule update --init --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{submodule_res=}") + + if "Already on " in try_branch_res or "fatal: invalid reference:" in try_branch_res: + try_branch_res = "" + + if submodule_sync_results.returncode != 0 and submodule_sync_res: + try_branch_res += f"\n{submodule_sync_res}" + if submodule_res: + try_branch_res += f"\n{submodule_res}" + + return try_branch_res + def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: Optional[str] = None) -> bool: self.tue_install_debug(f"tue-install-git {url=} {target_dir=} {version=}") + + # ToDo: convert _git_https_or_ssh to python + cmd = f"bash -c '_git_https_or_ssh {url}'" + cmd, cmds = _which_split_cmd(cmd) + url_old = url + url = sp.check_output(cmds, text=True).strip() + if not url: + self.tue_install_error(f"repo: '{url}' is invalid. It is generated from: '{url_old}'\n" + f"The problem will probably be solved by resourcing the setup") + # ToDo: This depends on behaviour of tue-install-error + return False + + if target_dir is None: + # ToDo: convert _git_url_to_repos_dir to python + cmd = f"bash -c '_git_url_to_repos_dir {url}'" + cmd, cmds = _which_split_cmd(cmd) + target_dir = sp.check_output(cmds, text=True).strip() + if not target_dir: + self.tue_install_error(f"Could not create target_dir path from the git url: '{url}'") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not target_dir: + self.tue_install_error(f"target_dir is specified, but empty") + # ToDo: This depends on behaviour of tue-install-error + return False + + if not os.path.isdir(target_dir): + cmd = f"git clone --recursive {url} {target_dir}" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + clone_results = sp.run(cmds, stdout=sp.PIPE, stderr=sp.STDOUT, text=True) + res = clone_results.stdout.strip() + if clone_results.returncode != 0: + self.tue_install_error(f"Failed to clone {url} to {target_dir}\n{res}") + # ToDo: This depends on behaviour of tue-install-error + return False + + self._git_pull_queue.add(target_dir) + else: + if target_dir in self._git_pull_queue: + self.tue_install_debug("Repo previously pulled, skipping") + res = "" + else: + cmd = f"git -C {target_dir} config --get remote.origin.url" + cmd, cmds = _which_split_cmd(cmd) + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + current_url = sp.check_output(cmds, text=True).strip() + # If different, switch url + if current_url != url: + cmd = f"git -C {target_dir} remote set-url origin {url}" + sub = self._default_background_popen(cmd) + if sub.returncode != 0: + self.tue_install_error(f"Could not change git url of '{target_dir}' to '{url}'" + f"({sub.returncode}):\n {repr(cmd)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + self.tue_install_info(f"url has switched to '{url}'") + + cmd = f"git -C {target_dir} pull --ff-only --prune" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{res=}") + + self._git_pull_queue.add(target_dir) + + cmd = f"git -C {target_dir} submodule sync --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + sp_results = sp.run(cmds, text=True, stdout=sp.PIPE) + submodule_sync_res = sp_results.stdout.strip() + self.tue_install_debug(f"{submodule_sync_res=}") + if sp_results.returncode != 0 and submodule_sync_res: + res += f"\n{submodule_sync_res}" + + + cmd = f"git -C {target_dir} submodule update --init --recursive" + self.tue_install_debug(f"{cmd}") + cmd, cmds = _which_split_cmd(cmd) + submodule_res = sp.check_output(cmds, stderr=sp.STDOUT, text=True).strip() + self.tue_install_debug(f"{submodule_res=}") + if submodule_res: + res += f"\n{submodule_res}" + + if "Already up to date." in res: + res = "" + + self.tue_install_debug(f"Desired version: {version}") + version_cache_file = os.path.join(self._version_cache_dir, target_dir.lstrip(os.sep)) + if version: + os.makedirs(os.path.dirname(version_cache_file), exist_ok=True) + with open(version_cache_file, "w") as f: + f.write(version) + + try_branch_res = self._try_branch_git(target_dir, version) + if try_branch_res: + res += f"\n{try_branch_res}" + else: + if os.path.isfile(version_cache_file): + os.remove(version_cache_file) + + if self._branch: + self.tue_install_debug(f"Desired branch: {self._branch}") + try_branch_res = self._try_branch_git(target_dir, self._branch) + if try_branch_res: + res += f"\n{try_branch_res}" + + + self._show_update_msg(self._current_target, res) return True def tue_install_apply_patch(self, patch_file: str, target_dir: str) -> bool: diff --git a/src/tue_get/resources/installer_impl.bash b/src/tue_get/resources/installer_impl.bash index 709504b8b..af4632dc9 100644 --- a/src/tue_get/resources/installer_impl.bash +++ b/src/tue_get/resources/installer_impl.bash @@ -94,7 +94,22 @@ function tue-install-target function tue-install-git { - echo -e "tue-install-git: $*" + local url targetdir version + url=$1 + shift + for i in "$@" + do + case $i in + --target-dir=* ) + targetdir="${i#*=}" + ;; + --version=* ) + version="${i#*=}" ;; + * ) + tue-install-error "Unknown input variable ${i}" ;; + esac + done + echo -e "tue-install-git: ${url} ${targetdir} ${version}" local return_value read -r return_value return $(("$return_value")) From 137b665f5ce5982d9edc4a78621e1cc23f7003a2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 00:42:17 +0100 Subject: [PATCH 40/47] Remove incorrect typehinting --- src/tue_get/installer_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 0c3de60c4..56c86586e 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -159,7 +159,7 @@ def _set_target(self, target: str): self._current_target_dir = parent_target_dir def _out_handler(self, sub: BackgroundPopen, line: str) -> None: - def _write_stdin(msg: str) -> None: + def _write_stdin(msg) -> None: if sub.returncode is not None: self.tue_install_error( f"Cannot write to stdin of process {sub.pid} as it has already terminated {line=}" From 0a23249daa9c0527503d8fcbcfd4c85e76fb10a7 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 00:45:00 +0100 Subject: [PATCH 41/47] less verbose cmd printing --- src/tue_get/installer_impl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 56c86586e..52261f1bf 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -282,7 +282,7 @@ def _err_handler(self, sub: BackgroundPopen, line: str) -> None: def _default_background_popen(self, cmd: str) -> BackgroundPopen: cmd, cmds = _which_split_cmd(cmd) - self.tue_install_echo(repr(cmd)) + self.tue_install_debug(repr(cmd)) sub = BackgroundPopen( args=cmds, out_handler=self._out_handler, err_handler=self._err_handler, stdin=sp.PIPE, text=True ) @@ -836,7 +836,7 @@ def tue_install_add_text(self, source_file: str, target_file: str) -> bool: cmd = f"bash -c \"echo -e '{target_text}' | {sudo_cmd}tee {target_file_path}\"" cmd, cmds = _which_split_cmd(cmd) - self.tue_install_echo(repr(cmd)) + self.tue_install_debug(repr(cmd)) sub = BackgroundPopen( args=cmds, err_handler=self._err_handler, @@ -880,7 +880,7 @@ def _out_handler_installed_pkgs(_: BackgroundPopen, line: str) -> None: # Based on https://stackoverflow.com/questions/1298066 cmd = "dpkg-query -W -f '${package} ${status}\n'" cmd, cmds = _which_split_cmd(cmd) - self.tue_install_echo(repr(cmd)) + self.tue_install_debug(repr(cmd)) sub = BackgroundPopen( args=cmds, out_handler=_out_handler_installed_pkgs, # Needed to prevent buffer to get full @@ -1071,7 +1071,7 @@ def tue_install_snap_now(self, pkgs: List[str]) -> bool: cmd = "snap list" cmd, cmds = _which_split_cmd(cmd) - self.tue_install_echo(repr(cmd)) + self.tue_install_debug(repr(cmd)) sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) sub.wait() if sub.returncode != 0: @@ -1126,7 +1126,7 @@ def tue_install_gem_now(self, pkgs: List[str]) -> bool: cmd = "gem list" cmd, cmds = _which_split_cmd(cmd) - self.tue_install_echo(repr(cmd)) + self.tue_install_debug(repr(cmd)) sub = BackgroundPopen(args=cmds, err_handler=self._err_handler, stdout=sp.PIPE, text=True) sub.wait() if sub.returncode != 0: From f179e9d0bf93974cf4710184ab21a11c7f89f403 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 00:46:44 +0100 Subject: [PATCH 42/47] Apply black --- src/tue_get/installer_impl.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 52261f1bf..4dea8c93a 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -61,7 +61,12 @@ class InstallerImpl: _sources_list_dir = os.path.join(os.sep, "etc", "apt", "sources.list.d") def __init__( - self, branch: Optional[str] = None, ros_test_deps: bool = False, ros_doc_deps: bool = False, skip_ros_deps: bool = False, debug: bool = False + self, + branch: Optional[str] = None, + ros_test_deps: bool = False, + ros_doc_deps: bool = False, + skip_ros_deps: bool = False, + debug: bool = False, ): self._branch = branch self._ros_test_deps = ros_test_deps @@ -527,8 +532,10 @@ def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: O url_old = url url = sp.check_output(cmds, text=True).strip() if not url: - self.tue_install_error(f"repo: '{url}' is invalid. It is generated from: '{url_old}'\n" - f"The problem will probably be solved by resourcing the setup") + self.tue_install_error( + f"repo: '{url}' is invalid. It is generated from: '{url_old}'\n" + f"The problem will probably be solved by resourcing the setup" + ) # ToDo: This depends on behaviour of tue-install-error return False @@ -574,8 +581,10 @@ def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: O cmd = f"git -C {target_dir} remote set-url origin {url}" sub = self._default_background_popen(cmd) if sub.returncode != 0: - self.tue_install_error(f"Could not change git url of '{target_dir}' to '{url}'" - f"({sub.returncode}):\n {repr(cmd)}") + self.tue_install_error( + f"Could not change git url of '{target_dir}' to '{url}'" + f"({sub.returncode}):\n {repr(cmd)}" + ) # ToDo: This depends on behaviour of tue-install-error return False @@ -598,7 +607,6 @@ def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: O if sp_results.returncode != 0 and submodule_sync_res: res += f"\n{submodule_sync_res}" - cmd = f"git -C {target_dir} submodule update --init --recursive" self.tue_install_debug(f"{cmd}") cmd, cmds = _which_split_cmd(cmd) @@ -630,7 +638,6 @@ def tue_install_git(self, url: str, target_dir: Optional[str] = None, version: O if try_branch_res: res += f"\n{try_branch_res}" - self._show_update_msg(self._current_target, res) return True From 94ceb3b2303fa9a670656ae7cc8fe54cb76451e1 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 01:25:26 +0100 Subject: [PATCH 43/47] Remove unused comment --- src/tue_get/installer_impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 4dea8c93a..0481e3f9a 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -479,7 +479,6 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: return True def _show_update_msg(self, repo, msg: str = None): - # shellcheck disable=SC2086,SC2116 if msg: print_msg = "\n" print_msg += f" {colored(repo, attrs=['bold'])}" From e0e98b500fa27b2fafff5a7d465a6918872e2825 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 10 Jan 2023 01:26:30 +0100 Subject: [PATCH 44/47] Fix tue-install-ros --- src/tue_get/installer_impl.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 0481e3f9a..e0476bf2f 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -1217,7 +1217,8 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - ros_pkg_name = self._current_target.lstrip("ros-") + # Remove 'ros-' prefix + ros_pkg_name = self._current_target[4:] if "-" in ros_pkg_name: correct_ros_pkg_name = ros_pkg_name.replace("-", "_") self.tue_install_error( @@ -1301,6 +1302,7 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: return False self.tue_install_debug(f"{target_dir=}") + target_sub_dir = os.path.join(target_dir, sub_dir) ros_pkg_dir = os.path.join(ros_package_install_dir, ros_pkg_name) @@ -1312,7 +1314,7 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDo: This depends on behaviour of tue-install-error return False - if not os.path.isdir(os.path.join(target_dir, sub_dir)): + if not os.path.isdir(target_sub_dir): self.tue_install_error(f"Subdirectory '{sub_dir}' does not exist for url '{url}'") # ToDo: This depends on behaviour of tue-install-error return False @@ -1320,16 +1322,17 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # Test if the current symbolic link points to the same repository dir. If not, give a warning # because it means the source URL has changed if os.path.islink(ros_pkg_dir): - if os.path.realpath(ros_pkg_dir) != os.path.realpath(os.path.join(target_dir, sub_dir)): + if os.path.realpath(ros_pkg_dir) != os.path.realpath(target_sub_dir): self.tue_install_info(f"url has changed to {url}/{sub_dir}") os.remove(ros_pkg_dir) - os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + os.symlink(target_sub_dir, ros_pkg_dir) elif os.path.isdir(ros_pkg_dir): self.tue_install_error(f"Can not create a symlink at '{ros_pkg_dir}' as it is a directory") # ToDo: This depends on behaviour of tue-install-error return False elif not os.path.exists(ros_pkg_dir): - os.symlink(os.path.join(target_dir, sub_dir), ros_pkg_dir) + self.tue_install_debug(f"Creating symlink from {target_sub_dir} to {ros_pkg_dir}") + os.symlink(target_sub_dir, ros_pkg_dir) else: self.tue_install_error(f"'{ros_pkg_dir}' should not exist or be a symlink. Any other option is incorrect") From bd3786b9f3d56774872005439791b005a009d7d2 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 13 Jan 2023 12:09:26 +0100 Subject: [PATCH 45/47] Fixed naming of python file --- src/tue_get/installer_impl.py | 2 +- src/tue_get/util/__init__.py | 2 +- src/tue_get/util/{BackgroundPopen.py => background_popen.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/tue_get/util/{BackgroundPopen.py => background_popen.py} (100%) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index e0476bf2f..3e32ea645 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -19,7 +19,7 @@ from tue_get.catkin_package_parser import catkin_package_parser from tue_get.install_yaml_parser import installyaml_parser -from tue_get.util.BackgroundPopen import BackgroundPopen +from tue_get.util.background_popen import BackgroundPopen from tue_get.util.grep import grep_directory, grep_file CI = None diff --git a/src/tue_get/util/__init__.py b/src/tue_get/util/__init__.py index a384ea7ad..1259abb53 100644 --- a/src/tue_get/util/__init__.py +++ b/src/tue_get/util/__init__.py @@ -1,2 +1,2 @@ -from . import BackgroundPopen +from . import background_popen from . import grep diff --git a/src/tue_get/util/BackgroundPopen.py b/src/tue_get/util/background_popen.py similarity index 100% rename from src/tue_get/util/BackgroundPopen.py rename to src/tue_get/util/background_popen.py From 29ac21d2a44decaf5500180f50bf98764993b5cb Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Fri, 13 Jan 2023 12:10:39 +0100 Subject: [PATCH 46/47] Add remaining install logic --- src/tue_get/installer_impl.py | 202 +++++++++++++++++++++++++++++++++- 1 file changed, 199 insertions(+), 3 deletions(-) diff --git a/src/tue_get/installer_impl.py b/src/tue_get/installer_impl.py index 3e32ea645..eda9883f9 100644 --- a/src/tue_get/installer_impl.py +++ b/src/tue_get/installer_impl.py @@ -131,6 +131,14 @@ def __init__( def __del__(self): shutil.rmtree(self._state_dir) + def _get_installed_targets(self) -> List[str]: + return [ + pkg for pkg in os.listdir(self._installed_dir) if os.path.isfile(os.path.join(self._installed_dir, pkg)) + ] + + def _target_exist(self, target: str) -> bool: + return os.path.isdir(os.path.join(self._targets_dir, target)) + def _log_to_file(self, msg: str, end="\n") -> None: with open(self._log_file, "a") as f: f.write(msg + end) @@ -361,7 +369,7 @@ def tue_install_target(self, target: str, now: bool = False) -> bool: self.tue_install_debug(f"Installing target: {target}") # Check if valid target received as input - if not os.path.isdir(os.path.join(self._targets_dir, target)): + if not self._target_exist(target): self.tue_install_debug(f"Target '{target}' does not exist.") return False @@ -1376,7 +1384,195 @@ def tue_install_ros(self, source_type: str, **kwargs) -> bool: # ToDO: TUE_INSTALL_PKG_DIR was set ros_pkg_dir which was then use in tue-install-apply-patch; we are not doing that not (yet) in python return True + def install(self, targets: List[str]) -> bool: + if not targets: + self.tue_install_error("No targets to install") + # ToDo: This depends on behaviour of tue-install-error + return False + + missing_targets = [] + for target in targets: + if not self._target_exist(target): + missing_targets.append(target) + + if missing_targets: + self.tue_install_error(f"The following installed targets don't exist (anymore):\n{sorted(missing_targets)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for target in targets: + self.tue_install_debug(f"Installing target '{target}'") + if not self.tue_install_target(target): + self.tue_install_error(f"Failed to install target '{target}'") + return False + + # Mark as installed + self.tue_install_debug(f"Marking '{target}' as installed after successful installation") + Path(os.path.join(self._installed_dir, target)).touch(exist_ok=True) + + return True + + def update(self, targets: List[str]) -> bool: + if not targets: + targets = self._get_installed_targets() + else: + installed_targets = self._get_installed_targets() + for target in targets: + if target not in installed_targets: + self.tue_install_error(f"Target '{target}' is not installed") + # ToDo: This depends on behaviour of tue-install-error + return False + + missing_targets = [] + for target in targets: + if not self._target_exist(target): + missing_targets.append(target) + + if missing_targets: + self.tue_install_error(f"The following installed targets don't exist (anymore):\n{sorted(missing_targets)}") + # ToDo: This depends on behaviour of tue-install-error + return False + + for target in targets: + self.tue_install_debug(f"Updating target '{target}'") + if not self.tue_install_target(target): + self.tue_install_error(f"Failed to update target '{target}'") + return False + self.tue_install_debug(f"{target} succesfully updated") + + return True + + def print_queued_logs(self) -> bool: + if self._info_logs: + logs = "\n ".join(self._info_logs) + print(f"Some information you may have missed:\n\n {logs}\n") + + if self._warn_logs: + logs = "\n ".join(self._warn_logs) + print(f"Overview of warnings:\n\n {logs}\n") + + return True + + def install_queued_pkgs(self) -> bool: + if self._ppas: + with self._set_target("PPA-ADD"): + self.tue_install_debug(f"calling tue-install-ppa-now {self._ppas}") + if not self.tue_install_ppa_now(self._ppas): + self.tue_install_error(f"Failed to add PPA's: {self._ppas}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._systems: + with self._set_target("APT-GET"): + self.tue_install_debug(f"calling tue-install-system-now {self._systems}") + if not self.tue_install_system_now(self._systems): + self.tue_install_error(f"Failed to install system packages: {self._systems}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._pips: + with self._set_target("PIP"): + self.tue_install_debug(f"calling tue-install-pip-now {self._pips}") + if not self.tue_install_pip_now(self._pips): + self.tue_install_error(f"Failed to install pip packages: {self._pips}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._snaps: + with self._set_target("SNAP"): + self.tue_install_debug(f"calling tue-install-snap-now {self._snaps}") + if not self.tue_install_snap_now(self._snaps): + self.tue_install_error(f"Failed to install snap packages: {self._snaps}") + # ToDo: This depends on behaviour of tue-install-error + return False + + if self._gems: + with self._set_target("GEM"): + self.tue_install_debug(f"calling tue-install-gem-now {self._gems}") + if not self.tue_install_gem_now(self._gems): + self.tue_install_error(f"Failed to install gem packages: {self._gems}") + # ToDo: This depends on behaviour of tue-install-error + return False + + return True + if __name__ == "__main__": - bla = InstallerImpl(debug=True) - bla.tue_install_target("test", True) + import argparse + import sys + + ros_test_depends = os.environ.get("TUE_INSTALL_TEST_DEPENDS", False) + ros_doc_depends = os.environ.get("TUE_INSTALL_DOC_DEPENDS", False) + + parser = argparse.ArgumentParser(prog="tue-get", description="Installs all your (ROS) dependencies") + parser.add_argument("--branch", "-b", help="Branch to checkout", default=None) + parser.add_argument("--debug", "-d", action="store_true", help="Enable debug output", default=False) + parser.add_argument("--no-ros-deps", action="store_true", help="Skip resolving of ROS dependencies", default=False) + parser.add_argument( + "mode", + choices=["install", "update"], + type=str, + help="Install OR update the targets", + ) + m = parser.add_mutually_exclusive_group(required=False) + m.add_argument( + "--test-depends", + dest="test_depends", + action="store_true", + help="Also resolve ROS test dependencies", + default=ros_test_depends, + ) + m.add_argument( + "--no-test-depends", + dest="test_depends", + action="store_false", + help="Do not resolve ROS test dependencies", + default=ros_test_depends, + ) + m = parser.add_mutually_exclusive_group(required=False) + m.add_argument( + "--doc-depends", + dest="doc_depends", + action="store_true", + help="Also resolve ROS doc dependencies", + default=ros_doc_depends, + ) + m.add_argument( + "--no-doc-depends", + dest="doc_depends", + action="store_false", + help="Do not resolve ROS doc dependencies", + default=ros_doc_depends, + ) + parser.add_argument("targets", help="Targets to install", nargs=argparse.REMAINDER) + + args = parser.parse_args() + if args.mode == "install" and not args.targets: + parser.error("Minimal one target should be specified, when installing") + + installer = InstallerImpl( + branch=args.branch, + debug=args.debug, + skip_ros_deps=args.no_ros_deps, + ros_test_deps=args.test_depends, + ros_doc_deps=args.doc_depends, + ) + + targets = sorted(args.targets) + if args.mode == "install": + if not installer.install(targets): + installer.tue_install_error(f"Failed to install targets: {targets}") + sys.exit(1) + elif args.mode == "update": + if not installer.update(targets): + installer.tue_install_error(f"Failed to update targets: {targets}") + sys.exit(1) + + if not installer.print_queued_logs(): + pass + + if not installer.install_queued_pkgs(): + sys.exit(1) + + installer.tue_install_echo("Installer completed successfully") + sys.exit(0) From 2cb7eb9427288601a0e528dc9360174099a49af1 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Tue, 14 Feb 2023 12:05:22 +0100 Subject: [PATCH 47/47] Fix Black Linting --- src/tue_get/catkin_package_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tue_get/catkin_package_parser.py b/src/tue_get/catkin_package_parser.py index 0723d96c0..2114ead34 100644 --- a/src/tue_get/catkin_package_parser.py +++ b/src/tue_get/catkin_package_parser.py @@ -12,7 +12,6 @@ def catkin_package_parser( warnings: Optional[List[str]] = None, context: Optional[Mapping] = None, ) -> Tuple[Package, List[Dependency]]: - if context is None: context = os.environ dep_types = []