From 3905d7487694e887d61f025f158689dad045bfd8 Mon Sep 17 00:00:00 2001 From: PetervDooren Date: Wed, 4 Jan 2023 13:07:35 +0100 Subject: [PATCH 1/7] first attempt --- installer/parse_package_xml_2.py | 116 +++++++++++++++++++++++++++++++ setup/versioning-test.bash | 52 ++++++++++++++ 2 files changed, 168 insertions(+) create mode 100755 installer/parse_package_xml_2.py create mode 100644 setup/versioning-test.bash diff --git a/installer/parse_package_xml_2.py b/installer/parse_package_xml_2.py new file mode 100755 index 000000000..ed90610d0 --- /dev/null +++ b/installer/parse_package_xml_2.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python3 + +from typing import Mapping +import os +import sys +import xml.etree.ElementTree as ET + +from catkin_pkg.condition import evaluate_condition + +import rospkg +rospack = rospkg.RosPack() + +import copy + + +def main() -> int: + """ + Print all dependencies of the specified package + """ + + if len(sys.argv) != 2: + print("Usage: parse_package_xml_2 PACKAGE.XML") + return 1 + + deps = recursive_get_deps(sys.argv[1]) + for dep in deps: + #print(f"Dependency on {dep[0]} with version {dep[1]}") + print("\n".join(dep)) + return 0 + + +def recursive_get_deps(path: str) -> set: + dep_set = packagexml_parser(path)["deps"] + total_dep_set = copy.deepcopy(dep_set) + for dep in dep_set: + # get dependencies of the dependency + try: + dep_path = rospack.get_path(dep[0]) + "/package.xml" + except rospkg.common.ResourceNotFound: + print(f"could not find {dep[0]}") + continue + dep_dep_set = recursive_get_deps(dep_path) + total_dep_set |= dep_dep_set + return total_dep_set + + +def packagexml_parser(path: str) -> Mapping: + """ + parse the xml file and return a list of dependencies as a mapping + @param path: absolute path to the xml file to be parsed + @return Mapping with the following fields: + parsed: ... + deps: list of strings naming the dependencies of the package + """ + tree = ET.parse(path) + doc = tree.getroot() + + dep_set = set() + + dep_types = [] + dep_versions = [] + fields = ["name", "version", "description", "maintainer", "export"] + parsed = {} + + if os.getenv("TUE_INSTALL_SKIP_ROS_DEPS", "false") == "false": + dep_types.extend( + [ + "build_depend", + "buildtool_depend", + "build_export_depend", + "buildtool_export_depend", + "exec_depend", + "depend", + "run_depend", + ] + ) + dep_versions.extend(["version_eq"]) + + if os.getenv("TUE_INSTALL_TEST_DEPEND", "false") == "true": + dep_types.append("test_depend") + + if os.getenv("TUE_INSTALL_DOC_DEPEND", "false") == "true": + dep_types.append("doc_depend") + + for types in fields + dep_types: + parsed[types] = set() + + for dep_type in dep_types: + deps = doc.findall(dep_type) + for dep in deps: + if evaluate_condition(dep.attrib.get("condition", None), os.environ): + version = None + for dep_version in dep_versions: + version = dep.get(dep_version) + version = str(version) + parsed[dep_type] |= {(dep.text, version)} + dep_set |= parsed[dep_type] + + for field in fields: + values = doc.findall(field) + if field == "export": + for exports in values: + parsed["build_type"] = {bt.text for bt in exports if bt.tag == "build_type"} + else: + parsed[field] |= {value.text for value in values} + if field == "maintainer": + emails = {} + for value in values: + emails[value.text] = value.attrib.get("email") + parsed["emails"] = emails + + return {"parser": parsed, "deps": dep_set} + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup/versioning-test.bash b/setup/versioning-test.bash new file mode 100644 index 000000000..ad683c6b4 --- /dev/null +++ b/setup/versioning-test.bash @@ -0,0 +1,52 @@ + +function tue-check-version +{ + if [ -z "$1" ] + then + echo "Invalid tue-check-version: needs package as argument." + return 1 + fi + + local ros_pkg_name=$1 + + local ros_pkg_dir + ros_pkg_dir="$ROS_PACKAGE_INSTALL_DIR"/"$ros_pkg_name" + + local pkg_xml="$ros_pkg_dir"/package.xml + + local deps + deps=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_package_xml_2.py "$pkg_xml") + for d in $deps + do + echo "depends on $d" + done +} + +function _tue-dir-version +{ + [ -d "$1" ] || return 1 + + local fs + fs=$(ls "$1") + for f in $fs + do + local pkg + pkg=$f + local version + version=$(rosversion "$f") + echo "package $pkg has version $version" + done +} + +# ---------------------------------------------------------------------------------------------------- + +function tue-version +{ + _tue-dir-version "$TUE_SYSTEM_DIR"/src +} + +# Initialize +TUE_INSTALL_SCRIPTS_DIR=$TUE_DIR/installer + +# Initialize +ROS_PACKAGE_INSTALL_DIR=$TUE_SYSTEM_DIR/src From 72508f2711b31f26a2fb04234618c0e0ed2f7db6 Mon Sep 17 00:00:00 2001 From: PetervDooren Date: Wed, 4 Jan 2023 16:08:36 +0100 Subject: [PATCH 2/7] remove recursion and add lists --- installer/parse_package_xml_2.py | 35 +++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/installer/parse_package_xml_2.py b/installer/parse_package_xml_2.py index ed90610d0..1a9c52155 100755 --- a/installer/parse_package_xml_2.py +++ b/installer/parse_package_xml_2.py @@ -24,24 +24,39 @@ def main() -> int: deps = recursive_get_deps(sys.argv[1]) for dep in deps: - #print(f"Dependency on {dep[0]} with version {dep[1]}") - print("\n".join(dep)) + print(f"Dependency on {dep[0]} with version {dep[1]}") + #print("\n".join(dep)) return 0 def recursive_get_deps(path: str) -> set: - dep_set = packagexml_parser(path)["deps"] - total_dep_set = copy.deepcopy(dep_set) - for dep in dep_set: + parsed_packages = list() + to_parse_packages = list() + deps = set() + + direct_deps = packagexml_parser(path)["deps"] + to_parse_packages.extend([dep[0] for dep in direct_deps]) + deps |= direct_deps + + while len(to_parse_packages) > 0: + package = to_parse_packages.pop() + parsed_packages.append(package) + # get dependencies of the dependency try: - dep_path = rospack.get_path(dep[0]) + "/package.xml" + dep_path = rospack.get_path(package) + "/package.xml" except rospkg.common.ResourceNotFound: - print(f"could not find {dep[0]}") + print(f"could not find {package}") continue - dep_dep_set = recursive_get_deps(dep_path) - total_dep_set |= dep_dep_set - return total_dep_set + dep_set = packagexml_parser(dep_path)["deps"] + for dep in dep_set: + if dep[0] in parsed_packages: + continue + if dep[0] not in to_parse_packages: + # TODO check if package dep version also matches the one in the set. + to_parse_packages.append(dep[0]) + deps |= {dep} + return deps def packagexml_parser(path: str) -> Mapping: From 66c2d1670e849f267bb33b0d4437c15f36c3a7d4 Mon Sep 17 00:00:00 2001 From: PetervDooren Date: Sat, 7 Jan 2023 10:35:43 +0100 Subject: [PATCH 3/7] add named tuple as datatype --- installer/parse_package_xml_2.py | 51 ++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/installer/parse_package_xml_2.py b/installer/parse_package_xml_2.py index 1a9c52155..faccc0423 100755 --- a/installer/parse_package_xml_2.py +++ b/installer/parse_package_xml_2.py @@ -9,8 +9,9 @@ import rospkg rospack = rospkg.RosPack() +from collections import namedtuple -import copy +Dependency = namedtuple('Dependency', ['name', 'req_version', 'actual_version', 'comparator']) def main() -> int: @@ -22,21 +23,32 @@ def main() -> int: print("Usage: parse_package_xml_2 PACKAGE.XML") return 1 + mismatch_found = False deps = recursive_get_deps(sys.argv[1]) - for dep in deps: - print(f"Dependency on {dep[0]} with version {dep[1]}") + for key, value in deps.items(): + if value.req_version != "None" and value.req_version != value.actual_version: + print(f"Package {value.name}, current version {value.actual_version}, required version {value.req_version}") + mismatch_found = True #print("\n".join(dep)) + + if mismatch_found: + print("Mismatch found in dependencies") + else: + print("Dependencies are consistent") + return 0 -def recursive_get_deps(path: str) -> set: +def recursive_get_deps(path: str) -> dict: parsed_packages = list() to_parse_packages = list() - deps = set() + deps = dict() - direct_deps = packagexml_parser(path)["deps"] + parsed_mapping = packagexml_parser(path) + direct_deps = parsed_mapping["deps"] to_parse_packages.extend([dep[0] for dep in direct_deps]) - deps |= direct_deps + for dep in direct_deps: + deps[dep[0]] = Dependency(dep[0], dep[1], '0.0.0', 0) while len(to_parse_packages) > 0: package = to_parse_packages.pop() @@ -46,16 +58,31 @@ def recursive_get_deps(path: str) -> set: try: dep_path = rospack.get_path(package) + "/package.xml" except rospkg.common.ResourceNotFound: - print(f"could not find {package}") + #print(f"could not find {package}, assuming not a ros package") + del deps[package] continue - dep_set = packagexml_parser(dep_path)["deps"] + + parsed_mapping = packagexml_parser(dep_path) + # register current version + version_set = parsed_mapping["version"] + if len(version_set) != 1: + print(f"Package {package} should have 1 version, instead its version is {version_set}") + raise Exception #TODO better exception + # update actual version + deps[package] = Dependency(deps[package].name, + deps[package].req_version, + list(version_set)[0], + deps[package].comparator) + + dep_set = parsed_mapping["deps"] for dep in dep_set: if dep[0] in parsed_packages: + if dep[0] in deps and deps[dep[0]].req_version != dep[1]: + print(f"inconsistent dependency. According to package {package}, package {dep[0]} should have version {dep[1]}, but another package requires {deps[dep[0]].req_version}") continue if dep[0] not in to_parse_packages: - # TODO check if package dep version also matches the one in the set. to_parse_packages.append(dep[0]) - deps |= {dep} + deps[dep[0]] = Dependency(dep[0], dep[1], 'Problems!', 0) return deps @@ -124,7 +151,7 @@ def packagexml_parser(path: str) -> Mapping: emails[value.text] = value.attrib.get("email") parsed["emails"] = emails - return {"parser": parsed, "deps": dep_set} + return {"parser": parsed, "deps": dep_set, "version": parsed["version"]} if __name__ == "__main__": From 32e13f0913ec13404e173e6f79fb76e5f035926c Mon Sep 17 00:00:00 2001 From: PetervDooren Date: Sat, 7 Jan 2023 10:36:56 +0100 Subject: [PATCH 4/7] renamed python file --- .../{parse_package_xml_2.py => check_dependency_versions.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename installer/{parse_package_xml_2.py => check_dependency_versions.py} (100%) diff --git a/installer/parse_package_xml_2.py b/installer/check_dependency_versions.py similarity index 100% rename from installer/parse_package_xml_2.py rename to installer/check_dependency_versions.py From f6fb8d06922bcc6a1a127c8f747b8d616040b1de Mon Sep 17 00:00:00 2001 From: PetervDooren Date: Sat, 7 Jan 2023 10:39:48 +0100 Subject: [PATCH 5/7] remove experimentation script --- setup/versioning-test.bash | 52 -------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 setup/versioning-test.bash diff --git a/setup/versioning-test.bash b/setup/versioning-test.bash deleted file mode 100644 index ad683c6b4..000000000 --- a/setup/versioning-test.bash +++ /dev/null @@ -1,52 +0,0 @@ - -function tue-check-version -{ - if [ -z "$1" ] - then - echo "Invalid tue-check-version: needs package as argument." - return 1 - fi - - local ros_pkg_name=$1 - - local ros_pkg_dir - ros_pkg_dir="$ROS_PACKAGE_INSTALL_DIR"/"$ros_pkg_name" - - local pkg_xml="$ros_pkg_dir"/package.xml - - local deps - deps=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_package_xml_2.py "$pkg_xml") - for d in $deps - do - echo "depends on $d" - done -} - -function _tue-dir-version -{ - [ -d "$1" ] || return 1 - - local fs - fs=$(ls "$1") - for f in $fs - do - local pkg - pkg=$f - local version - version=$(rosversion "$f") - echo "package $pkg has version $version" - done -} - -# ---------------------------------------------------------------------------------------------------- - -function tue-version -{ - _tue-dir-version "$TUE_SYSTEM_DIR"/src -} - -# Initialize -TUE_INSTALL_SCRIPTS_DIR=$TUE_DIR/installer - -# Initialize -ROS_PACKAGE_INSTALL_DIR=$TUE_SYSTEM_DIR/src From be7edbe4d2a5a17254077e78fe5fb8c7368f2481 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 13:49:57 +0100 Subject: [PATCH 6/7] Apply black linting --- installer/check_dependency_versions.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/installer/check_dependency_versions.py b/installer/check_dependency_versions.py index faccc0423..76b69c884 100755 --- a/installer/check_dependency_versions.py +++ b/installer/check_dependency_versions.py @@ -8,10 +8,11 @@ from catkin_pkg.condition import evaluate_condition import rospkg + rospack = rospkg.RosPack() from collections import namedtuple -Dependency = namedtuple('Dependency', ['name', 'req_version', 'actual_version', 'comparator']) +Dependency = namedtuple("Dependency", ["name", "req_version", "actual_version", "comparator"]) def main() -> int: @@ -29,7 +30,7 @@ def main() -> int: if value.req_version != "None" and value.req_version != value.actual_version: print(f"Package {value.name}, current version {value.actual_version}, required version {value.req_version}") mismatch_found = True - #print("\n".join(dep)) + # print("\n".join(dep)) if mismatch_found: print("Mismatch found in dependencies") @@ -48,7 +49,7 @@ def recursive_get_deps(path: str) -> dict: direct_deps = parsed_mapping["deps"] to_parse_packages.extend([dep[0] for dep in direct_deps]) for dep in direct_deps: - deps[dep[0]] = Dependency(dep[0], dep[1], '0.0.0', 0) + deps[dep[0]] = Dependency(dep[0], dep[1], "0.0.0", 0) while len(to_parse_packages) > 0: package = to_parse_packages.pop() @@ -58,7 +59,7 @@ def recursive_get_deps(path: str) -> dict: try: dep_path = rospack.get_path(package) + "/package.xml" except rospkg.common.ResourceNotFound: - #print(f"could not find {package}, assuming not a ros package") + # print(f"could not find {package}, assuming not a ros package") del deps[package] continue @@ -67,22 +68,23 @@ def recursive_get_deps(path: str) -> dict: version_set = parsed_mapping["version"] if len(version_set) != 1: print(f"Package {package} should have 1 version, instead its version is {version_set}") - raise Exception #TODO better exception + raise Exception # TODO better exception # update actual version - deps[package] = Dependency(deps[package].name, - deps[package].req_version, - list(version_set)[0], - deps[package].comparator) + deps[package] = Dependency( + deps[package].name, deps[package].req_version, list(version_set)[0], deps[package].comparator + ) dep_set = parsed_mapping["deps"] for dep in dep_set: if dep[0] in parsed_packages: if dep[0] in deps and deps[dep[0]].req_version != dep[1]: - print(f"inconsistent dependency. According to package {package}, package {dep[0]} should have version {dep[1]}, but another package requires {deps[dep[0]].req_version}") + print( + f"inconsistent dependency. According to package {package}, package {dep[0]} should have version {dep[1]}, but another package requires {deps[dep[0]].req_version}" + ) continue if dep[0] not in to_parse_packages: to_parse_packages.append(dep[0]) - deps[dep[0]] = Dependency(dep[0], dep[1], 'Problems!', 0) + deps[dep[0]] = Dependency(dep[0], dep[1], "Problems!", 0) return deps From abae2ad6915412b6dcf13225313a75f519049470 Mon Sep 17 00:00:00 2001 From: Matthijs van der Burgh Date: Sat, 7 Jan 2023 13:50:39 +0100 Subject: [PATCH 7/7] Remove commented code --- installer/check_dependency_versions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/installer/check_dependency_versions.py b/installer/check_dependency_versions.py index 76b69c884..3276f994d 100755 --- a/installer/check_dependency_versions.py +++ b/installer/check_dependency_versions.py @@ -30,7 +30,6 @@ def main() -> int: if value.req_version != "None" and value.req_version != value.actual_version: print(f"Package {value.name}, current version {value.actual_version}, required version {value.req_version}") mismatch_found = True - # print("\n".join(dep)) if mismatch_found: print("Mismatch found in dependencies") @@ -59,7 +58,6 @@ def recursive_get_deps(path: str) -> dict: try: dep_path = rospack.get_path(package) + "/package.xml" except rospkg.common.ResourceNotFound: - # print(f"could not find {package}, assuming not a ros package") del deps[package] continue