allowing create_package_and_install to install relative deps automagically. (Azure#27832)

Co-authored-by: Scott Beddall <[email protected]>

Author: scbedd

eng/tox/create_package_and_install.py

@@ -9,24 +9,25 @@
 # it should be executed from tox with `{toxenvdir}/python` to ensure that the package
 # can be successfully tested from within a tox environment.
 
-from subprocess import check_call
+from subprocess import check_call, CalledProcessError
 import argparse
 import os
 import logging
 import sys
 import glob
 import shutil
-import pdb
 from pkg_resources import parse_version
 
 from tox_helper_tasks import find_whl, find_sdist, get_pip_list_output
 from ci_tools.parsing import ParsedSetup, parse_require
 from ci_tools.build import create_package
+from ci_tools.functions import get_package_from_repo
 
 logging.getLogger().setLevel(logging.INFO)
 
 from ci_tools.parsing import ParsedSetup
 
+
 def cleanup_build_artifacts(build_folder):
     # clean up egginfo
     results = glob.glob(os.path.join(build_folder, "*.egg-info"))
@@ -170,7 +171,7 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
         os.mkdir(tmp_dl_folder)
 
     # preview version is enabled when installing dev build so pip will install dev build version from devpos feed
-    if os.getenv("SetDevVersion", 'false') == 'true':
+    if os.getenv("SetDevVersion", "false") == "true":
         commands_options.append("--pre")
 
     if args.cache_dir:
@@ -197,7 +198,9 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
                 logging.info("Installing {w} from fresh built package.".format(w=built_package))
 
             if not args.pre_download_disabled:
-                requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(args.target_setup), "setup.py")).requires
+                requirements = ParsedSetup.from_path(
+                    os.path.join(os.path.abspath(args.target_setup), "setup.py")
+                ).requires
                 azure_requirements = [req.split(";")[0] for req in requirements if req.startswith("azure")]
 
                 if azure_requirements:
@@ -238,19 +241,28 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
                                 addition_necessary = False
 
                         if addition_necessary:
+                            # we only want to add an additional rec for download if it actually exists
+                            # in the upstream feed (either dev or pypi)
+                            # if it doesn't, we should just install the relative dep if its an azure package
                             installation_additions.append(req)
 
                     if installation_additions:
-                        download_command.extend(installation_additions)
-                        download_command.extend(commands_options)
+                        non_present_reqs = []
+                        for addition in installation_additions:
+                            try:
+                                check_call(
+                                    download_command + [addition] + commands_options,
+                                    env=dict(os.environ, PIP_EXTRA_INDEX_URL=""),
+                                )
+                            except CalledProcessError as e:
+                                req_name, req_specifier = parse_require(addition)
+                                non_present_reqs.append(req_name)
 
-                        check_call(download_command, env=dict(os.environ, PIP_EXTRA_INDEX_URL=""))
                         additional_downloaded_reqs = [
                             os.path.abspath(os.path.join(tmp_dl_folder, pth)) for pth in os.listdir(tmp_dl_folder)
-                        ]
+                        ] + [get_package_from_repo(relative_req).folder for relative_req in non_present_reqs]
 
             commands = [sys.executable, "-m", "pip", "install", built_pkg_path]
-
             commands.extend(additional_downloaded_reqs)
             commands.extend(commands_options)
 

tools/azure-sdk-tools/ci_tools/functions.py

@@ -203,27 +203,33 @@ def is_required_version_on_pypi(package_name, spec):
     return versions
 
 
-def get_version_from_repo(pkg_name: str, repo_root: str = None):
+def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup:
     root_dir = discover_repo_root(repo_root)
 
-    # find version for the package from source. This logic should be revisited to find version from devops feed
     glob_path = os.path.join(root_dir, "sdk", "*", pkg_name, "setup.py")
     paths = glob.glob(glob_path)
+
     if paths:
         setup_py_path = paths[0]
         parsed_setup = ParsedSetup.from_path(setup_py_path)
+        return parsed_setup
+
+    return None
 
+
+def get_version_from_repo(pkg_name: str, repo_root: str = None):
+    pkg_info = get_package_from_repo(pkg_name, repo_root)
+    if pkg_info:
         # Remove dev build part if version for this package is already updated to dev build
         # When building package with dev build version, version for packages in same service is updated to dev build
         # and other packages will not have dev build number
         # strip dev build number so we can check if package exists in PyPI and replace
-
-        version_obj = Version(parsed_setup.version)
+        version_obj = Version(pkg_info.version)
         if version_obj.pre:
             if version_obj.pre[0] == DEV_BUILD_IDENTIFIER:
-                version = version_obj.base_version
+                return version_obj.base_version
 
-        return version
+        return str(version_obj)
     else:
         logging.error("setup.py is not found for package {} to identify current version".format(pkg_name))
         exit(1)

tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -46,6 +46,7 @@ def __init__(
         self.namespace = name_space
         self.package_data = package_data
         self.include_package_data = include_package_data
+        self.folder = os.path.dirname(self.setup_filename)
 
     @classmethod
     def from_path(cls, parse_directory_or_file: str):
@@ -200,7 +201,7 @@ def parse_require(req: str) -> Tuple[str, SpecifierSet]:
         return [pkg_name, None]
 
     # regex details ripped from https://peps.python.org/pep-0508/
-    isolated_spec = re.sub(r'^([a-zA-Z0-9\-\_\.]+)(\[[a-zA-Z0-9\-\_\.\,]*\])?', "", str(req_object))
+    isolated_spec = re.sub(r"^([a-zA-Z0-9\-\_\.]+)(\[[a-zA-Z0-9\-\_\.\,]*\])?", "", str(req_object))
     spec = SpecifierSet(isolated_spec)
     return (pkg_name, spec)
 

tools/azure-sdk-tools/tests/test_parse_functionality.py

@@ -113,3 +113,4 @@ def test_sdk_sample_setup(test_patch):
     assert result.namespace == "ci_tools"
     assert "pytyped" in result.package_data
     assert result.include_package_data == True
+    assert result.folder == package_root