remove the implicit namespace packages

Author: aignas

python/config_settings/BUILD.bazel

@@ -217,6 +217,15 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "is_venvs_site_packages",
+    flag_values = {
+        ":venvs_site_packages": VenvsSitePackages.YES,
+    },
+    # NOTE: Only public because it is used in whl_library repos.
+    visibility = ["//visibility:public"],
+)
+
 define_pypi_internal_flags(
     name = "define_pypi_internal_flags",
 )

python/private/pypi/namespace_pkgs.bzl

@@ -1,38 +1,59 @@
 """Utilities to get where we should write namespace pkg paths."""
 
-EXTS = [
-    ".py",
-    ".pyd",
-    ".so",
-    ".pyc",
-]
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
+
+_ext = struct(
+    py = ".py",
+    pyi = ".pyi",
+    pyd = ".pyd",
+    so = ".so",
+    pyc = ".pyc",
+)
+
+_INIT_CONTENTS = """\
+# __path__ manipulation added by bazel-contrib/rules_python to support namespace pkgs.
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+""".split("\n")
 
 def _add_all(dirname, dirs):
     dir_path = "."
     for dir_name in dirname.split("/"):
         dir_path = "{}/{}".format(dir_path, dir_name)
         dirs[dir_path[2:]] = None
 
-def _get_files(files, ignored_dirnames = [], root = None):
+def get_files(*, srcs, ignored_dirnames = [], root = None):
+    """Get the list of filenames to write the namespace pkg files.
+
+    Args:
+        srcs: {type}`src` a list of files to be passed to {bzl:obj}`py_library`
+            as `srcs` and `data`. This is usually a result of a {obj}`glob`.
+        ignored_dirnames: {type}`str` a list of patterns to ignore.
+        root: {type}`str` the prefix to use as the root.
+
+    Returns:
+        {type}`src` a list of paths to write the namespace pkg `__init__.py` file.
+    """
     dirs = {}
     ignored = {i: None for i in ignored_dirnames}
 
     if root:
         _add_all(root, ignored)
 
-    for file in files:
+    for file in srcs:
         dirname, _, filename = file.rpartition("/")
 
         if filename == "__init__.py":
             ignored[dirname] = None
             dirname, _, _ = dirname.rpartition("/")
-        elif filename.endswith(EXTS[0]):
+        elif filename.endswith(_ext.py):
+            pass
+        elif filename.endswith(_ext.pyc):
             pass
-        elif filename.endswith(EXTS[1]):
+        elif filename.endswith(_ext.pyd):
             pass
-        elif filename.endswith(EXTS[2]):
+        elif filename.endswith(_ext.pyi):
             pass
-        elif filename.endswith(EXTS[3]):
+        elif filename.endswith(_ext.so):
             pass
         else:
             continue
@@ -44,6 +65,25 @@ def _get_files(files, ignored_dirnames = [], root = None):
 
     return sorted([d for d in dirs if d not in ignored])
 
-namespace_pkgs = struct(
-    get_files = _get_files,
-)
+def create_inits(**kwargs):
+    """Create init files and return the list to be included `py_library` srcs.
+
+    Args:
+        **kwargs: passed to {obj}`get_files`.
+
+    Returns:
+        {type}`list[str]` to be included as part of `py_library`.
+    """
+    srcs = []
+    for out in get_files(**kwargs):
+        src = "{}/__init__.py".format(out)
+        srcs.append(srcs)
+
+        write_file(
+            name = "_gen_{}_namespace".format(out),
+            out = src,
+            content = _INIT_CONTENTS,
+            **kwargs
+        )
+
+    return srcs

python/private/pypi/whl_installer/arguments.py

@@ -57,11 +57,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser:
         action="store",
         help="Additional data exclusion parameters to add to the pip packages BUILD file.",
     )
-    parser.add_argument(
-        "--enable_implicit_namespace_pkgs",
-        action="store_true",
-        help="Disables conversion of implicit namespace packages into pkg-util style packages.",
-    )
     parser.add_argument(
         "--environment",
         action="store",

python/private/pypi/whl_installer/namespace_pkgs.py

@@ -27,7 +27,7 @@
 
 from pip._vendor.packaging.utils import canonicalize_name
 
-from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel
+from python.private.pypi.whl_installer import arguments, wheel
 
 
 def _configure_reproducible_wheels() -> None:
@@ -77,35 +77,10 @@ def _parse_requirement_for_extra(
     return None, None
 
 
-def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
-    """Converts native namespace packages to pkgutil-style packages
-
-    Namespace packages can be created in one of three ways. They are detailed here:
-    https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
-
-    'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
-    'native namespace packages' (1) do not.
-
-    We ensure compatibility with Bazel of method 1 by converting them into method 2.
-
-    Args:
-        wheel_dir: the directory of the wheel to convert
-    """
-
-    namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
-        wheel_dir,
-        ignored_dirnames=["%s/bin" % wheel_dir],
-    )
-
-    for ns_pkg_dir in namespace_pkg_dirs:
-        namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
-
-
 def _extract_wheel(
     wheel_file: str,
     extras: Dict[str, Set[str]],
     enable_pipstar: bool,
-    enable_implicit_namespace_pkgs: bool,
     platforms: List[wheel.Platform],
     installation_dir: Path = Path("."),
 ) -> None:
@@ -116,15 +91,11 @@ def _extract_wheel(
         installation_dir: the destination directory for installation of the wheel.
         extras: a list of extras to add as dependencies for the installed wheel
         enable_pipstar: if true, turns off certain operations.
-        enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
     """
 
     whl = wheel.Wheel(wheel_file)
     whl.unzip(installation_dir)
 
-    if not enable_implicit_namespace_pkgs:
-        _setup_namespace_pkg_compatibility(installation_dir)
-
     metadata = {
         "entry_points": [
             {
@@ -168,7 +139,6 @@ def main() -> None:
             wheel_file=whl,
             extras=extras,
             enable_pipstar=args.enable_pipstar,
-            enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
             platforms=arguments.get_platforms(args),
         )
         return

python/private/pypi/whl_installer/wheel_installer.py

@@ -173,9 +173,6 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
             json.encode(struct(arg = rctx.attr.pip_data_exclude)),
         ]
 
-    if rctx.attr.enable_implicit_namespace_pkgs:
-        args.append("--enable_implicit_namespace_pkgs")
-
     env = {}
     if rctx.attr.environment != None:
         for key, value in rctx.attr.environment.items():
@@ -389,6 +386,8 @@ def _whl_library_impl(rctx):
             metadata_name = metadata.name,
             metadata_version = metadata.version,
             requires_dist = metadata.requires_dist,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
             data_exclude = rctx.attr.pip_data_exclude,
@@ -457,6 +456,8 @@ def _whl_library_impl(rctx):
             name = whl_path.basename,
             dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
             entry_points = entry_points,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             dependencies = metadata["deps"],
             dependencies_by_platform = metadata["deps_by_platform"],