From 352d7db08984ed6ce9863a6885861a4322387d6a Mon Sep 17 00:00:00 2001 From: Kivi Kaitaniemi Date: Thu, 25 Apr 2024 01:44:15 +0300 Subject: [PATCH] Improve package building --- src/decman/config.py | 27 +- src/decman/lib/__init__.py | 23 +- src/decman/lib/aur.py | 676 ++++++++++++++++++------------- tests/test_package_management.py | 12 +- 4 files changed, 448 insertions(+), 290 deletions(-) diff --git a/src/decman/config.py b/src/decman/config.py index b4e1600..ea52375 100644 --- a/src/decman/config.py +++ b/src/decman/config.py @@ -141,22 +141,39 @@ class Commands: """ return ["less", file] - def make_chroot(self, chroot_root_dir: str, - with_pkgs: list[str]) -> list[str]: + def make_chroot(self, chroot_dir: str, with_pkgs: list[str]) -> list[str]: """ Running this command creates a new arch chroot to the chroot directory and installs the given packages there. """ - return ["mkarchroot", chroot_root_dir] + with_pkgs + return ["mkarchroot", chroot_dir] + with_pkgs - def make_chroot_pkg(self, chroot_dir: str, user: str, + def install_chroot_packages(self, chroot_dir: str, packages: list[str]): + """ + Running this command installs the given packages to the given chroot. + """ + return [ + "arch-nspawn", chroot_dir, "pacman", "-S", "--needed", + "--noconfirm" + ] + packages + + def remove_chroot_packages(self, chroot_dir: str, packages: list[str]): + """ + Running this command removes the given packages from the given chroot. + """ + return ["arch-nspawn", chroot_dir, "pacman", "-Rsu", "--noconfirm" + ] + packages + + def make_chroot_pkg(self, chroot_wd_dir: str, user: str, pkgfiles_to_install: list[str]) -> list[str]: """ Running this command creates a package file using the given chroot. The package is created as the user and the pkg_files_to_install are installed in the chroot before the package is created. """ - makechrootpkg_cmd = ["makechrootpkg", "-r", chroot_dir, "-U", user] + makechrootpkg_cmd = [ + "makechrootpkg", "-c", "-r", chroot_wd_dir, "-U", user + ] for pkgfile in pkgfiles_to_install: makechrootpkg_cmd += ["-I", pkgfile] diff --git a/src/decman/lib/__init__.py b/src/decman/lib/__init__.py index 0c97e43..de58605 100644 --- a/src/decman/lib/__init__.py +++ b/src/decman/lib/__init__.py @@ -18,6 +18,13 @@ _GRAY_PREFIX = "\033[90m" _RESET_SUFFIX = "\033[m" +def print_continuation(msg: str): + """ + Prints a message without a prefix. + """ + print(f"{_DECMAN_MSG_TAG}\t {msg}") + + def print_error(error_msg: str): """ Prints an error message to the user. @@ -218,6 +225,9 @@ class Pacman: Interface for interacting with pacman. """ + def __init__(self): + self._installable = {} + def get_installed(self) -> list[str]: """ Returns a list of installed packages. @@ -239,9 +249,14 @@ class Pacman: """ Returns True if a dependency can be installed using pacman. """ - return subprocess.run(conf.commands.is_installable(dep), - check=False, - capture_output=True).returncode == 0 + if dep in self._installable: + return self._installable[dep] + + result = subprocess.run(conf.commands.is_installable(dep), + check=False, + capture_output=True).returncode == 0 + self._installable[dep] = result + return result def get_versioned_foreign_packages(self) -> list[tuple[str, str]]: """ @@ -295,7 +310,7 @@ class Pacman: subprocess.run( conf.commands.set_as_explicitly_installed(as_explicit), check=True, - capture_output=True) + capture_output=conf.quiet_output) except subprocess.CalledProcessError as error: raise UserFacingError( "Failed to install foreign packages.") from error diff --git a/src/decman/lib/aur.py b/src/decman/lib/aur.py index 08492ff..5e447fc 100644 --- a/src/decman/lib/aur.py +++ b/src/decman/lib/aur.py @@ -2,15 +2,12 @@ Module for interacting with the AUR. Optional dependencies are ignored when installing AUR packages. -Make and check dependencies are grouped together. Terminology: - package (pkg): name of an package from pacman repos or AUR - dependency (dep): (virtual) package required when building and running a package - dependency package (dep pkg): dependency that has been resolved to a package name -- build dependency: (virtual) package required when building a package (makedepends + checkdepends) -- build dependency package: build dependency that has been resolved to a package name - all dependencies: normal dependencies and build dependencies combined """ @@ -34,6 +31,24 @@ def strip_dependency(dep: str) -> str: return rx.sub("", dep) +def is_devel(package: str) -> bool: + """ + Returns True if the given package is a devel package. + """ + devel_suffixes = [ + "-git", + "-hg", + "-bzr", + "-svn", + "-cvs", + "-darcs", + ] + for suffix in devel_suffixes: + if package.endswith(suffix): + return True + return False + + class PackageInfo: """ Simplified information about an package. @@ -43,17 +58,41 @@ class PackageInfo: def __init__(self, pkgname: str, pkgbase: str, version: str, provides: list[str], dependencies: list[str], - make_and_check_dependencies: list[str], git_url: str): + make_dependencies: list[str], check_dependencies: list[str], + git_url: str, pacman: l.Pacman): self.pkgname = pkgname self.pkgbase = pkgbase self.version = version - self.dependencies = dependencies - self.build_dependencies = make_and_check_dependencies self.provides = provides self.git_url = git_url - self._aur_deps = None - self._pacman_deps = None - self._pacman_all_deps = None + + self.foreign_dependencies_stripped = [] + self.foreign_make_dependencies_stripped = [] + self.foreign_check_dependencies_stripped = [] + self.pacman_dependencies = [] + self.pacman_make_dependencies = [] + self.pacman_check_dependencies = [] + + for dep in dependencies: + if pacman.is_installable(dep): + self.pacman_dependencies.append(dep) + else: + self.foreign_dependencies_stripped.append( + strip_dependency(dep)) + + for make_dep in make_dependencies: + if pacman.is_installable(make_dep): + self.pacman_make_dependencies.append(make_dep) + else: + self.foreign_make_dependencies_stripped.append( + strip_dependency(make_dep)) + + for check_dep in check_dependencies: + if pacman.is_installable(check_dep): + self.pacman_check_dependencies.append(check_dep) + else: + self.foreign_check_dependencies_stripped.append( + strip_dependency(check_dep)) def pkg_file_prefix(self) -> str: """ @@ -61,71 +100,10 @@ class PackageInfo: """ return f"{self.pkgname}-{self.version}" - def all_foreign_dependencies_stripped(self, pacman: l.Pacman) -> list[str]: - """ - Returs a list of dependencies that cannot be installed from pacman repos. - Includes build dependencies. - - Removes version spefications from package names. - """ - if self._aur_deps is not None: - return self._aur_deps - - result = [] - - for p in self.dependencies: - if not pacman.is_installable(p): - result.append(strip_dependency(p)) - - for p in self.build_dependencies: - if not pacman.is_installable(p): - result.append(strip_dependency(p)) - - self._aur_deps = result - return result - - def all_pacman_dependencies(self, pacman: l.Pacman) -> list[str]: - """ - Returs a list of dependencies that can be installed from pacman repos. - Includes build dependencies. - """ - if self._pacman_all_deps is not None: - return self._pacman_all_deps - - result = [] - - for p in self.dependencies: - if pacman.is_installable(p): - result.append(strip_dependency(p)) - - for p in self.build_dependencies: - if pacman.is_installable(p): - result.append(strip_dependency(p)) - - self._pacman_all_deps = result - return result - - def pacman_dependencies(self, pacman: l.Pacman) -> list[str]: - """ - Returs a list of dependencies that can be installed from pacman repos. - Doesn't include build dependencies. - """ - if self._pacman_deps is not None: - return self._pacman_deps - - result = [] - - for p in self.dependencies: - if pacman.is_installable(p): - result.append(strip_dependency(p)) - - self._pacman_deps = result - return result - class ForeignPackage: """ - Class used to keep track of AUR/user dependency packages of an AUR/user package. + Class used to keep track of AUR/user recursive dependency packages of an AUR/user package. """ def __init__(self, name: str): @@ -202,6 +180,9 @@ class DepGraph: child_pkgname, DepNode(ForeignPackage(child_pkgname))) self.package_nodes[child_pkgname] = child_node + if len(child_node.children) == 0: + self._childless_node_names.add(child_pkgname) + if parent_pkgname is None: return @@ -210,9 +191,8 @@ class DepGraph: if parent_node.is_pkgname_in_parents_recursive(child_pkgname): raise l.UserFacingError( f"Foreign package dependency cycle detected involving '{child_pkgname}' \ - and '{parent_pkgname}'. Foreign package dependencies are also required \ - during package building and therefore dependency cycles cannot be handled." - ) +and '{parent_pkgname}'. Foreign package dependencies are also required \ +during package building and therefore dependency cycles cannot be handled.") parent_node.children[child_pkgname] = child_node child_node.parents[parent_pkgname] = parent_node @@ -220,9 +200,6 @@ class DepGraph: if parent_pkgname in self._childless_node_names: self._childless_node_names.remove(parent_pkgname) - if len(child_node.children) == 0: - self._childless_node_names.add(child_pkgname) - def get_and_remove_outer_dep_pkgs(self) -> list[ForeignPackage]: """ Returns all childless nodes of the dependency package graph and removes them. @@ -252,7 +229,8 @@ class ExtendedPackageSearch: Results are cached and user defined packages are preferred. """ - def __init__(self): + def __init__(self, pacman: l.Pacman): + self._pacman = pacman self._package_info_cache: dict[str, PackageInfo] = {} self._dep_provider_cache: dict[str, PackageInfo] = {} self._user_packages: list[PackageInfo] = [] @@ -268,6 +246,12 @@ class ExtendedPackageSearch: Tried caching the given packages. Virtual packages may not be cached """ + packages = list( + filter(lambda p: p not in self._package_info_cache, packages)) + + if len(packages) == 0: + return + l.print_debug(f"Trying to cache {packages}.") max_pkgs_per_request = 200 @@ -306,13 +290,12 @@ class ExtendedPackageSearch: pkgbase=result["PackageBase"], version=result["Version"], dependencies=result.get("Depends", []), - make_and_check_dependencies=result.get( - "MakeDepends", []) + - result.get("CheckDepends", []), + make_dependencies=result.get("MakeDepends", []), + check_dependencies=result.get("CheckDepends", []), provides=result.get("Provides", []), git_url= - f"https://aur.archlinux.org/{result['PackageBase']}.git" - ) + f"https://aur.archlinux.org/{result['PackageBase']}.git", + pacman=self._pacman) self._package_info_cache[pkgname] = info l.print_debug("Request completed.") @@ -361,11 +344,12 @@ class ExtendedPackageSearch: pkgbase=result["PackageBase"], version=result["Version"], dependencies=result.get("Depends", []), - make_and_check_dependencies=result.get("MakeDepends", []) + - result.get("CheckDepends", []), + make_dependencies=result.get("MakeDepends", []), + check_dependencies=result.get("CheckDepends", []), provides=result.get("Provides", []), - git_url=f"https://aur.archlinux.org/{result['PackageBase']}.git" - ) + git_url= + f"https://aur.archlinux.org/{result['PackageBase']}.git", + pacman=self._pacman) self._package_info_cache[package] = info @@ -472,6 +456,56 @@ class ExtendedPackageSearch: return info +class ResolvedDependencies: + """ + Result of dependency resolution. + """ + + def __init__(self): + self.pacman_deps: set[str] = set() + self.foreign_pkgs: set[str] = set() + self.foreign_dep_pkgs: set[str] = set() + self.foreign_build_dep_pkgs: set[str] = set() + self.build_order: list[str] = [] + self.packages: dict[str, ForeignPackage] = {} + self._pkgbases_to_pkgs: dict[str, set[str]] = {} + self._pkgs_to_pkgbases: dict[str, str] = {} + + def add_pkgbase_info(self, pkgname: str, pkgbase: str): + """ + Adds information about a which package belongs in which package base. + """ + pkgs = self._pkgbases_to_pkgs.get(pkgbase, set()) + pkgs.add(pkgname) + self._pkgbases_to_pkgs[pkgbase] = pkgs + self._pkgs_to_pkgbases[pkgname] = pkgbase + + def get_pkgbase(self, pkgname: str) -> str: + """ + Returns the package base of an package. + """ + return self._pkgs_to_pkgbases[pkgname] + + def get_pkgs_with_common_pkgbase(self, pkgname: str) -> set[str]: + """ + Returns all packages that have the same package base as the given package. + """ + pkgbase = self._pkgs_to_pkgbases[pkgname] + return self._pkgbases_to_pkgs[pkgbase] + + def all_pkgbases(self) -> list[str]: + """ + Returns all pkgbases. + """ + return list(self._pkgbases_to_pkgs) + + def get_some_pkgname(self, pkgbase: str) -> str: + """ + Returns some package name that the given pkgbase has. + """ + return list(self._pkgbases_to_pkgs[pkgbase])[0] + + class ForeignPackageManager: """ Class for dealing with AUR/user packages. @@ -519,56 +553,84 @@ class ForeignPackageManager: Installs the given AUR/user packages and their dependencies (both pacman/AUR). """ - if as_explicit is None: - as_explicit = foreign_pkgs + if len(foreign_pkgs) == 0: + return - all_foreign_pkgs, pacman_deps = self.resolve_dependencies(foreign_pkgs) + resolved_dependencies = self.resolve_dependencies(foreign_pkgs) l.print_summary( - f"The following foreign packages will be installed: {' '.join(map(lambda p: p.name, all_foreign_pkgs))}" - ) + "The following foreign packages will be installed explicitly:") + l.print_continuation("") + l.print_continuation( + f"\t{' '.join(resolved_dependencies.foreign_pkgs)}") + l.print_continuation("") + + if resolved_dependencies.foreign_dep_pkgs: + l.print_summary( + "The following foreign packages will be installed as dependencies:" + ) + l.print_continuation("") + l.print_continuation( + f"\t{' '.join(resolved_dependencies.foreign_dep_pkgs)}") + l.print_continuation("") + + if resolved_dependencies.foreign_build_dep_pkgs: + l.print_summary( + "The following foreign packages will be built in order to install other packages. They will not be installed:" + ) + l.print_continuation("") + l.print_continuation( + f"\t{' '.join(resolved_dependencies.foreign_build_dep_pkgs)}") + l.print_continuation("") if not l.prompt_confirm("Proceed?", default=True): raise l.UserFacingError("Installing aborted.") l.print_summary( "Installing AUR/user package dependencies from pacman.") - self._pacman.install_dependencies(list(pacman_deps)) + self._pacman.install_dependencies( + list(resolved_dependencies.pacman_deps)) - to_install = [] + try: + with PackageBuilder(self._search, self._store, + resolved_dependencies) as builder: + while resolved_dependencies.build_order: + to_build = resolved_dependencies.build_order.pop(0) - while all_foreign_pkgs: - pkg_to_build = all_foreign_pkgs.pop(0) + pkgbase = resolved_dependencies.get_pkgbase(to_build) + package_names = resolved_dependencies.get_pkgs_with_common_pkgbase( + to_build) - # resolve_dependencies gets info for every package so this cannot be None - pkgbase = self._search.get_package_info( - pkg_to_build.name - ).pkgbase # pyright: ignore[reportOptionalMemberAccess] - with_same_pkgbase = [] + packages = [ + resolved_dependencies.packages[pkgname] + for pkgname in package_names + ] - for other in all_foreign_pkgs: - other_pkgbase = self._search.get_package_info( - other.name - ).pkgbase # pyright: ignore[reportOptionalMemberAccess] - if other_pkgbase == pkgbase: - with_same_pkgbase.append(other) + builder.build_packages(pkgbase, packages, force) + except (subprocess.CalledProcessError, OSError) as e: + raise l.UserFacingError("Failed to build packages.") from e - for other in with_same_pkgbase: - all_foreign_pkgs.remove(other) + if as_explicit is None: + as_explicit = list(resolved_dependencies.foreign_pkgs) - to_install += self._build_pkg(pkgbase, - [pkg_to_build] + with_same_pkgbase, - force) + packages_to_install = list(resolved_dependencies.foreign_pkgs) + packages_to_install += list(resolved_dependencies.foreign_dep_pkgs) - if to_install or force: + package_files_to_install = [] + for pkg in packages_to_install: + built_pkg = self._store.get_package(pkg) + assert built_pkg is not None + _, path = built_pkg + package_files_to_install.append(path) + + if package_files_to_install or force: l.print_summary("Installing AUR/user packages.") - self._pacman.install_files(to_install, as_explicit) + self._pacman.install_files(package_files_to_install, as_explicit) else: l.print_summary("No packages to install.") def resolve_dependencies( - self, foreign_packages: list[str] - ) -> tuple[list[ForeignPackage], set[str]]: + self, foreign_packages: list[str]) -> ResolvedDependencies: """ Resolves AUR/user dependencies of AUR/user packages. @@ -583,7 +645,9 @@ class ForeignPackageManager: l.print_summary("Resolving AUR / user package dependencies.") l.print_debug(f"Packages: {foreign_packages}") - pacman_deps = set() + result = ResolvedDependencies() + result.foreign_pkgs = set(foreign_packages) + graph = DepGraph() for name in foreign_packages: @@ -593,6 +657,23 @@ class ForeignPackageManager: to_process = list(foreign_packages) total_processed = 0 + def process_dep(pkgname: str, depname: str, add_to: set[str]): + dep_info = self._search.find_provider(depname) + + if dep_info is None: + raise l.UserFacingError( + f"Failed to find '{depname}' from AUR or user provided packages." + ) + + add_to.add(dep_info.pkgname) + + l.print_debug( + f"Adding dependency {dep_info.pkgname} to package {pkgname}.") + graph.add_requirement(dep_info.pkgname, pkgname) + if dep_info.pkgname not in seen_packages: + to_process.append(dep_info.pkgname) + seen_packages.add(dep_info.pkgname) + while to_process: pkgname = to_process.pop() @@ -602,36 +683,25 @@ class ForeignPackageManager: f"Failed to find '{pkgname}' from AUR or user provided packages." ) - pacman_deps.update(info.pacman_dependencies(self._pacman)) - depnames = info.all_foreign_dependencies_stripped(self._pacman) - self._search.try_caching_packages(depnames) + result.pacman_deps.update(info.pacman_dependencies) + result.add_pkgbase_info(pkgname, info.pkgbase) - for depname in depnames: - dep_info = self._search.find_provider(depname) + build_deps = info.foreign_make_dependencies_stripped + info.foreign_check_dependencies_stripped - if dep_info is None: - raise l.UserFacingError( - f"Failed to find '{depname}' from AUR or user provided packages." - ) + self._search.try_caching_packages( + info.foreign_dependencies_stripped + build_deps) - l.print_debug( - f"Adding dependency {dep_info.pkgname} to package {pkgname}." - ) - graph.add_requirement(dep_info.pkgname, pkgname) - if dep_info.pkgname not in seen_packages: - to_process.append(dep_info.pkgname) - seen_packages.add(dep_info.pkgname) + for depname in info.foreign_dependencies_stripped: + process_dep(pkgname, depname, result.foreign_dep_pkgs) + + for depname in build_deps: + process_dep(pkgname, depname, result.foreign_build_dep_pkgs) total_processed += 1 l.print_info(f"{total_processed}/{len(seen_packages)}.") l.print_summary("Determining build order.") - build_order = self._determine_build_order(graph) - return (build_order, pacman_deps) - - def _determine_build_order(self, graph: DepGraph) -> list[ForeignPackage]: - build_order = [] while True: to_add = graph.get_and_remove_outer_dep_pkgs() @@ -639,16 +709,134 @@ class ForeignPackageManager: break for pkg in to_add: - if pkg not in build_order: + if pkg not in result.packages: l.print_debug(f"Adding {pkg} to build_order.") - build_order.append(pkg) + result.build_order.append(pkg.name) + result.packages[pkg.name] = pkg - return build_order + return result - def _build_pkg(self, package_base: str, packages: list[ForeignPackage], - force: bool) -> list[str]: + def should_upgrade_package(self, + package: str, + installed_version: str, + fetched_version: str, + upgrade_devel=False) -> bool: """ - Builds package(s) with the same package base. Returns a list of package files to install. + Returns True if a package should be upgraded. + """ + + if upgrade_devel and is_devel(package): + return True + + try: + result = int( + subprocess.run(conf.commands.compare_versions( + installed_version, fetched_version), + check=True, + stdout=subprocess.PIPE).stdout.decode()) + return result < 0 + except (ValueError, subprocess.CalledProcessError) as error: + raise l.UserFacingError("Failed to compare versions.") from error + + +class PackageBuilder: + """ + Used for building packages in a chroot. + """ + + always_included_packages = ["base-devel", "git"] + + def __init__(self, search: ExtendedPackageSearch, store: l.Store, + resolved_deps: ResolvedDependencies): + self._search = search + self._store = store + self._resolved_deps = resolved_deps + self.chroot_wd_dir = os.path.join(conf.build_dir, "chroot") + self.chroot_dir = os.path.join(self.chroot_wd_dir, "root") + self.pkgbase_dir_map = {} + self.original_wd = "" + + def __enter__(self): + self.store_wd() + self.create_build_environment() + + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.restore_wd() + self.remove_build_environment() + + def store_wd(self): + """ + Remembers the current working directory as the original working directory. + """ + self.original_wd = os.getcwd() + + def restore_wd(self): + """ + Returns to the original working directory. + """ + os.chdir(self.original_wd) + + def create_build_environment(self): + """ + Creates a new chroot and clones all PKGBUILDS. + """ + l.print_summary("Creating a build environment..") + + if os.path.exists(conf.build_dir): + l.print_info("Removing previous build directory.") + self.remove_build_environment() + + l.print_info("Getting all PKGBUILDS.") + + # Set up PKGBUILDS + for pkgbase in self._resolved_deps.all_pkgbases(): + pkgbuild_dir = os.path.join(conf.build_dir, pkgbase) + self.pkgbase_dir_map[pkgbase] = pkgbuild_dir + os.makedirs(pkgbuild_dir) + os.chdir(pkgbuild_dir) + + git_url = self._search.get_package_info( + self._resolved_deps.get_some_pkgname(pkgbase) + ).git_url # pyright: ignore[reportOptionalMemberAccess] + + l.print_debug(f"Git URL for '{pkgbase}' is '{git_url}'") + self._git_clone_and_review_pkgbuild(pkgbase, git_url) + shutil.chown(pkgbuild_dir, user=conf.makepkg_user) + + l.print_summary("Creating a new chroot.") + os.makedirs(self.chroot_wd_dir) + + # Remove GNUPGHOME from mkarchroot environment variables since it may interfere with + # the chroot creation + mkarchroot_env_vars = os.environ.copy() + try: + del mkarchroot_env_vars["GNUPGHOME"] + l.print_debug( + "Removed GNUPGHOME variable from mkarchroot environment.") + except KeyError: + pass + + subprocess.run(conf.commands.make_chroot( + self.chroot_dir, PackageBuilder.always_included_packages + + list(self._resolved_deps.pacman_deps)), + env=mkarchroot_env_vars, + check=True, + capture_output=conf.quiet_output) + + def remove_build_environment(self): + """ + Deletes the build environment. + """ + shutil.rmtree(conf.build_dir) + + def build_packages(self, package_base: str, packages: list[ForeignPackage], + force: bool): + """ + Builds package(s) with the same package base. + + Set force to true to force rebuilds of packages that are already cached """ package_names = list(map(lambda p: p.name, packages)) @@ -659,99 +847,58 @@ class ForeignPackageManager: l.print_summary( f"Skipped building '{' '.join(package_names)}'. Already up to date." ) - return [] + return l.print_summary(f"To build '{' '.join(package_names)}'.") - chroot_pacman_pkgs, chroot_pkg_files = self._get_chroot_packages( + chroot_new_pacman_pkgs, chroot_pkg_files = self._get_chroot_packages( packages) - chroot_dir = os.path.join(conf.build_dir, "chroot") - pkgbuild_dir = os.path.join(conf.build_dir, "pkgbuild") + pkgbuild_dir = self.pkgbase_dir_map[package_base] + os.chdir(pkgbuild_dir) l.print_debug( - f"Chroot dir is: '{chroot_dir}', pkgbuild dir is '{pkgbuild_dir}'." + f"Chroot dir is: '{self.chroot_dir}', pkgbuild dir is '{pkgbuild_dir}'." ) - prev_wd = os.getcwd() + l.print_info("Installing build dependencies to chroot.") - try: - os.makedirs(conf.pkg_cache_dir, exist_ok=True) + subprocess.run(conf.commands.install_chroot_packages( + self.chroot_dir, + chroot_new_pacman_pkgs + PackageBuilder.always_included_packages), + check=True, + capture_output=conf.quiet_output) - if os.path.exists(conf.build_dir): - l.print_info("Removing previous build directory.") - shutil.rmtree(conf.build_dir) + l.print_info("Making package.") - l.print_info("Setting up build directory.") - os.makedirs(pkgbuild_dir) - os.makedirs(chroot_dir) + subprocess.run(conf.commands.make_chroot_pkg(self.chroot_wd_dir, + conf.makepkg_user, + chroot_pkg_files), + check=True, + capture_output=conf.quiet_output) - os.chdir(pkgbuild_dir) + for pkgname in package_names: + file = self._find_pkgfile(pkgname, pkgbuild_dir) - git_url = self._search.get_package_info( - package_names[0] - ).git_url # pyright: ignore[reportOptionalMemberAccess] + dest = shutil.copy(file, conf.pkg_cache_dir) - l.print_debug(f"Git URL for '{package_base}' is '{git_url}'") + version = self._search.get_package_info( + pkgname).version # pyright: ignore[reportOptionalMemberAccess] - self.git_clone_and_review_pkgbuild(package_base, git_url) - shutil.chown(pkgbuild_dir, user=conf.makepkg_user) + l.print_debug( + f"Adding '{pkgname}', version: '{version}' to cache as file '{dest}'." + ) - l.print_summary(f"Building: '{' '.join(package_names)}'.") + self._store.add_package_to_cache(pkgname, version, dest) - # Remove GNUPGHOME from mkarchroot environment variables since it may interfere with - # the chroot creation - mkarchroot_env_vars = os.environ.copy() - try: - del mkarchroot_env_vars["GNUPGHOME"] - l.print_debug( - "Removed GNUPGHOME variable from mkarchroot environment.") - except KeyError: - pass + l.print_info("Removing build dependencies from chroot.") - l.print_info("Creating a new chroot.") + subprocess.run(conf.commands.remove_chroot_packages( + self.chroot_dir, chroot_new_pacman_pkgs), + check=True, + capture_output=conf.quiet_output) - subprocess.run(conf.commands.make_chroot( - os.path.join(chroot_dir, "root"), - ["base-devel"] + chroot_pacman_pkgs), - env=mkarchroot_env_vars, - check=True, - capture_output=conf.quiet_output) - - l.print_info("Making package.") - - subprocess.run(conf.commands.make_chroot_pkg( - chroot_dir, conf.makepkg_user, chroot_pkg_files), - check=True, - capture_output=conf.quiet_output) - - package_files = [] - - for pkgname in package_names: - file = self._find_pkgfile(pkgname, pkgbuild_dir) - - dest = shutil.copy(file, conf.pkg_cache_dir) - - version = self._search.get_package_info( - pkgname - ).version # pyright: ignore[reportOptionalMemberAccess] - - l.print_debug( - f"Adding '{pkgname}', version: '{version}' to cache as file '{dest}'." - ) - - self._store.add_package_to_cache(pkgname, version, dest) - package_files.append(dest) - - l.print_summary(f"Finished building: '{' '.join(package_names)}'.") - except (subprocess.CalledProcessError, OSError) as error: - raise l.UserFacingError( - f"Failed to build package(s) '{' '.join(map(lambda p: p.name, packages))}'." - ) from error - finally: - os.chdir(prev_wd) - - return package_files + l.print_summary(f"Finished building: '{' '.join(package_names)}'.") def _are_all_pkgs_cached(self, pkgs: list[ForeignPackage]) -> bool: for pkg in pkgs: @@ -764,7 +911,7 @@ class ForeignPackageManager: pkg.name ).version # pyright: ignore[reportOptionalMemberAccess] - if cached_version != fetched_version or self.is_devel(pkg.name): + if cached_version != fetched_version or is_devel(pkg.name): return False return True @@ -772,18 +919,23 @@ class ForeignPackageManager: self, pkgs_to_build: list[ForeignPackage] ) -> tuple[list[str], list[str]]: """ - Returns a tuple of pacman packages and built foreign pkgs files that are needed in the - chroot before building. pkgs_to_build share the same pkgbase. + Returns a tuple of pacman build dependencies and built foreign pkgs files that are needed + in the chroot before building. pkgs_to_build share the same pkgbase. """ - chroot_pacman_pkgs = set() + chroot_pacman_build_deps = set() chroot_foreign_pkgs = set() + def add_to_pacman_build_deps(deps: list[str]): + for dep in deps: + if dep not in self._resolved_deps.pacman_deps: + chroot_pacman_build_deps.add(dep) + for pkg in pkgs_to_build: info = self._search.get_package_info(pkg.name) assert info is not None - chroot_pacman_pkgs.update( - info.all_pacman_dependencies(self._pacman)) + add_to_pacman_build_deps(info.pacman_make_dependencies) + add_to_pacman_build_deps(info.pacman_check_dependencies) foreign_deps = pkg.get_all_recursive_foreign_deps() chroot_foreign_pkgs.update(foreign_deps) @@ -792,8 +944,9 @@ class ForeignPackageManager: for dep in foreign_deps: dep_info = self._search.get_package_info(dep) assert dep_info is not None - chroot_pacman_pkgs.update( - dep_info.all_pacman_dependencies(self._pacman)) + + add_to_pacman_build_deps(dep_info.pacman_make_dependencies) + add_to_pacman_build_deps(dep_info.pacman_check_dependencies) # Packages with the same pkgbase might depend on each other, # but they don't need to be installed for the build to succeed. @@ -812,7 +965,7 @@ class ForeignPackageManager: chroot_foreign_pkg_files.append(file) - return (list(chroot_pacman_pkgs), chroot_foreign_pkg_files) + return (list(chroot_pacman_build_deps), chroot_foreign_pkg_files) def _find_pkgfile(self, pkgname: str, pkgbuild_dir: str) -> str: # HACK: Because we don't know the pkgarch we can't be sure what is the build result. @@ -838,27 +991,33 @@ class ForeignPackageManager: return matches[0] - def git_clone_and_review_pkgbuild(self, pkgbase: str, git_url: str): + def _git_clone_and_review_pkgbuild(self, pkgbase: str, git_url: str): """ Clones an PKGBUILD to the current directory. The user is prompted to review the PKGBUILD and confirm if the package should be built. """ try: - subprocess.run(conf.commands.git_clone(git_url, "."), check=True) + subprocess.run(conf.commands.git_clone(git_url, "."), + check=True, + capture_output=conf.quiet_output) - latest_reviewed_commit = self._store.pkgbuild_latest_reviewed_commits.get( - pkgbase) - if latest_reviewed_commit is None: - for file in os.scandir("."): - if file.is_file() and not file.name.startswith("."): - subprocess.run(conf.commands.review_file(file.path), - check=True) - else: - subprocess.run(conf.commands.git_diff(latest_reviewed_commit), - check=True) + if l.prompt_confirm(f"Review PKGBUILD for {pkgbase}?", + default=True): + latest_reviewed_commit = self._store.pkgbuild_latest_reviewed_commits.get( + pkgbase) + if latest_reviewed_commit is None: + for file in os.scandir("."): + if file.is_file() and not file.name.startswith("."): + subprocess.run(conf.commands.review_file( + file.path), + check=True) + else: + subprocess.run( + conf.commands.git_diff(latest_reviewed_commit), + check=True) - if l.prompt_confirm("Proceed with building?", default=True): + if l.prompt_confirm("Build this package?", default=True): commit_id = subprocess.run( conf.commands.git_get_commit_id(), check=True, @@ -872,42 +1031,3 @@ class ForeignPackageManager: raise l.UserFacingError( f"Failed to clone and review PKGBUILD from {git_url}" ) from error - - def should_upgrade_package(self, - package: str, - installed_version: str, - fetched_version: str, - upgrade_devel=False) -> bool: - """ - Returns True if a package should be upgraded. - """ - - if upgrade_devel and self.is_devel(package): - return True - - try: - result = int( - subprocess.run(conf.commands.compare_versions( - installed_version, fetched_version), - check=True, - stdout=subprocess.PIPE).stdout.decode()) - return result < 0 - except (ValueError, subprocess.CalledProcessError) as error: - raise l.UserFacingError("Failed to compare versions.") from error - - def is_devel(self, package: str) -> bool: - """ - Returns True if the given package is a devel package. - """ - devel_suffixes = [ - "-git", - "-hg", - "-bzr", - "-svn", - "-cvs", - "-darcs", - ] - for suffix in devel_suffixes: - if package.endswith(suffix): - return True - return False diff --git a/tests/test_package_management.py b/tests/test_package_management.py index de4f1f2..6f33ec2 100644 --- a/tests/test_package_management.py +++ b/tests/test_package_management.py @@ -8,8 +8,9 @@ from decman.lib.aur import ForeignPackageManager, DepGraph, ForeignPackage, Exte class TestAUR(unittest.TestCase): def setUp(self) -> None: - self.aur = ForeignPackageManager(Store(), Pacman(), - ExtendedPackageSearch()) + pacman = Pacman() + self.aur = ForeignPackageManager(Store(), pacman, + ExtendedPackageSearch(pacman)) def test_should_upgrade_package_returns_true_on_newer_version(self): self.assertTrue( @@ -59,6 +60,8 @@ class TestDepGraph(unittest.TestCase): graph = DepGraph() graph.add_requirement("A", None) + graph.add_requirement("V", None) + graph.add_requirement("B1", "A") graph.add_requirement("B2", "A") graph.add_requirement("B3", "A") @@ -71,6 +74,8 @@ class TestDepGraph(unittest.TestCase): graph.add_requirement("C2", "D") + v = ForeignPackage("V") + a = ForeignPackage("A") a.add_foreign_dependency_packages(["B1", "B2", "B3", "C1", "C2", "D"]) @@ -90,7 +95,8 @@ class TestDepGraph(unittest.TestCase): d = ForeignPackage("D") d.add_foreign_dependency_packages(["C2"]) - self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [c2, b3]) + self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), + [c2, b3, v]) self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [d]) self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [c1]) self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [b1])