JustAnyone/decman
1
0
Fork 0
mirror of https://github.com/kiviktnm/decman.git synced 2026-09-19 12:08:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve package building

Browse Source
This commit is contained in:
Kivi Kaitaniemi
2024-04-25 01:44:15 +03:00
parent b1be2661e5
commit 352d7db089
4 changed files with 448 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/decman/config.py
+22 -5
View File
@@ -141,22 +141,39 @@ class Commands:
"""
return ["less", file]
def make_chroot(self, chroot_root_dir: str,
with_pkgs: list[str]) -> list[str]:
def make_chroot(self, chroot_dir: str, with_pkgs: list[str]) -> list[str]:
"""
Running this command creates a new arch chroot to the chroot directory and installs the
given packages there.
"""
return ["mkarchroot", chroot_root_dir] + with_pkgs
return ["mkarchroot", chroot_dir] + with_pkgs
def make_chroot_pkg(self, chroot_dir: str, user: str,
def install_chroot_packages(self, chroot_dir: str, packages: list[str]):
"""
Running this command installs the given packages to the given chroot.
"""
return [
"arch-nspawn", chroot_dir, "pacman", "-S", "--needed",
"--noconfirm"
] + packages
def remove_chroot_packages(self, chroot_dir: str, packages: list[str]):
"""
Running this command removes the given packages from the given chroot.
"""
return ["arch-nspawn", chroot_dir, "pacman", "-Rsu", "--noconfirm"
] + packages
def make_chroot_pkg(self, chroot_wd_dir: str, user: str,
pkgfiles_to_install: list[str]) -> list[str]:
"""
Running this command creates a package file using the given chroot.
The package is created as the user and the pkg_files_to_install are installed
in the chroot before the package is created.
"""
makechrootpkg_cmd = ["makechrootpkg", "-r", chroot_dir, "-U", user]
makechrootpkg_cmd = [
"makechrootpkg", "-c", "-r", chroot_wd_dir, "-U", user
]
for pkgfile in pkgfiles_to_install:
makechrootpkg_cmd += ["-I", pkgfile]
src/decman/lib/__init__.py
+17 -2
View File
@@ -18,6 +18,13 @@ _GRAY_PREFIX = "\033[90m"
_RESET_SUFFIX = "\033[m"
def print_continuation(msg: str):
"""
Prints a message without a prefix.
"""
print(f"{_DECMAN_MSG_TAG}\t {msg}")
def print_error(error_msg: str):
"""
Prints an error message to the user.
@@ -218,6 +225,9 @@ class Pacman:
Interface for interacting with pacman.
"""
def __init__(self):
self._installable = {}
def get_installed(self) -> list[str]:
"""
Returns a list of installed packages.
@@ -239,9 +249,14 @@ class Pacman:
"""
Returns True if a dependency can be installed using pacman.
"""
return subprocess.run(conf.commands.is_installable(dep),
if dep in self._installable:
return self._installable[dep]
result = subprocess.run(conf.commands.is_installable(dep),
check=False,
capture_output=True).returncode == 0
self._installable[dep] = result
return result
def get_versioned_foreign_packages(self) -> list[tuple[str, str]]:
"""
@@ -295,7 +310,7 @@ class Pacman:
subprocess.run(
conf.commands.set_as_explicitly_installed(as_explicit),
check=True,
capture_output=True)
capture_output=conf.quiet_output)
except subprocess.CalledProcessError as error:
raise UserFacingError(
"Failed to install foreign packages.") from error
src/decman/lib/aur.py
+367 -247
View File
@@ -2,15 +2,12 @@
Module for interacting with the AUR.
Optional dependencies are ignored when installing AUR packages.
Make and check dependencies are grouped together.
Terminology:
- package (pkg): name of an package from pacman repos or AUR
- dependency (dep): (virtual) package required when building and running a package
- dependency package (dep pkg): dependency that has been resolved to a package name
- build dependency: (virtual) package required when building a package (makedepends + checkdepends)
- build dependency package: build dependency that has been resolved to a package name
- all dependencies: normal dependencies and build dependencies combined
"""
@@ -34,6 +31,24 @@ def strip_dependency(dep: str) -> str:
return rx.sub("", dep)
def is_devel(package: str) -> bool:
"""
Returns True if the given package is a devel package.
"""
devel_suffixes = [
"-git",
"-hg",
"-bzr",
"-svn",
"-cvs",
"-darcs",
]
for suffix in devel_suffixes:
if package.endswith(suffix):
return True
return False
class PackageInfo:
"""
Simplified information about an package.
@@ -43,17 +58,41 @@ class PackageInfo:
def __init__(self, pkgname: str, pkgbase: str, version: str,
provides: list[str], dependencies: list[str],
make_and_check_dependencies: list[str], git_url: str):
make_dependencies: list[str], check_dependencies: list[str],
git_url: str, pacman: l.Pacman):
self.pkgname = pkgname
self.pkgbase = pkgbase
self.version = version
self.dependencies = dependencies
self.build_dependencies = make_and_check_dependencies
self.provides = provides
self.git_url = git_url
self._aur_deps = None
self._pacman_deps = None
self._pacman_all_deps = None
self.foreign_dependencies_stripped = []
self.foreign_make_dependencies_stripped = []
self.foreign_check_dependencies_stripped = []
self.pacman_dependencies = []
self.pacman_make_dependencies = []
self.pacman_check_dependencies = []
for dep in dependencies:
if pacman.is_installable(dep):
self.pacman_dependencies.append(dep)
else:
self.foreign_dependencies_stripped.append(
strip_dependency(dep))
for make_dep in make_dependencies:
if pacman.is_installable(make_dep):
self.pacman_make_dependencies.append(make_dep)
else:
self.foreign_make_dependencies_stripped.append(
strip_dependency(make_dep))
for check_dep in check_dependencies:
if pacman.is_installable(check_dep):
self.pacman_check_dependencies.append(check_dep)
else:
self.foreign_check_dependencies_stripped.append(
strip_dependency(check_dep))
def pkg_file_prefix(self) -> str:
"""
@@ -61,71 +100,10 @@ class PackageInfo:
"""
return f"{self.pkgname}-{self.version}"
def all_foreign_dependencies_stripped(self, pacman: l.Pacman) -> list[str]:
"""
Returs a list of dependencies that cannot be installed from pacman repos.
Includes build dependencies.
Removes version spefications from package names.
"""
if self._aur_deps is not None:
return self._aur_deps
result = []
for p in self.dependencies:
if not pacman.is_installable(p):
result.append(strip_dependency(p))
for p in self.build_dependencies:
if not pacman.is_installable(p):
result.append(strip_dependency(p))
self._aur_deps = result
return result
def all_pacman_dependencies(self, pacman: l.Pacman) -> list[str]:
"""
Returs a list of dependencies that can be installed from pacman repos.
Includes build dependencies.
"""
if self._pacman_all_deps is not None:
return self._pacman_all_deps
result = []
for p in self.dependencies:
if pacman.is_installable(p):
result.append(strip_dependency(p))
for p in self.build_dependencies:
if pacman.is_installable(p):
result.append(strip_dependency(p))
self._pacman_all_deps = result
return result
def pacman_dependencies(self, pacman: l.Pacman) -> list[str]:
"""
Returs a list of dependencies that can be installed from pacman repos.
Doesn't include build dependencies.
"""
if self._pacman_deps is not None:
return self._pacman_deps
result = []
for p in self.dependencies:
if pacman.is_installable(p):
result.append(strip_dependency(p))
self._pacman_deps = result
return result
class ForeignPackage:
"""
Class used to keep track of AUR/user dependency packages of an AUR/user package.
Class used to keep track of AUR/user recursive dependency packages of an AUR/user package.
"""
def __init__(self, name: str):
@@ -202,6 +180,9 @@ class DepGraph:
child_pkgname, DepNode(ForeignPackage(child_pkgname)))
self.package_nodes[child_pkgname] = child_node
if len(child_node.children) == 0:
self._childless_node_names.add(child_pkgname)
if parent_pkgname is None:
return
@@ -211,8 +192,7 @@ class DepGraph:
raise l.UserFacingError(
f"Foreign package dependency cycle detected involving '{child_pkgname}' \
and '{parent_pkgname}'. Foreign package dependencies are also required \
during package building and therefore dependency cycles cannot be handled."
)
during package building and therefore dependency cycles cannot be handled.")
parent_node.children[child_pkgname] = child_node
child_node.parents[parent_pkgname] = parent_node
@@ -220,9 +200,6 @@ class DepGraph:
if parent_pkgname in self._childless_node_names:
self._childless_node_names.remove(parent_pkgname)
if len(child_node.children) == 0:
self._childless_node_names.add(child_pkgname)
def get_and_remove_outer_dep_pkgs(self) -> list[ForeignPackage]:
"""
Returns all childless nodes of the dependency package graph and removes them.
@@ -252,7 +229,8 @@ class ExtendedPackageSearch:
Results are cached and user defined packages are preferred.
"""
def __init__(self):
def __init__(self, pacman: l.Pacman):
self._pacman = pacman
self._package_info_cache: dict[str, PackageInfo] = {}
self._dep_provider_cache: dict[str, PackageInfo] = {}
self._user_packages: list[PackageInfo] = []
@@ -268,6 +246,12 @@ class ExtendedPackageSearch:
Tried caching the given packages. Virtual packages may not be cached
"""
packages = list(
filter(lambda p: p not in self._package_info_cache, packages))
if len(packages) == 0:
return
l.print_debug(f"Trying to cache {packages}.")
max_pkgs_per_request = 200
@@ -306,13 +290,12 @@ class ExtendedPackageSearch:
pkgbase=result["PackageBase"],
version=result["Version"],
dependencies=result.get("Depends", []),
make_and_check_dependencies=result.get(
"MakeDepends", []) +
result.get("CheckDepends", []),
make_dependencies=result.get("MakeDepends", []),
check_dependencies=result.get("CheckDepends", []),
provides=result.get("Provides", []),
git_url=
f"https://aur.archlinux.org/{result['PackageBase']}.git"
)
f"https://aur.archlinux.org/{result['PackageBase']}.git",
pacman=self._pacman)
self._package_info_cache[pkgname] = info
l.print_debug("Request completed.")
@@ -361,11 +344,12 @@ class ExtendedPackageSearch:
pkgbase=result["PackageBase"],
version=result["Version"],
dependencies=result.get("Depends", []),
make_and_check_dependencies=result.get("MakeDepends", []) +
result.get("CheckDepends", []),
make_dependencies=result.get("MakeDepends", []),
check_dependencies=result.get("CheckDepends", []),
provides=result.get("Provides", []),
git_url=f"https://aur.archlinux.org/{result['PackageBase']}.git"
)
git_url=
f"https://aur.archlinux.org/{result['PackageBase']}.git",
pacman=self._pacman)
self._package_info_cache[package] = info
@@ -472,6 +456,56 @@ class ExtendedPackageSearch:
return info
class ResolvedDependencies:
"""
Result of dependency resolution.
"""
def __init__(self):
self.pacman_deps: set[str] = set()
self.foreign_pkgs: set[str] = set()
self.foreign_dep_pkgs: set[str] = set()
self.foreign_build_dep_pkgs: set[str] = set()
self.build_order: list[str] = []
self.packages: dict[str, ForeignPackage] = {}
self._pkgbases_to_pkgs: dict[str, set[str]] = {}
self._pkgs_to_pkgbases: dict[str, str] = {}
def add_pkgbase_info(self, pkgname: str, pkgbase: str):
"""
Adds information about a which package belongs in which package base.
"""
pkgs = self._pkgbases_to_pkgs.get(pkgbase, set())
pkgs.add(pkgname)
self._pkgbases_to_pkgs[pkgbase] = pkgs
self._pkgs_to_pkgbases[pkgname] = pkgbase
def get_pkgbase(self, pkgname: str) -> str:
"""
Returns the package base of an package.
"""
return self._pkgs_to_pkgbases[pkgname]
def get_pkgs_with_common_pkgbase(self, pkgname: str) -> set[str]:
"""
Returns all packages that have the same package base as the given package.
"""
pkgbase = self._pkgs_to_pkgbases[pkgname]
return self._pkgbases_to_pkgs[pkgbase]
def all_pkgbases(self) -> list[str]:
"""
Returns all pkgbases.
"""
return list(self._pkgbases_to_pkgs)
def get_some_pkgname(self, pkgbase: str) -> str:
"""
Returns some package name that the given pkgbase has.
"""
return list(self._pkgbases_to_pkgs[pkgbase])[0]
class ForeignPackageManager:
"""
Class for dealing with AUR/user packages.
@@ -519,56 +553,84 @@ class ForeignPackageManager:
Installs the given AUR/user packages and their dependencies (both pacman/AUR).
"""
if as_explicit is None:
as_explicit = foreign_pkgs
if len(foreign_pkgs) == 0:
return
all_foreign_pkgs, pacman_deps = self.resolve_dependencies(foreign_pkgs)
resolved_dependencies = self.resolve_dependencies(foreign_pkgs)
l.print_summary(
f"The following foreign packages will be installed: {' '.join(map(lambda p: p.name, all_foreign_pkgs))}"
"The following foreign packages will be installed explicitly:")
l.print_continuation("")
l.print_continuation(
f"\t{' '.join(resolved_dependencies.foreign_pkgs)}")
l.print_continuation("")
if resolved_dependencies.foreign_dep_pkgs:
l.print_summary(
"The following foreign packages will be installed as dependencies:"
)
l.print_continuation("")
l.print_continuation(
f"\t{' '.join(resolved_dependencies.foreign_dep_pkgs)}")
l.print_continuation("")
if resolved_dependencies.foreign_build_dep_pkgs:
l.print_summary(
"The following foreign packages will be built in order to install other packages. They will not be installed:"
)
l.print_continuation("")
l.print_continuation(
f"\t{' '.join(resolved_dependencies.foreign_build_dep_pkgs)}")
l.print_continuation("")
if not l.prompt_confirm("Proceed?", default=True):
raise l.UserFacingError("Installing aborted.")
l.print_summary(
"Installing AUR/user package dependencies from pacman.")
self._pacman.install_dependencies(list(pacman_deps))
self._pacman.install_dependencies(
list(resolved_dependencies.pacman_deps))
to_install = []
try:
with PackageBuilder(self._search, self._store,
resolved_dependencies) as builder:
while resolved_dependencies.build_order:
to_build = resolved_dependencies.build_order.pop(0)
while all_foreign_pkgs:
pkg_to_build = all_foreign_pkgs.pop(0)
pkgbase = resolved_dependencies.get_pkgbase(to_build)
package_names = resolved_dependencies.get_pkgs_with_common_pkgbase(
to_build)
# resolve_dependencies gets info for every package so this cannot be None
pkgbase = self._search.get_package_info(
pkg_to_build.name
).pkgbase # pyright: ignore[reportOptionalMemberAccess]
with_same_pkgbase = []
packages = [
resolved_dependencies.packages[pkgname]
for pkgname in package_names
]
for other in all_foreign_pkgs:
other_pkgbase = self._search.get_package_info(
other.name
).pkgbase # pyright: ignore[reportOptionalMemberAccess]
if other_pkgbase == pkgbase:
with_same_pkgbase.append(other)
builder.build_packages(pkgbase, packages, force)
except (subprocess.CalledProcessError, OSError) as e:
raise l.UserFacingError("Failed to build packages.") from e
for other in with_same_pkgbase:
all_foreign_pkgs.remove(other)
if as_explicit is None:
as_explicit = list(resolved_dependencies.foreign_pkgs)
to_install += self._build_pkg(pkgbase,
[pkg_to_build] + with_same_pkgbase,
force)
packages_to_install = list(resolved_dependencies.foreign_pkgs)
packages_to_install += list(resolved_dependencies.foreign_dep_pkgs)
if to_install or force:
package_files_to_install = []
for pkg in packages_to_install:
built_pkg = self._store.get_package(pkg)
assert built_pkg is not None
_, path = built_pkg
package_files_to_install.append(path)
if package_files_to_install or force:
l.print_summary("Installing AUR/user packages.")
self._pacman.install_files(to_install, as_explicit)
self._pacman.install_files(package_files_to_install, as_explicit)
else:
l.print_summary("No packages to install.")
def resolve_dependencies(
self, foreign_packages: list[str]
) -> tuple[list[ForeignPackage], set[str]]:
self, foreign_packages: list[str]) -> ResolvedDependencies:
"""
Resolves AUR/user dependencies of AUR/user packages.
@@ -583,7 +645,9 @@ class ForeignPackageManager:
l.print_summary("Resolving AUR / user package dependencies.")
l.print_debug(f"Packages: {foreign_packages}")
pacman_deps = set()
result = ResolvedDependencies()
result.foreign_pkgs = set(foreign_packages)
graph = DepGraph()
for name in foreign_packages:
@@ -593,6 +657,23 @@ class ForeignPackageManager:
to_process = list(foreign_packages)
total_processed = 0
def process_dep(pkgname: str, depname: str, add_to: set[str]):
dep_info = self._search.find_provider(depname)
if dep_info is None:
raise l.UserFacingError(
f"Failed to find '{depname}' from AUR or user provided packages."
)
add_to.add(dep_info.pkgname)
l.print_debug(
f"Adding dependency {dep_info.pkgname} to package {pkgname}.")
graph.add_requirement(dep_info.pkgname, pkgname)
if dep_info.pkgname not in seen_packages:
to_process.append(dep_info.pkgname)
seen_packages.add(dep_info.pkgname)
while to_process:
pkgname = to_process.pop()
@@ -602,36 +683,25 @@ class ForeignPackageManager:
f"Failed to find '{pkgname}' from AUR or user provided packages."
)
pacman_deps.update(info.pacman_dependencies(self._pacman))
depnames = info.all_foreign_dependencies_stripped(self._pacman)
self._search.try_caching_packages(depnames)
result.pacman_deps.update(info.pacman_dependencies)
result.add_pkgbase_info(pkgname, info.pkgbase)
for depname in depnames:
dep_info = self._search.find_provider(depname)
build_deps = info.foreign_make_dependencies_stripped + info.foreign_check_dependencies_stripped
if dep_info is None:
raise l.UserFacingError(
f"Failed to find '{depname}' from AUR or user provided packages."
)
self._search.try_caching_packages(
info.foreign_dependencies_stripped + build_deps)
l.print_debug(
f"Adding dependency {dep_info.pkgname} to package {pkgname}."
)
graph.add_requirement(dep_info.pkgname, pkgname)
if dep_info.pkgname not in seen_packages:
to_process.append(dep_info.pkgname)
seen_packages.add(dep_info.pkgname)
for depname in info.foreign_dependencies_stripped:
process_dep(pkgname, depname, result.foreign_dep_pkgs)
for depname in build_deps:
process_dep(pkgname, depname, result.foreign_build_dep_pkgs)
total_processed += 1
l.print_info(f"{total_processed}/{len(seen_packages)}.")
l.print_summary("Determining build order.")
build_order = self._determine_build_order(graph)
return (build_order, pacman_deps)
def _determine_build_order(self, graph: DepGraph) -> list[ForeignPackage]:
build_order = []
while True:
to_add = graph.get_and_remove_outer_dep_pkgs()
@@ -639,65 +709,104 @@ class ForeignPackageManager:
break
for pkg in to_add:
if pkg not in build_order:
if pkg not in result.packages:
l.print_debug(f"Adding {pkg} to build_order.")
build_order.append(pkg)
result.build_order.append(pkg.name)
result.packages[pkg.name] = pkg
return build_order
return result
def _build_pkg(self, package_base: str, packages: list[ForeignPackage],
force: bool) -> list[str]:
def should_upgrade_package(self,
package: str,
installed_version: str,
fetched_version: str,
upgrade_devel=False) -> bool:
"""
Builds package(s) with the same package base. Returns a list of package files to install.
Returns True if a package should be upgraded.
"""
package_names = list(map(lambda p: p.name, packages))
# Rebuild is only needed if at least one package is not in the cache.
if self._are_all_pkgs_cached(packages) and not force:
l.print_summary(
f"Skipped building '{' '.join(package_names)}'. Already up to date."
)
return []
l.print_summary(f"To build '{' '.join(package_names)}'.")
chroot_pacman_pkgs, chroot_pkg_files = self._get_chroot_packages(
packages)
chroot_dir = os.path.join(conf.build_dir, "chroot")
pkgbuild_dir = os.path.join(conf.build_dir, "pkgbuild")
l.print_debug(
f"Chroot dir is: '{chroot_dir}', pkgbuild dir is '{pkgbuild_dir}'."
)
prev_wd = os.getcwd()
if upgrade_devel and is_devel(package):
return True
try:
os.makedirs(conf.pkg_cache_dir, exist_ok=True)
result = int(
subprocess.run(conf.commands.compare_versions(
installed_version, fetched_version),
check=True,
stdout=subprocess.PIPE).stdout.decode())
return result < 0
except (ValueError, subprocess.CalledProcessError) as error:
raise l.UserFacingError("Failed to compare versions.") from error
class PackageBuilder:
"""
Used for building packages in a chroot.
"""
always_included_packages = ["base-devel", "git"]
def __init__(self, search: ExtendedPackageSearch, store: l.Store,
resolved_deps: ResolvedDependencies):
self._search = search
self._store = store
self._resolved_deps = resolved_deps
self.chroot_wd_dir = os.path.join(conf.build_dir, "chroot")
self.chroot_dir = os.path.join(self.chroot_wd_dir, "root")
self.pkgbase_dir_map = {}
self.original_wd = ""
def __enter__(self):
self.store_wd()
self.create_build_environment()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.restore_wd()
self.remove_build_environment()
def store_wd(self):
"""
Remembers the current working directory as the original working directory.
"""
self.original_wd = os.getcwd()
def restore_wd(self):
"""
Returns to the original working directory.
"""
os.chdir(self.original_wd)
def create_build_environment(self):
"""
Creates a new chroot and clones all PKGBUILDS.
"""
l.print_summary("Creating a build environment..")
if os.path.exists(conf.build_dir):
l.print_info("Removing previous build directory.")
shutil.rmtree(conf.build_dir)
self.remove_build_environment()
l.print_info("Setting up build directory.")
l.print_info("Getting all PKGBUILDS.")
# Set up PKGBUILDS
for pkgbase in self._resolved_deps.all_pkgbases():
pkgbuild_dir = os.path.join(conf.build_dir, pkgbase)
self.pkgbase_dir_map[pkgbase] = pkgbuild_dir
os.makedirs(pkgbuild_dir)
os.makedirs(chroot_dir)
os.chdir(pkgbuild_dir)
git_url = self._search.get_package_info(
package_names[0]
self._resolved_deps.get_some_pkgname(pkgbase)
).git_url # pyright: ignore[reportOptionalMemberAccess]
l.print_debug(f"Git URL for '{package_base}' is '{git_url}'")
self.git_clone_and_review_pkgbuild(package_base, git_url)
l.print_debug(f"Git URL for '{pkgbase}' is '{git_url}'")
self._git_clone_and_review_pkgbuild(pkgbase, git_url)
shutil.chown(pkgbuild_dir, user=conf.makepkg_user)
l.print_summary(f"Building: '{' '.join(package_names)}'.")
l.print_summary("Creating a new chroot.")
os.makedirs(self.chroot_wd_dir)
# Remove GNUPGHOME from mkarchroot environment variables since it may interfere with
# the chroot creation
@@ -709,49 +818,87 @@ class ForeignPackageManager:
except KeyError:
pass
l.print_info("Creating a new chroot.")
subprocess.run(conf.commands.make_chroot(
os.path.join(chroot_dir, "root"),
["base-devel"] + chroot_pacman_pkgs),
self.chroot_dir, PackageBuilder.always_included_packages +
list(self._resolved_deps.pacman_deps)),
env=mkarchroot_env_vars,
check=True,
capture_output=conf.quiet_output)
def remove_build_environment(self):
"""
Deletes the build environment.
"""
shutil.rmtree(conf.build_dir)
def build_packages(self, package_base: str, packages: list[ForeignPackage],
force: bool):
"""
Builds package(s) with the same package base.
Set force to true to force rebuilds of packages that are already cached
"""
package_names = list(map(lambda p: p.name, packages))
# Rebuild is only needed if at least one package is not in the cache.
if self._are_all_pkgs_cached(packages) and not force:
l.print_summary(
f"Skipped building '{' '.join(package_names)}'. Already up to date."
)
return
l.print_summary(f"To build '{' '.join(package_names)}'.")
chroot_new_pacman_pkgs, chroot_pkg_files = self._get_chroot_packages(
packages)
pkgbuild_dir = self.pkgbase_dir_map[package_base]
os.chdir(pkgbuild_dir)
l.print_debug(
f"Chroot dir is: '{self.chroot_dir}', pkgbuild dir is '{pkgbuild_dir}'."
)
l.print_info("Installing build dependencies to chroot.")
subprocess.run(conf.commands.install_chroot_packages(
self.chroot_dir,
chroot_new_pacman_pkgs + PackageBuilder.always_included_packages),
check=True,
capture_output=conf.quiet_output)
l.print_info("Making package.")
subprocess.run(conf.commands.make_chroot_pkg(
chroot_dir, conf.makepkg_user, chroot_pkg_files),
subprocess.run(conf.commands.make_chroot_pkg(self.chroot_wd_dir,
conf.makepkg_user,
chroot_pkg_files),
check=True,
capture_output=conf.quiet_output)
package_files = []
for pkgname in package_names:
file = self._find_pkgfile(pkgname, pkgbuild_dir)
dest = shutil.copy(file, conf.pkg_cache_dir)
version = self._search.get_package_info(
pkgname
).version # pyright: ignore[reportOptionalMemberAccess]
pkgname).version # pyright: ignore[reportOptionalMemberAccess]
l.print_debug(
f"Adding '{pkgname}', version: '{version}' to cache as file '{dest}'."
)
self._store.add_package_to_cache(pkgname, version, dest)
package_files.append(dest)
l.print_info("Removing build dependencies from chroot.")
subprocess.run(conf.commands.remove_chroot_packages(
self.chroot_dir, chroot_new_pacman_pkgs),
check=True,
capture_output=conf.quiet_output)
l.print_summary(f"Finished building: '{' '.join(package_names)}'.")
except (subprocess.CalledProcessError, OSError) as error:
raise l.UserFacingError(
f"Failed to build package(s) '{' '.join(map(lambda p: p.name, packages))}'."
) from error
finally:
os.chdir(prev_wd)
return package_files
def _are_all_pkgs_cached(self, pkgs: list[ForeignPackage]) -> bool:
for pkg in pkgs:
@@ -764,7 +911,7 @@ class ForeignPackageManager:
pkg.name
).version # pyright: ignore[reportOptionalMemberAccess]
if cached_version != fetched_version or self.is_devel(pkg.name):
if cached_version != fetched_version or is_devel(pkg.name):
return False
return True
@@ -772,18 +919,23 @@ class ForeignPackageManager:
self, pkgs_to_build: list[ForeignPackage]
) -> tuple[list[str], list[str]]:
"""
Returns a tuple of pacman packages and built foreign pkgs files that are needed in the
chroot before building. pkgs_to_build share the same pkgbase.
Returns a tuple of pacman build dependencies and built foreign pkgs files that are needed
in the chroot before building. pkgs_to_build share the same pkgbase.
"""
chroot_pacman_pkgs = set()
chroot_pacman_build_deps = set()
chroot_foreign_pkgs = set()
def add_to_pacman_build_deps(deps: list[str]):
for dep in deps:
if dep not in self._resolved_deps.pacman_deps:
chroot_pacman_build_deps.add(dep)
for pkg in pkgs_to_build:
info = self._search.get_package_info(pkg.name)
assert info is not None
chroot_pacman_pkgs.update(
info.all_pacman_dependencies(self._pacman))
add_to_pacman_build_deps(info.pacman_make_dependencies)
add_to_pacman_build_deps(info.pacman_check_dependencies)
foreign_deps = pkg.get_all_recursive_foreign_deps()
chroot_foreign_pkgs.update(foreign_deps)
@@ -792,8 +944,9 @@ class ForeignPackageManager:
for dep in foreign_deps:
dep_info = self._search.get_package_info(dep)
assert dep_info is not None
chroot_pacman_pkgs.update(
dep_info.all_pacman_dependencies(self._pacman))
add_to_pacman_build_deps(dep_info.pacman_make_dependencies)
add_to_pacman_build_deps(dep_info.pacman_check_dependencies)
# Packages with the same pkgbase might depend on each other,
# but they don't need to be installed for the build to succeed.
@@ -812,7 +965,7 @@ class ForeignPackageManager:
chroot_foreign_pkg_files.append(file)
return (list(chroot_pacman_pkgs), chroot_foreign_pkg_files)
return (list(chroot_pacman_build_deps), chroot_foreign_pkg_files)
def _find_pkgfile(self, pkgname: str, pkgbuild_dir: str) -> str:
# HACK: Because we don't know the pkgarch we can't be sure what is the build result.
@@ -838,27 +991,33 @@ class ForeignPackageManager:
return matches[0]
def git_clone_and_review_pkgbuild(self, pkgbase: str, git_url: str):
def _git_clone_and_review_pkgbuild(self, pkgbase: str, git_url: str):
"""
Clones an PKGBUILD to the current directory.
The user is prompted to review the PKGBUILD and confirm if the package should be built.
"""
try:
subprocess.run(conf.commands.git_clone(git_url, "."), check=True)
subprocess.run(conf.commands.git_clone(git_url, "."),
check=True,
capture_output=conf.quiet_output)
if l.prompt_confirm(f"Review PKGBUILD for {pkgbase}?",
default=True):
latest_reviewed_commit = self._store.pkgbuild_latest_reviewed_commits.get(
pkgbase)
if latest_reviewed_commit is None:
for file in os.scandir("."):
if file.is_file() and not file.name.startswith("."):
subprocess.run(conf.commands.review_file(file.path),
subprocess.run(conf.commands.review_file(
file.path),
check=True)
else:
subprocess.run(conf.commands.git_diff(latest_reviewed_commit),
subprocess.run(
conf.commands.git_diff(latest_reviewed_commit),
check=True)
if l.prompt_confirm("Proceed with building?", default=True):
if l.prompt_confirm("Build this package?", default=True):
commit_id = subprocess.run(
conf.commands.git_get_commit_id(),
check=True,
@@ -872,42 +1031,3 @@ class ForeignPackageManager:
raise l.UserFacingError(
f"Failed to clone and review PKGBUILD from {git_url}"
) from error
def should_upgrade_package(self,
package: str,
installed_version: str,
fetched_version: str,
upgrade_devel=False) -> bool:
"""
Returns True if a package should be upgraded.
"""
if upgrade_devel and self.is_devel(package):
return True
try:
result = int(
subprocess.run(conf.commands.compare_versions(
installed_version, fetched_version),
check=True,
stdout=subprocess.PIPE).stdout.decode())
return result < 0
except (ValueError, subprocess.CalledProcessError) as error:
raise l.UserFacingError("Failed to compare versions.") from error
def is_devel(self, package: str) -> bool:
"""
Returns True if the given package is a devel package.
"""
devel_suffixes = [
"-git",
"-hg",
"-bzr",
"-svn",
"-cvs",
"-darcs",
]
for suffix in devel_suffixes:
if package.endswith(suffix):
return True
return False
tests/test_package_management.py
+9 -3
View File
@@ -8,8 +8,9 @@ from decman.lib.aur import ForeignPackageManager, DepGraph, ForeignPackage, Exte
class TestAUR(unittest.TestCase):
def setUp(self) -> None:
self.aur = ForeignPackageManager(Store(), Pacman(),
ExtendedPackageSearch())
pacman = Pacman()
self.aur = ForeignPackageManager(Store(), pacman,
ExtendedPackageSearch(pacman))
def test_should_upgrade_package_returns_true_on_newer_version(self):
self.assertTrue(
@@ -59,6 +60,8 @@ class TestDepGraph(unittest.TestCase):
graph = DepGraph()
graph.add_requirement("A", None)
graph.add_requirement("V", None)
graph.add_requirement("B1", "A")
graph.add_requirement("B2", "A")
graph.add_requirement("B3", "A")
@@ -71,6 +74,8 @@ class TestDepGraph(unittest.TestCase):
graph.add_requirement("C2", "D")
v = ForeignPackage("V")
a = ForeignPackage("A")
a.add_foreign_dependency_packages(["B1", "B2", "B3", "C1", "C2", "D"])
@@ -90,7 +95,8 @@ class TestDepGraph(unittest.TestCase):
d = ForeignPackage("D")
d.add_foreign_dependency_packages(["C2"])
self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [c2, b3])
self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(),
[c2, b3, v])
self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [d])
self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [c1])
self.assertCountEqual(graph.get_and_remove_outer_dep_pkgs(), [b1])