iv/repos-cmp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Repos: support repo addons and repos with unmets

Browse Source 
In the early bootstrap stage (like loongarch64 now)
repository may have some unmets (mostly build unmets)
that are worked around by adding additional (e.g. noarch)
source into the build environment.

This commit tries to model this by adding a set of binaries
to repo (we call this 'addon') and using it to satisfy
build requiremets if none are provided by the repo itself.

This commit also improves build report accuracy in presence
of unments: packages that are required to resolve the unmets
we find are added to build requirements of the package
with special marker.
This commit is contained in:
Ivan A. Melnikov 2023-08-14 14:09:26 +04:00
parent d040c3f6a4
commit 29059c3ae4
1 changed files with 65 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
repos_cmp/repos.py
View File

@ -148,6 +148,8 @@ class Repository:
self.binaries = binaries
self.bits = bits
self.reverse_prov = {} # name -> [(provide, binary)]
self.addon = None
self.reverse_prov_addon = {}
self.update_indexes()
def copy(self, new_name):
@ -156,12 +158,19 @@ class Repository:
dict(self.binaries),
self.bits)
def update_indexes(self):
@classmethod
def _reverse_provides_index(cls, binaries):
if not binaries:
return {}
rprov = collections.defaultdict(list)
for b in self.binaries.values():
for b in binaries.values():
for p in b.provides:
rprov[p.name].append((p, b))
self.reverse_prov = dict(rprov)
return dict(rprov)
def update_indexes(self):
self.reverse_prov = self._reverse_provides_index(self.binaries)
self.reverse_prov_addon = self._reverse_provides_index(self.addon)
@classmethod
def load(cls, repo_name, path, arch, components=DEFAULT_COMPONENTS):
@ -197,11 +206,18 @@ class Repository:
if b.source_name in sources:
yield b
def providers(self, dependency):
for item in self.reverse_prov.get(dependency.name, ()):
def _providers(self, dependency, index):
for item in index.get(dependency.name, ()):
if item[0].is_provide_for(dependency):
yield item
def providers(self, dependency):
yield from self._providers(dependency, self.reverse_prov)
def addon_providers(self, dependency):
if self.addon:
yield from self._providers(dependency, self.reverse_prov_addon)
def source_for_dependency(self, dependency):
return [dep[1].source_rpm
for dep in self.reverse_prov[dependency]]
@ -240,6 +256,8 @@ class Repository:
continue
LOG.debug("looking up %s", dep)
providers = set(item[1] for item in self.providers(dep))
if not providers:
providers = set(item[1] for item in self.addon_providers(dep))
if not providers:
unmets.append(dep)
continue
@ -259,7 +277,6 @@ class Repository:
for prov in p.provides:
chroot_provides[prov.name].add(prov)
stack.extend(p.requires)
return unmets, chroot_binaries
@ -296,6 +313,11 @@ def have_same_source(repoA, repoB, source_name):
_BUILDREQ = '0000-BR'
_BUILD_UNMET = '0001-UN'
def _is_build_kind(kind):
return kind in (_BUILDREQ, _BUILD_UNMET)
def _raw_build_report(from_repo, to_repo, source_name, ignore=(), prefer=()):
@ -303,25 +325,38 @@ def _raw_build_report(from_repo, to_repo, source_name, ignore=(), prefer=()):
Returns an iterable over tuples (kind, dep, provider).
"""
LOG.debug("raw Build report for %s", source_name)
# XXX: assumes from_repo is x86_64
assert from_repo.bits == 64
translate = (to_repo.bits != 64)
missing_buildreqs = ((_BUILDREQ, dep)
for dep in from_repo.sources[source_name].requires)
missing_binreqs = itertools.chain.from_iterable(
((b.name.decode(), dep) for dep in b.requires)
for b in from_repo.binaries_from(source_name))
result = set()
# find what's missing for the build chroot in to_repo
build_unmets, _b = to_repo.chroot_for(
from_repo.sources[source_name].requires, prefer=prefer)
for kind, dep in itertools.chain(missing_buildreqs, missing_binreqs):
# distinguish unmets from build requirements that are missing
buildreqs = {dep for dep in from_repo.sources[source_name].requires}
missing_buildreqs = (
((_BUILDREQ if dep in buildreqs else _BUILD_UNMET), dep)
for dep in build_unmets)
# add requirements for the binary packages this source package
# produces in the from_repo; if needed, try to translate from 64 to 32 bits
binreqs = ((b.name.decode(), (_from_64bit_dep(dep) if translate else dep))
for b in from_repo.binaries_from(source_name)
for dep in b.requires)
result = set()
for kind, dep in itertools.chain(missing_buildreqs, binreqs):
# skip some platform-specific stuff
if any(x in dep.name for x in _SPECIAL_DEPS):
continue
# if needed, try to translate from 64 to 32 bits
the_dep = _from_64bit_dep(dep) if translate else dep
# skip dependencies already present in to_repo
if any(to_repo.providers(the_dep)):
if any(to_repo.providers(dep)):
continue
# if the repository has an addon, it can cover BR
if _is_build_kind(kind) and any(to_repo.addon_providers(dep)):
continue
# skip inter-sub-package dependencies
if any(p.source_name == source_name
@ -409,8 +444,10 @@ def recursive_build_report(from_repo, to_repo, *source_names,
reports.append(format_triplet_report(report))
# find all sources in the report
bld_deps = set(p.source_name for k, _d, p in report if k == _BUILDREQ)
pkg_deps = set(p.source_name for k, _d, p in report if k != _BUILDREQ)
bld_deps = set(p.source_name
for k, _d, p in report if _is_build_kind(k))
pkg_deps = set(p.source_name
for k, _d, p in report if not _is_build_kind(k))
pkg_deps -= bld_deps
build_source_deps[cur] = sorted(bld_deps)
@ -636,3 +673,13 @@ if __name__ == '__main__':
la64 = Repository.load_from_config('loongarch64', CONFIG)
except Exception:
pass
else:
try:
la64_addon = Repository.load_from_config('la64_addon', CONFIG)
except Exception:
pass
else:
la64x = la64.copy('loongarch64-plus')
la64x.addon = la64_addon.binaries
la64x.update_indexes()
del la64_addon