Commit 294d0392 authored by Julian Andres Klode's avatar Julian Andres Klode
Browse files

Merge branch 'pu/distro-info-templates' into 'master'

Do not hardcode Debian + Ubuntu codenames, use distro-info-data

See merge request apt-team/python-apt!52
parents 65484a46 ffd66c90
......@@ -23,6 +23,7 @@
from __future__ import print_function
import csv
import errno
import logging
import os
......@@ -34,6 +35,87 @@ import apt_pkg
from apt_pkg import gettext as _
def _expand_template(template, csv_path):
"""Expand the given template.
A template file consists of a header, followed by paragraphs
of templated suites, followed by a footer. A templated suite
is any paragraph where the Suite field contains {.
This function expands all templated suites using the information
found in the CSV file supplied by distro-info-data.
It yields lines of template info.
"""
known_suites = set()
# Copy out any header, and gather all hardcoded suites
with apt_pkg.TagFile(template) as tmpl:
for section in tmpl:
if "X-Exclude-Suites" in section:
known_suites.update(section["X-Exclude-Suites"].split(", "))
if "Suite" in section:
if "{" in section["Suite"]:
break
known_suites.add(section["Suite"])
yield from str(section).splitlines()
else:
# We did not break, so we did copy all of them
return
for section in tmpl:
if "Suite" in section:
known_suites.add(section["Suite"])
with open(csv_path) as csv_object:
releases = reversed(list(csv.DictReader(csv_object)))
# Perform template substitution on the middle of the list
for rel in releases:
if rel["series"] in known_suites:
continue
yield ""
rel["version"] = rel["version"].replace(" LTS", "")
with apt_pkg.TagFile(template) as tmpl:
for section in tmpl:
# Only work on template sections, this skips head and tails
if "Suite" not in section or "{" not in section["Suite"]:
continue
if "X-Version" in section:
# Version requirements. Maybe should be made nicer
ver = rel["version"]
if any(
(field.startswith("le") and
apt_pkg.version_compare(field[3:], ver) < 0) or
(field.startswith("ge") and
apt_pkg.version_compare(field[3:], ver) > 0)
for field in section["X-Version"].split(", ")):
continue
for line in str(section).format(**rel).splitlines():
if line.startswith("X-Version"):
continue
yield line
# Copy out remaining suites
with apt_pkg.TagFile(template) as tmpl:
# Skip the head again, we don't want to copy it twice
for section in tmpl:
if "Suite" in section and "{" in section["Suite"]:
break
for section in tmpl:
# Ignore any template parts and copy the rest out,
# this is the inverse of the template substitution loop
if "Suite" in section and "{" in section["Suite"]:
continue
yield from str(section).splitlines()
class Template(object):
def __init__(self):
......@@ -181,92 +263,93 @@ class DistInfo(object):
map_mirror_sets = {}
dist_fname = "%s/%s.info" % (base_dir, dist)
with open(dist_fname) as dist_file:
template = None
component = None
for line in dist_file:
tokens = line.split(':', 1)
if len(tokens) < 2:
continue
field = tokens[0].strip()
value = tokens[1].strip()
if field == 'ChangelogURI':
self.changelogs_uri = _(value)
elif field == 'MetaReleaseURI':
self.metarelease_uri = value
elif field == 'Suite':
self.finish_template(template, component)
component = None
template = Template()
template.name = value
template.distribution = dist
template.match_name = "^%s$" % value
elif field == 'MatchName':
template.match_name = value
elif field == 'ParentSuite':
template.child = True
for nanny in self.templates:
# look for parent and add back ref to it
if nanny.name == value:
template.parents.append(nanny)
nanny.children.append(template)
elif field == 'Available':
template.available = apt_pkg.string_to_bool(value)
elif field == 'Official':
template.official = apt_pkg.string_to_bool(value)
elif field == 'RepositoryType':
template.type = value
elif field == 'BaseURI' and not template.base_uri:
template.base_uri = value
elif field == 'BaseURI-%s' % self.arch:
template.base_uri = value
elif field == 'MatchURI' and not template.match_uri:
template.match_uri = value
elif field == 'MatchURI-%s' % self.arch:
template.match_uri = value
elif (field == 'MirrorsFile' or
field == 'MirrorsFile-%s' % self.arch):
# Make the path absolute.
value = os.path.isabs(value) and value or \
os.path.abspath(os.path.join(base_dir, value))
if value not in map_mirror_sets:
mirror_set = {}
try:
with open(value) as value_f:
mirror_data = list(filter(
match_mirror_line.match,
[x.strip() for x in value_f]))
except Exception:
print("WARNING: Failed to read mirror file")
mirror_data = []
for line in mirror_data:
if line.startswith("#LOC:"):
location = match_loc.sub(r"\1", line)
continue
(proto, hostname, dir) = split_url(line)
if hostname in mirror_set:
mirror_set[hostname].add_repository(proto, dir)
else:
mirror_set[hostname] = Mirror(
proto, hostname, dir, location)
map_mirror_sets[value] = mirror_set
template.mirror_set = map_mirror_sets[value]
elif field == 'Description':
template.description = _(value)
elif field == 'Component':
if (component and not
template.has_component(component.name)):
template.components.append(component)
component = Component(value)
elif field == 'CompDescription':
component.set_description(_(value))
elif field == 'CompDescriptionLong':
component.set_description_long(_(value))
elif field == 'ParentComponent':
component.set_parent_component(value)
self.finish_template(template, component)
template = None
component = None
csv_fname = "/usr/share/distro-info/{}.csv".format(dist.lower())
template = None
component = None
for line in _expand_template(dist_fname, csv_fname):
tokens = line.split(':', 1)
if len(tokens) < 2:
continue
field = tokens[0].strip()
value = tokens[1].strip()
if field == 'ChangelogURI':
self.changelogs_uri = _(value)
elif field == 'MetaReleaseURI':
self.metarelease_uri = value
elif field == 'Suite':
self.finish_template(template, component)
component = None
template = Template()
template.name = value
template.distribution = dist
template.match_name = "^%s$" % value
elif field == 'MatchName':
template.match_name = value
elif field == 'ParentSuite':
template.child = True
for nanny in self.templates:
# look for parent and add back ref to it
if nanny.name == value:
template.parents.append(nanny)
nanny.children.append(template)
elif field == 'Available':
template.available = apt_pkg.string_to_bool(value)
elif field == 'Official':
template.official = apt_pkg.string_to_bool(value)
elif field == 'RepositoryType':
template.type = value
elif field == 'BaseURI' and not template.base_uri:
template.base_uri = value
elif field == 'BaseURI-%s' % self.arch:
template.base_uri = value
elif field == 'MatchURI' and not template.match_uri:
template.match_uri = value
elif field == 'MatchURI-%s' % self.arch:
template.match_uri = value
elif (field == 'MirrorsFile' or
field == 'MirrorsFile-%s' % self.arch):
# Make the path absolute.
value = os.path.isabs(value) and value or \
os.path.abspath(os.path.join(base_dir, value))
if value not in map_mirror_sets:
mirror_set = {}
try:
with open(value) as value_f:
mirror_data = list(filter(
match_mirror_line.match,
[x.strip() for x in value_f]))
except Exception:
print("WARNING: Failed to read mirror file")
mirror_data = []
for line in mirror_data:
if line.startswith("#LOC:"):
location = match_loc.sub(r"\1", line)
continue
(proto, hostname, dir) = split_url(line)
if hostname in mirror_set:
mirror_set[hostname].add_repository(proto, dir)
else:
mirror_set[hostname] = Mirror(
proto, hostname, dir, location)
map_mirror_sets[value] = mirror_set
template.mirror_set = map_mirror_sets[value]
elif field == 'Description':
template.description = _(value)
elif field == 'Component':
if (component and not
template.has_component(component.name)):
template.components.append(component)
component = Component(value)
elif field == 'CompDescription':
component.set_description(_(value))
elif field == 'CompDescriptionLong':
component.set_description_long(_(value))
elif field == 'ParentComponent':
component.set_parent_component(value)
self.finish_template(template, component)
template = None
component = None
def finish_template(self, template, component):
" finish the current tempalte "
......
_ChangelogURI: http://packages.debian.org/changelogs/pool/%s/%s/%s/%s_%s/changelog
X-Exclude-Suites: buzz, rex, bo, hamm, slink, potato, woody, experimental
Suite: bullseye
Suite: {series}
RepositoryType: deb
BaseURI: http://deb.debian.org/debian/
MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){{0,1}}|deb\.|httpredir\.)debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 11 'bullseye'
_Description: Debian {version} '{codename}'
Component: main
_CompDescription: Officially supported
Component: contrib
......@@ -13,243 +14,33 @@ _CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: bullseye/updates
Suite: {series}-security
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: bullseye
ParentSuite: {series}
_Description: Security updates
X-Version: ge 11
Suite: bullseye-updates
RepositoryType: deb
ParentSuite: bullseye
_Description: Recommended updates
Suite: bullseye-proposed-updates
RepositoryType: deb
ParentSuite: bullseye
_Description: Proposed updates
Suite: buster
RepositoryType: deb
BaseURI: http://deb.debian.org/debian/
MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 10 'buster'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: buster/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: buster
_Description: Security updates
Suite: buster-updates
RepositoryType: deb
ParentSuite: buster
_Description: Recommended updates
Suite: buster-proposed-updates
RepositoryType: deb
ParentSuite: buster
_Description: Proposed updates
Suite: stretch
RepositoryType: deb
BaseURI: http://deb.debian.org/debian/
MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 9 'stretch'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: stretch/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: stretch
_Description: Security updates
Suite: stretch-updates
RepositoryType: deb
ParentSuite: stretch
_Description: Recommended updates
Suite: stretch-proposed-updates
RepositoryType: deb
ParentSuite: stretch
_Description: Proposed updates
Suite: jessie
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 8 'jessie'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: jessie/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: jessie
_Description: Security updates
Suite: jessie-updates
RepositoryType: deb
ParentSuite: jessie
_Description: Recommended updates
Suite: jessie-proposed-updates
RepositoryType: deb
ParentSuite: jessie
_Description: Proposed updates
Suite: wheezy
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 7 'Wheezy'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: wheezy/updates
Suite: {series}/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: wheezy
ParentSuite: {series}
_Description: Security updates
X-Version: le 10
Suite: wheezy-updates
Suite: {series}-updates
RepositoryType: deb
ParentSuite: wheezy
ParentSuite: {series}
_Description: Recommended updates
X-Version: ge 7
Suite: wheezy-proposed-updates
RepositoryType: deb
ParentSuite: wheezy
_Description: Proposed updates
Suite: squeeze
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 6.0 'Squeeze'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: squeeze-proposed-updates
RepositoryType: deb
ParentSuite: squeeze
_Description: Proposed updates
Suite: squeeze/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: squeeze
_Description: Security updates
Suite: lenny
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 5.0 'Lenny'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: lenny-proposed-updates
Suite: {series}-proposed-updates
RepositoryType: deb
ParentSuite: lenny
ParentSuite: {series}
_Description: Proposed updates
Suite: lenny/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: lenny
_Description: Security updates
Suite: etch
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 4.0 'Etch'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: etch-proposed-updates
RepositoryType: deb
ParentSuite: etch
_Description: Proposed updates
Suite: etch/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: etch
_Description: Security updates
Suite: sarge
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
MirrorsFile: Debian.mirrors
_Description: Debian 3.1 'Sarge'
Component: main
_CompDescription: Officially supported
Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
Suite: sarge-proposed-updates
RepositoryType: deb
ParentSuite: sarge
_Description: Proposed updates
Suite: sarge/updates
RepositoryType: deb
BaseURI: http://security.debian.org/
MatchURI: security\.debian\.org
ParentSuite: sarge
_Description: Security updates
Suite: stable
RepositoryType: deb
BaseURI: http://http.us.debian.org/debian/
......@@ -301,3 +92,4 @@ Component: contrib
_CompDescription: DFSG-compatible Software with Non-Free Dependencies
Component: non-free
_CompDescription: Non-DFSG-compatible Software
......@@ -8,6 +8,19 @@ LSB release name is used, e.g. 'Ubuntu.info' on a Ubuntu system.
Furthermore all .info files are used to render the channels presented in the
sources list more user friendly.
Abstract paragraphs
-------------------
If the Suite name contains a {, the entire paragraph becomes a template
that gets formatted with distro-info-data data. See the Ubuntu.info.in
and Debian.info.in for examples.
Note that in such a case, if these paragraphs use { in their values for
non-template things, such as in regular expressions; they need to be
escaped - { to {{ and } to }}.
Abstract paragraphs can only be in the middle of the file, the head and the
tail are copied unmodified. If abstract paragraphs are interleaved with
concrete ones, the behavior is undefined.
Tags
----
......
This diff is collapsed.
......@@ -9,6 +9,7 @@ Build-Depends: apt (>= 1.0.9.4),
apt-utils,
debhelper-compat (= 12),
dh-python,
distro-info-data,
fakeroot,
libapt-pkg-dev (>= 1.9.11~),
python3-all (>= 3.3),
......@@ -67,7 +68,7 @@ Description: Python interface to libapt-pkg (locales)
Package: python3-apt
Architecture: any
Multi-Arch: allowed
Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common
Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common, distro-info-data
Recommends: lsb-release, iso-codes
Provides: ${python3:Provides}
Suggests: python3-apt-dbg, python-apt-doc, apt
......
Tests: run-tests
Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2)
Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2), distro-info-data
#!/usr/bin/python3
#
# usage: ./utils/expand_info.py build/data/templates/Debian.info \
# /usr/share/distro-info/debian.csv
import sys
import aptsources.distinfo
for line in aptsources.distinfo._expand_template(sys.argv[1], sys.argv[2]):
print(line)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment