Merge branch 'pu/distro-info-templates' into 'master'

Do not hardcode Debian + Ubuntu codenames, use distro-info-data

See merge request apt-team/python-apt!52

aptsources/distinfo.py

@@ -23,6 +23,7 @@
 
 from __future__ import print_function
 
+import csv
 import errno
 import logging
 import os
@@ -34,6 +35,87 @@ import apt_pkg
 from apt_pkg import gettext as _
 
 
+def _expand_template(template, csv_path):
+    """Expand the given template.
+
+    A template file consists of a header, followed by paragraphs
+    of templated suites, followed by a footer. A templated suite
+    is any paragraph where the Suite field contains {.
+
+    This function expands all templated suites using the information
+    found in the CSV file supplied by distro-info-data.
+
+    It yields lines of template info.
+    """
+
+    known_suites = set()
+
+    # Copy out any header, and gather all hardcoded suites
+    with apt_pkg.TagFile(template) as tmpl:
+        for section in tmpl:
+            if "X-Exclude-Suites" in section:
+                known_suites.update(section["X-Exclude-Suites"].split(", "))
+            if "Suite" in section:
+                if "{" in section["Suite"]:
+                    break
+
+                known_suites.add(section["Suite"])
+
+            yield from str(section).splitlines()
+        else:
+            # We did not break, so we did copy all of them
+            return
+
+        for section in tmpl:
+            if "Suite" in section:
+                known_suites.add(section["Suite"])
+
+    with open(csv_path) as csv_object:
+        releases = reversed(list(csv.DictReader(csv_object)))
+
+    # Perform template substitution on the middle of the list
+    for rel in releases:
+        if rel["series"] in known_suites:
+            continue
+        yield ""
+        rel["version"] = rel["version"].replace(" LTS", "")
+        with apt_pkg.TagFile(template) as tmpl:
+            for section in tmpl:
+                # Only work on template sections, this skips head and tails
+                if "Suite" not in section or "{" not in section["Suite"]:
+                    continue
+                if "X-Version" in section:
+                    # Version requirements. Maybe should be made nicer
+                    ver = rel["version"]
+                    if any(
+                            (field.startswith("le") and
+                             apt_pkg.version_compare(field[3:], ver) < 0) or
+                            (field.startswith("ge") and
+                             apt_pkg.version_compare(field[3:], ver) > 0)
+                            for field in section["X-Version"].split(", ")):
+                        continue
+
+                for line in str(section).format(**rel).splitlines():
+                    if line.startswith("X-Version"):
+                        continue
+                    yield line
+
+    # Copy out remaining suites
+    with apt_pkg.TagFile(template) as tmpl:
+        # Skip the head again, we don't want to copy it twice
+        for section in tmpl:
+            if "Suite" in section and "{" in section["Suite"]:
+                break
+
+        for section in tmpl:
+            # Ignore any template parts and copy the rest out,
+            # this is the inverse of the template substitution loop
+            if "Suite" in section and "{" in section["Suite"]:
+                continue
+
+            yield from str(section).splitlines()
+
+
 class Template(object):
 
     def __init__(self):
@@ -181,92 +263,93 @@ class DistInfo(object):
         map_mirror_sets = {}
 
         dist_fname = "%s/%s.info" % (base_dir, dist)
-        with open(dist_fname) as dist_file:
-            template = None
-            component = None
-            for line in dist_file:
-                tokens = line.split(':', 1)
-                if len(tokens) < 2:
-                    continue
-                field = tokens[0].strip()
-                value = tokens[1].strip()
-                if field == 'ChangelogURI':
-                    self.changelogs_uri = _(value)
-                elif field == 'MetaReleaseURI':
-                    self.metarelease_uri = value
-                elif field == 'Suite':
-                    self.finish_template(template, component)
-                    component = None
-                    template = Template()
-                    template.name = value
-                    template.distribution = dist
-                    template.match_name = "^%s$" % value
-                elif field == 'MatchName':
-                    template.match_name = value
-                elif field == 'ParentSuite':
-                    template.child = True
-                    for nanny in self.templates:
-                        # look for parent and add back ref to it
-                        if nanny.name == value:
-                            template.parents.append(nanny)
-                            nanny.children.append(template)
-                elif field == 'Available':
-                    template.available = apt_pkg.string_to_bool(value)
-                elif field == 'Official':
-                    template.official = apt_pkg.string_to_bool(value)
-                elif field == 'RepositoryType':
-                    template.type = value
-                elif field == 'BaseURI' and not template.base_uri:
-                    template.base_uri = value
-                elif field == 'BaseURI-%s' % self.arch:
-                    template.base_uri = value
-                elif field == 'MatchURI' and not template.match_uri:
-                    template.match_uri = value
-                elif field == 'MatchURI-%s' % self.arch:
-                    template.match_uri = value
-                elif (field == 'MirrorsFile' or
-                      field == 'MirrorsFile-%s' % self.arch):
-                    # Make the path absolute.
-                    value = os.path.isabs(value) and value or \
-                            os.path.abspath(os.path.join(base_dir, value))
-                    if value not in map_mirror_sets:
-                        mirror_set = {}
-                        try:
-                            with open(value) as value_f:
-                                mirror_data = list(filter(
-                                    match_mirror_line.match,
-                                    [x.strip() for x in value_f]))
-                        except Exception:
-                            print("WARNING: Failed to read mirror file")
-                            mirror_data = []
-                        for line in mirror_data:
-                            if line.startswith("#LOC:"):
-                                location = match_loc.sub(r"\1", line)
-                                continue
-                            (proto, hostname, dir) = split_url(line)
-                            if hostname in mirror_set:
-                                mirror_set[hostname].add_repository(proto, dir)
-                            else:
-                                mirror_set[hostname] = Mirror(
-                                    proto, hostname, dir, location)
-                        map_mirror_sets[value] = mirror_set
-                    template.mirror_set = map_mirror_sets[value]
-                elif field == 'Description':
-                    template.description = _(value)
-                elif field == 'Component':
-                    if (component and not
-                            template.has_component(component.name)):
-                        template.components.append(component)
-                    component = Component(value)
-                elif field == 'CompDescription':
-                    component.set_description(_(value))
-                elif field == 'CompDescriptionLong':
-                    component.set_description_long(_(value))
-                elif field == 'ParentComponent':
-                    component.set_parent_component(value)
-            self.finish_template(template, component)
-            template = None
-            component = None
+        csv_fname = "/usr/share/distro-info/{}.csv".format(dist.lower())
+
+        template = None
+        component = None
+        for line in _expand_template(dist_fname, csv_fname):
+            tokens = line.split(':', 1)
+            if len(tokens) < 2:
+                continue
+            field = tokens[0].strip()
+            value = tokens[1].strip()
+            if field == 'ChangelogURI':
+                self.changelogs_uri = _(value)
+            elif field == 'MetaReleaseURI':
+                self.metarelease_uri = value
+            elif field == 'Suite':
+                self.finish_template(template, component)
+                component = None
+                template = Template()
+                template.name = value
+                template.distribution = dist
+                template.match_name = "^%s$" % value
+            elif field == 'MatchName':
+                template.match_name = value
+            elif field == 'ParentSuite':
+                template.child = True
+                for nanny in self.templates:
+                    # look for parent and add back ref to it
+                    if nanny.name == value:
+                        template.parents.append(nanny)
+                        nanny.children.append(template)
+            elif field == 'Available':
+                template.available = apt_pkg.string_to_bool(value)
+            elif field == 'Official':
+                template.official = apt_pkg.string_to_bool(value)
+            elif field == 'RepositoryType':
+                template.type = value
+            elif field == 'BaseURI' and not template.base_uri:
+                template.base_uri = value
+            elif field == 'BaseURI-%s' % self.arch:
+                template.base_uri = value
+            elif field == 'MatchURI' and not template.match_uri:
+                template.match_uri = value
+            elif field == 'MatchURI-%s' % self.arch:
+                template.match_uri = value
+            elif (field == 'MirrorsFile' or
+                  field == 'MirrorsFile-%s' % self.arch):
+                # Make the path absolute.
+                value = os.path.isabs(value) and value or \
+                        os.path.abspath(os.path.join(base_dir, value))
+                if value not in map_mirror_sets:
+                    mirror_set = {}
+                    try:
+                        with open(value) as value_f:
+                            mirror_data = list(filter(
+                                match_mirror_line.match,
+                                [x.strip() for x in value_f]))
+                    except Exception:
+                        print("WARNING: Failed to read mirror file")
+                        mirror_data = []
+                    for line in mirror_data:
+                        if line.startswith("#LOC:"):
+                            location = match_loc.sub(r"\1", line)
+                            continue
+                        (proto, hostname, dir) = split_url(line)
+                        if hostname in mirror_set:
+                            mirror_set[hostname].add_repository(proto, dir)
+                        else:
+                            mirror_set[hostname] = Mirror(
+                                proto, hostname, dir, location)
+                    map_mirror_sets[value] = mirror_set
+                template.mirror_set = map_mirror_sets[value]
+            elif field == 'Description':
+                template.description = _(value)
+            elif field == 'Component':
+                if (component and not
+                        template.has_component(component.name)):
+                    template.components.append(component)
+                component = Component(value)
+            elif field == 'CompDescription':
+                component.set_description(_(value))
+            elif field == 'CompDescriptionLong':
+                component.set_description_long(_(value))
+            elif field == 'ParentComponent':
+                component.set_parent_component(value)
+        self.finish_template(template, component)
+        template = None
+        component = None
 
     def finish_template(self, template, component):
         " finish the current tempalte "

data/templates/Debian.info.in

 _ChangelogURI: http://packages.debian.org/changelogs/pool/%s/%s/%s/%s_%s/changelog
+X-Exclude-Suites: buzz, rex, bo, hamm, slink, potato, woody, experimental
 
-Suite: bullseye
+Suite: {series}
 RepositoryType: deb
 BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
+MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){{0,1}}|deb\.|httpredir\.)debian\.org
 MirrorsFile: Debian.mirrors
-_Description: Debian 11 'bullseye'
+_Description: Debian {version} '{codename}'
 Component: main
 _CompDescription: Officially supported
 Component: contrib
@@ -13,243 +14,33 @@ _CompDescription: DFSG-compatible Software with Non-Free Dependencies
 Component: non-free
 _CompDescription: Non-DFSG-compatible Software
 
-Suite: bullseye/updates
+Suite: {series}-security
 RepositoryType: deb
 BaseURI: http://security.debian.org/
 MatchURI: security\.debian\.org
-ParentSuite: bullseye
+ParentSuite: {series}
 _Description: Security updates
+X-Version: ge 11
 
-Suite: bullseye-updates
-RepositoryType: deb
-ParentSuite: bullseye
-_Description: Recommended updates
-
-Suite: bullseye-proposed-updates
-RepositoryType: deb
-ParentSuite: bullseye
-_Description: Proposed updates
-
-Suite: buster
-RepositoryType: deb
-BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 10 'buster'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: buster/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: buster
-_Description: Security updates
-
-Suite: buster-updates
-RepositoryType: deb
-ParentSuite: buster
-_Description: Recommended updates
-
-Suite: buster-proposed-updates
-RepositoryType: deb
-ParentSuite: buster
-_Description: Proposed updates
-
-Suite: stretch
-RepositoryType: deb
-BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 9 'stretch'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: stretch/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: stretch
-_Description: Security updates
-
-Suite: stretch-updates
-RepositoryType: deb
-ParentSuite: stretch
-_Description: Recommended updates
-
-Suite: stretch-proposed-updates
-RepositoryType: deb
-ParentSuite: stretch
-_Description: Proposed updates
-
-Suite: jessie
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 8 'jessie'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: jessie/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: jessie
-_Description: Security updates
-
-Suite: jessie-updates
-RepositoryType: deb
-ParentSuite: jessie
-_Description: Recommended updates
-
-Suite: jessie-proposed-updates
-RepositoryType: deb
-ParentSuite: jessie
-_Description: Proposed updates
-
-Suite: wheezy
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 7 'Wheezy' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: wheezy/updates
+Suite: {series}/updates
 RepositoryType: deb
 BaseURI: http://security.debian.org/
 MatchURI: security\.debian\.org
-ParentSuite: wheezy
+ParentSuite: {series}
 _Description: Security updates
+X-Version: le 10
 
-Suite: wheezy-updates
+Suite: {series}-updates
 RepositoryType: deb
-ParentSuite: wheezy
+ParentSuite: {series}
 _Description: Recommended updates
+X-Version: ge 7
 
-Suite: wheezy-proposed-updates
-RepositoryType: deb
-ParentSuite: wheezy
-_Description: Proposed updates
-
-Suite: squeeze
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 6.0 'Squeeze' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: squeeze-proposed-updates
-RepositoryType: deb
-ParentSuite: squeeze
-_Description: Proposed updates
-
-Suite: squeeze/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: squeeze
-_Description: Security updates
-
-Suite: lenny
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 5.0 'Lenny' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: lenny-proposed-updates
+Suite: {series}-proposed-updates
 RepositoryType: deb
-ParentSuite: lenny
+ParentSuite: {series}
 _Description: Proposed updates
 
-Suite: lenny/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: lenny
-_Description: Security updates
-
-Suite: etch
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 4.0 'Etch'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: etch-proposed-updates
-RepositoryType: deb
-ParentSuite: etch
-_Description: Proposed updates
-
-Suite: etch/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: etch
-_Description: Security updates
-
-Suite: sarge
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 3.1 'Sarge'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: sarge-proposed-updates
-RepositoryType: deb
-ParentSuite: sarge
-_Description: Proposed updates
-
-Suite: sarge/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: sarge
-_Description: Security updates
-
 Suite: stable
 RepositoryType: deb
 BaseURI: http://http.us.debian.org/debian/
@@ -301,3 +92,4 @@ Component: contrib
 _CompDescription: DFSG-compatible Software with Non-Free Dependencies
 Component: non-free
 _CompDescription: Non-DFSG-compatible Software
+

data/templates/README.templates

@@ -8,6 +8,19 @@ LSB release name is used, e.g. 'Ubuntu.info' on a Ubuntu system.
 Furthermore all .info files are used to render the channels presented in the
 sources list more user friendly.
 
+Abstract paragraphs
+-------------------
+If the Suite name contains a {, the entire paragraph becomes a template
+that gets formatted with distro-info-data data. See the Ubuntu.info.in
+and Debian.info.in for examples.
+
+Note that in such a case, if these paragraphs use { in their values for
+non-template things, such as in regular expressions; they need to be
+escaped - { to {{ and } to }}.
+
+Abstract paragraphs can only be in the middle of the file, the head and the
+tail are copied unmodified. If abstract paragraphs are interleaved with
+concrete ones, the behavior is undefined.
 
 Tags
 ----

debian/control

@@ -9,6 +9,7 @@ Build-Depends: apt (>= 1.0.9.4),
                apt-utils,
                debhelper-compat (= 12),
                dh-python,
+               distro-info-data,
                fakeroot,
                libapt-pkg-dev (>= 1.9.11~),
                python3-all (>= 3.3),
@@ -67,7 +68,7 @@ Description: Python interface to libapt-pkg (locales)
 Package: python3-apt
 Architecture: any
 Multi-Arch: allowed
-Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common
+Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common, distro-info-data
 Recommends: lsb-release, iso-codes
 Provides: ${python3:Provides}
 Suggests: python3-apt-dbg, python-apt-doc, apt

debian/tests/control

 Tests: run-tests
-Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2)
+Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2), distro-info-data

utils/expand_info.py

+#!/usr/bin/python3
+#
+# usage: ./utils/expand_info.py build/data/templates/Debian.info \
+#                               /usr/share/distro-info/debian.csv
+
+import sys
+
+import aptsources.distinfo
+
+for line in aptsources.distinfo._expand_template(sys.argv[1], sys.argv[2]):
+    print(line)