Do not require hardcoding release names in distro info, cleanup Debian

and Ubuntu

Read the data from distro-info-data and expand templates
in-memory such that we don't have to duplicate data.

For Ubuntu, this changes some historical releases a bit more, but the
changes do seem correct, and the test suite still passes. Mostly just
some reorganization that was done in later releases passed down to older
releases.

LP: #1727470

aptsources/distinfo.py

@@ -23,6 +23,7 @@
 
 from __future__ import print_function
 
+import csv
 import errno
 import logging
 import os
@@ -34,6 +35,87 @@ import apt_pkg
 from apt_pkg import gettext as _
 
 
+def _expand_template(template, csv_path):
+    """Expand the given template.
+
+    A template file consists of a header, followed by paragraphs
+    of templated suites, followed by a footer. A templated suite
+    is any paragraph where the Suite field contains {.
+
+    This function expands all templated suites using the information
+    found in the CSV file supplied by distro-info-data.
+
+    It yields lines of template info.
+    """
+
+    known_suites = set()
+
+    # Copy out any header, and gather all hardcoded suites
+    with apt_pkg.TagFile(template) as tmpl:
+        for section in tmpl:
+            if "X-Exclude-Suites" in section:
+                known_suites.update(section["X-Exclude-Suites"].split(", "))
+            if "Suite" in section:
+                if "{" in section["Suite"]:
+                    break
+
+                known_suites.add(section["Suite"])
+
+            yield from str(section).splitlines()
+        else:
+            # We did not break, so we did copy all of them
+            return
+
+        for section in tmpl:
+            if "Suite" in section:
+                known_suites.add(section["Suite"])
+
+    with open(csv_path) as csv_object:
+        releases = reversed(list(csv.DictReader(csv_object)))
+
+    # Perform template substitution on the middle of the list
+    for rel in releases:
+        if rel["series"] in known_suites:
+            continue
+        yield ""
+        rel["version"] = rel["version"].replace(" LTS", "")
+        with apt_pkg.TagFile(template) as tmpl:
+            for section in tmpl:
+                # Only work on template sections, this skips head and tails
+                if "Suite" not in section or "{" not in section["Suite"]:
+                    continue
+                if "X-Version" in section:
+                    # Version requirements. Maybe should be made nicer
+                    ver = rel["version"]
+                    if any(
+                            (field.startswith("le") and
+                             apt_pkg.version_compare(field[3:], ver) < 0) or
+                            (field.startswith("ge") and
+                             apt_pkg.version_compare(field[3:], ver) > 0)
+                            for field in section["X-Version"].split(", ")):
+                        continue
+
+                for line in str(section).format(**rel).splitlines():
+                    if line.startswith("X-Version"):
+                        continue
+                    yield line
+
+    # Copy out remaining suites
+    with apt_pkg.TagFile(template) as tmpl:
+        # Skip the head again, we don't want to copy it twice
+        for section in tmpl:
+            if "Suite" in section and "{" in section["Suite"]:
+                break
+
+        for section in tmpl:
+            # Ignore any template parts and copy the rest out,
+            # this is the inverse of the template substitution loop
+            if "Suite" in section and "{" in section["Suite"]:
+                continue
+
+            yield from str(section).splitlines()
+
+
 class Template(object):
 
     def __init__(self):
@@ -181,10 +263,12 @@ class DistInfo(object):
         map_mirror_sets = {}
 
         dist_fname = "%s/%s.info" % (base_dir, dist)
-        with open(dist_fname) as dist_file:
+        csv_fname = "/usr/share/distro-info/{}.csv".format(dist.lower())
+
+        if True:
             template = None
             component = None
-            for line in dist_file:
+            for line in _expand_template(dist_fname, csv_fname):
                 tokens = line.split(':', 1)
                 if len(tokens) < 2:
                     continue

data/templates/Debian.info.in

 _ChangelogURI: http://packages.debian.org/changelogs/pool/%s/%s/%s/%s_%s/changelog
+X-Exclude-Suites: buzz, rex, bo, hamm, slink, potato, woody, experimental
 
-Suite: bullseye
+Suite: {series}
 RepositoryType: deb
 BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
+MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){{0,1}}|deb\.|httpredir\.)debian\.org
 MirrorsFile: Debian.mirrors
-_Description: Debian 11 'bullseye'
+_Description: Debian {version} '{codename}'
 Component: main
 _CompDescription: Officially supported
 Component: contrib
@@ -13,243 +14,24 @@ _CompDescription: DFSG-compatible Software with Non-Free Dependencies
 Component: non-free
 _CompDescription: Non-DFSG-compatible Software
 
-Suite: bullseye/updates
+Suite: {series}/updates
 RepositoryType: deb
 BaseURI: http://security.debian.org/
 MatchURI: security\.debian\.org
-ParentSuite: bullseye
+ParentSuite: {series}
 _Description: Security updates
 
-Suite: bullseye-updates
+Suite: {series}-updates
 RepositoryType: deb
-ParentSuite: bullseye
+ParentSuite: {series}
 _Description: Recommended updates
+X-Version: ge 7
 
-Suite: bullseye-proposed-updates
+Suite: {series}-proposed-updates
 RepositoryType: deb
-ParentSuite: bullseye
+ParentSuite: {series}
 _Description: Proposed updates
 
-Suite: buster
-RepositoryType: deb
-BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 10 'buster'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: buster/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: buster
-_Description: Security updates
-
-Suite: buster-updates
-RepositoryType: deb
-ParentSuite: buster
-_Description: Recommended updates
-
-Suite: buster-proposed-updates
-RepositoryType: deb
-ParentSuite: buster
-_Description: Proposed updates
-
-Suite: stretch
-RepositoryType: deb
-BaseURI: http://deb.debian.org/debian/
-MatchURI: ((http|ftp)[0-9]*\.([a-z]*\.){0,1}|deb\.|httpredir\.)debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 9 'stretch'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: stretch/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: stretch
-_Description: Security updates
-
-Suite: stretch-updates
-RepositoryType: deb
-ParentSuite: stretch
-_Description: Recommended updates
-
-Suite: stretch-proposed-updates
-RepositoryType: deb
-ParentSuite: stretch
-_Description: Proposed updates
-
-Suite: jessie
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 8 'jessie'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: jessie/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: jessie
-_Description: Security updates
-
-Suite: jessie-updates
-RepositoryType: deb
-ParentSuite: jessie
-_Description: Recommended updates
-
-Suite: jessie-proposed-updates
-RepositoryType: deb
-ParentSuite: jessie
-_Description: Proposed updates
-
-Suite: wheezy
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 7 'Wheezy' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: wheezy/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: wheezy
-_Description: Security updates
-
-Suite: wheezy-updates
-RepositoryType: deb
-ParentSuite: wheezy
-_Description: Recommended updates
-
-Suite: wheezy-proposed-updates
-RepositoryType: deb
-ParentSuite: wheezy
-_Description: Proposed updates
-
-Suite: squeeze
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 6.0 'Squeeze' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: squeeze-proposed-updates
-RepositoryType: deb
-ParentSuite: squeeze
-_Description: Proposed updates
-
-Suite: squeeze/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: squeeze
-_Description: Security updates
-
-Suite: lenny
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 5.0 'Lenny' 
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: lenny-proposed-updates
-RepositoryType: deb
-ParentSuite: lenny
-_Description: Proposed updates
-
-Suite: lenny/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: lenny
-_Description: Security updates
-
-Suite: etch
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 4.0 'Etch'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: etch-proposed-updates
-RepositoryType: deb
-ParentSuite: etch
-_Description: Proposed updates
-
-Suite: etch/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: etch
-_Description: Security updates
-
-Suite: sarge
-RepositoryType: deb
-BaseURI: http://http.us.debian.org/debian/
-MatchURI: ftp[0-9]*\.([a-z]*\.){0,1}debian\.org
-MirrorsFile: Debian.mirrors
-_Description: Debian 3.1 'Sarge'
-Component: main
-_CompDescription: Officially supported
-Component: contrib
-_CompDescription: DFSG-compatible Software with Non-Free Dependencies
-Component: non-free
-_CompDescription: Non-DFSG-compatible Software
-
-Suite: sarge-proposed-updates
-RepositoryType: deb
-ParentSuite: sarge
-_Description: Proposed updates
-
-Suite: sarge/updates
-RepositoryType: deb
-BaseURI: http://security.debian.org/
-MatchURI: security\.debian\.org
-ParentSuite: sarge
-_Description: Security updates
-
 Suite: stable
 RepositoryType: deb
 BaseURI: http://http.us.debian.org/debian/
@@ -301,3 +83,4 @@ Component: contrib
 _CompDescription: DFSG-compatible Software with Non-Free Dependencies
 Component: non-free
 _CompDescription: Non-DFSG-compatible Software
+

data/templates/README.templates

@@ -8,6 +8,19 @@ LSB release name is used, e.g. 'Ubuntu.info' on a Ubuntu system.
 Furthermore all .info files are used to render the channels presented in the
 sources list more user friendly.
 
+Abstract paragraphs
+-------------------
+If the Suite name contains a {, the entire paragraph becomes a template
+that gets formatted with distro-info-data data. See the Ubuntu.info.in
+and Debian.info.in for examples.
+
+Note that in such a case, if these paragraphs use { in their values for
+non-template things, such as in regular expressions; they need to be
+escaped - { to {{ and } to }}.
+
+Abstract paragraphs can only be in the middle of the file, the head and the
+tail are copied unmodified. If abstract paragraphs are interleaved with
+concrete ones, the behavior is undefined.
 
 Tags
 ----

debian/control

@@ -9,6 +9,7 @@ Build-Depends: apt (>= 1.0.9.4),
                apt-utils,
                debhelper-compat (= 12),
                dh-python,
+               distro-info-data,
                fakeroot,
                libapt-pkg-dev (>= 1.9.11~),
                python3-all (>= 3.3),
@@ -67,7 +68,7 @@ Description: Python interface to libapt-pkg (locales)
 Package: python3-apt
 Architecture: any
 Multi-Arch: allowed
-Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common
+Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, python-apt-common, distro-info-data
 Recommends: lsb-release, iso-codes
 Provides: ${python3:Provides}
 Suggests: python3-apt-dbg, python-apt-doc, apt

debian/tests/control

 Tests: run-tests
-Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2)
+Depends: @, apt-utils, python3-all, fakeroot, intltool, pycodestyle, pyflakes3, gnupg, dirmngr | gnupg (<< 2), distro-info-data

utils/expand_info.py

+#!/usr/bin/python3
+#
+# usage: ./utils/expand_info.py build/data/templates/Debian.info \
+#                               /usr/share/distro-info/debian.csv
+
+import sys
+
+import aptsources.distinfo
+
+for line in aptsources.distinfo._expand_template(sys.argv[1], sys.argv[2]):
+    print(line)