blessed_build_auto.groovy

// Copyright (C) 2018 Purism SPC
// SPDX-License-Identifier: GPL-3.0+
//
// Author: Guido Gunther <agx@sigxcpu.org>
// Editor: Arno Bauernöppel <arno.bauernoppel@puri.sm>
//
// Generate Pipeline jobs using to build Debian packages
// using the job-dsl plugin. This job is meant to be triggered
// when a new signed tag is pushed to git.
//
// https://github.com/jenkinsci/job-dsl-plugin
// API: https://jenkinsci.github.io/job-dsl-plugin/

@Grab('org.yaml:snakeyaml:1.17')
import org.yaml.snakeyaml.Yaml

// The Operating system we build for. This is the first part
// of the git branch: ${os}/${dist}
def os = 'debian'

// Default distribution to build for
def default_dists = ['amber-phone']

// Default architectures to build for
def default_archs = ['aarch64']

// Options to pass to git-buildpackage
def default_build_args = '--git-no-pristine-tar --git-pbuilder --git-ignore-new --git-ignore-branch --jobs=auto -nc --build=full -sa'

// the same but for sloppy builds
def sloppy_build_args = '--git-force-create --git-upstream-tree=SLOPPY'

def default_repo = "scratch"

// All jobs will be generated in this folder
def folder = 'debs'

// Artifacts
def artifacts = '*.deb,*.changes,*.dsc,*.xz,*.gz'

// Signature (keyid) for signing the packages before upload
def pkg_gpg_keyid = '1CBB2345A7F02749' //jenkins@arm02.puri.sm

// Who get's mail notifications
def email_to = 'librem5-builds@lists.community.puri.sm'
def email_from = 'arno.bauernoppel@puri.sm'

// Load jobs from YAML file
Yaml yaml = new Yaml()
def jobsfile = (new File(__FILE__)).parent + '/jobs.yml'
println("Using jobsfile : ${jobsfile}")
def pkgs = yaml.load(new File(jobsfile).text)

pkgs.each { pkg ->
        params = pkg.value
        deb_build_opts = params.get('deb_build_options', '')
        deb_build_profiles = params.get('deb_build_profiles', '')
        default_archs.each { arch ->
          default_dists.each { dist ->
        name = folder + '/' + 'deb-' + pkg.key + '-' + dist + '-' + arch
        branch = params.get('branch', os + '/' + dist)
        aptly_repo = params.get('repo', default_repo)
        submodule_update = params.get('submodule_update', true)

        dist_arg = '--git-dist=' + dist

        if (submodule_update) {
            submodule_update_cmd = 'git submodule update --init --recursive'
        } else {
            submodule_update_cmd = ''
        }

        //Currently builds are only for aarch64
        dpkg_args = ''

        // Sloppy builds always build a new upstream tarball to save us from patch
        // maintenance:
        // https://honk.sigxcpu.org/projects/git-buildpackage/manual-html/gbp.special.sloppytarball.html
        if (params.get('sloppy', false)) {
            // Not much todo here, we just want to make sure we create a fresh tarball
            // since for sloppy builds upstrem tarballs change on each commit:
            // creates a new tarball anyway:
            origtgz_cmd = ''
            build_args = "${default_build_args} ${sloppy_build_args} --git-debian-branch=\${head_tag}"
        } else { // regular blessed build
            build_args = "${default_build_args}"
            origtgz_cmd = """
            git fetch origin pristine-tar || true
            gbp export-orig --pristine-tar || gbp export-orig --no-pristine-tar
            """
        }

        scm_tag_cmd = """commit="\$(git for-each-ref --sort=taggerdate --format '%(objectname)' refs/tags | tail -n 1)"; head_tag="\$(git describe --exact-match --tags \${commit})" 2> /dev/null || exit 128""".stripIndent()
        scm_verify_tag_cmd =  "/bin/false" // save default
        dev_mail = params.get('gpg-allowed-dev','')

        if(! dev_mail.equals('')) {
            dev_mail = "'"  + dev_mail + "'"
            //verify dev tag in output
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" 2>&1 | grep -F @ | perl -e "\\\\\$ret=0; while(<>){ \\\\\$_=~/Good signature/ and \\\\\$ret++; index(\\\\\$_, ${dev_mail}) != -1 and \\\\\$ret++; }; exit \\\\\$ret - 2;" || exit 64""".stripIndent()
        } else {
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
        }

        scm_checkout_cmd = """cd build
            ${submodule_update_cmd}
            git clean -dfx
            $scm_tag_cmd
            git checkout -f "\${head_tag}"
        """.stripIndent()

        build_cmd = """
            rm -f *.deb *.changes *.dsc *.upload *.tar.*
            ${scm_checkout_cmd}
            ${scm_verify_tag_cmd}

            #If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
            echo "Changelog: \$(dpkg-parsechangelog -S Distribution)"
            dpkg-parsechangelog -S Distribution | grep -qsE  '^(amber-phone(-staging)?|purple)\$' || exit 32
            #If it is purple change to amber-phone (temporary)
            sed -i  '1 s/ purple;/ amber-phone;/' debian/changelog

            # get the upstream tarball if necessary
            ${origtgz_cmd}

            export DEB_BUILD_OPTIONS='${deb_build_opts.join(' ')}'
            export DEB_BUILD_PROFILES='${deb_build_profiles.join(' ')}'
            echo "Building ${pkg.key} tag: \${head_tag}"
            gbp buildpackage ${[dist_arg, build_args, dpkg_args].join(' ')}
        """.stripIndent()

        upload_cmd = """
            #Workaround to cache secret key
            sudo /usr/bin/gpg --batch --pinentry-mode loopback --passphrase-file /home/jenkins/.secret_key -o /dev/null --decrypt /home/jenkins/.workaround.gpg

            [ \$(echo ./*.changes | wc -w) -ne 1 ] && exit 4

            changes_file=\$(ls ./*.changes)
            source_changes=\$(echo \${changes_file} | sed -e 's/_arm64/_source/')
            mergechanges -S \${changes_file} \${changes_file} > \${source_changes}
            sudo /usr/bin/debsign -k "${pkg_gpg_keyid}" "\${source_changes}" || exit 16

            echo "Used changes file: "
            cat "\${source_changes}"

            # finally upload the package
            dput -c /etc/dput.cf pureos-ftp "\${source_changes}" || exit 32;

            #Check with lintian for errors
            lintian -c "\${source_changes}" || exit 64
        """.stripIndent()

        node_label = "${os}-${arch}"

        pipelineJob(name) {

        triggers {
            scm('H H/6 * * *')
        }

        quietPeriod(30)

        definition {
          cps {
            sandbox()
            script("""
              node {
                properties properties: [
                    disableConcurrentBuilds()
                ]

                try {
                    def unstable_reason = ""
                    def head_tag = ""

                    stage('Build Debian Package') {
                        node ('${node_label}') {
                            checkout([
                                changelog: true,
                                poll: true,
                                userRemoteConfigs: [[url: '${params.url}', refspec: '+refs/tags/*:refs/remotes/origin/tags/*']],
                                    \$class: 'GitSCM',
                                    branches: [[name: '**']],
                                    extensions: [[\$class: 'RelativeTargetDirectory',
                                    relativeTargetDir: 'build'],
                                ],
                            ])

                            def result = sh returnStatus: true, script: '''${build_cmd}'''

                            if(result != 0) {
                                if(result == 128) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "No tag on HEAD was found or tag was not signed. Aborting build."
                                    throw "not built"
                                } else if (result == 64) {
                                    currentBuild.result = 'NOT_BUILT'
                                    if(params.get('gpg-allowed-dev', false)) {
                                        print "Tag verification with key of " + params.get('gpg-allowed-dev', false) + " failed!"
                                    } else {
                                        print "Tag verification failed!"
                                    }
                                    throw "not built"
                                } else if (result == 32) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "Wrong distribution in changelog found."
                                    throw "not built"
                                } else {
                                    currentBuild.result = 'FAILURE'
                                    print "Script returned " + result + ". Aborting build."
                                }
                            }
                        }
                    }

                    stage('Save Artifacts') {
                        node ('${node_label}') {
                            archiveArtifacts '${artifacts}'
                        }
                    }

                    stage('Upload Debian Packages') {
                        node ('${node_label}') {
                            def result = sh returnStatus: true, script: '''${upload_cmd}'''
                            if(result != 0) {
                                if (result == 64) {
                                    currentBuild.result = 'UNSTABLE'
                                    unstable_reason += 'lintian failed'
                                    print "Lintian check of package failed."
                                } else if (result == 32) {
                                    currentBuild.result = 'FAILURE'
                                    print "Upload of package failed."
                                } else if (result == 16) {
                                    currentBuild.result = 'FAILURE'
                                    print "Corresponding .dsc file not found for .changes file."
                                } else if (result == 8) {
                                    currentBuild.result = 'FAILURE'
                                    print "No .changes file found."
                                } else if (result == 4) {
                                    currentBuild.result = 'FAILURE'
                                    print "More than one .changes file found."
                                } else {
                                    currentBuild.result = 'FAILURE'
                                    print "Script returned " + result + ". Aborting build."
                                }
                            }
                        }
                    }

                    stage('Test Package') {
                        node ('${node_label}') {
                            if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'SUCCESS') {
                                sh '[ -d "/tmp/$pkg.key" ] || mkdir "/tmp/$pkg.key"'
                                def result = sh returnStatus: true, script: 'sudo /usr/local/bin/docker_run_test.sh "$pkg.key" "${dist}"'
                                if(result != 0) {
                                    if(result == 128) {
                                        currentBuild.result = 'UNSTABLE'
                                        if(unstable_reason == "") {
                                            unstable_reason = "tests failed"
                                        } else {
                                            unstable_reason = "\${unstable_reason} and tests failed"
                                        }
                                        print "Package tests failed."
                                    } else {
                                        currentBuild.result = 'FAILURE'
                                        print "Script returned " + result + ". Aborting build."
                                    }
                                }
                            }
                        }
                    }

                    stage('Postbuild') {
                        unstable_reason = (unstable_reason != "")? "(\${unstable_reason})" : ""

                        def dev_mail = sh script: "git tag -v '\${head_tag}' 2>&1 | grep -F @ | perl -pe 's/.*<([^<]+)>[^>]+\\\$/\\\$1/g'",
                                          returnStdout: true
                        def body = ""
                        if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE') {
                            def ending = ""
                            if(currentBuild.result == 'UNSTABLE') {
                                ending = "was unstable."
                            } else {
                                ending = "has failed."
                            }
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and \${ending}\\n"
                            body = "\${body}You can find the build logs amnd the test results attached.\\n"
                            body = "\${body}If you need further information e.g. the build environment or other runtime information, please contact:\\n"
                            body = "\${body}Build maintainer:     arno.bauernoppel@puri.sm\\n"
                            body = "\${body}Build mailing list:   https://lists.community.puri.sm/listinfo/librem5-builds\\n"
                            body = "\${body}The build maintainer: \${dev_mail}\\n"
                        } else if (currentBuild.result != 'NOT_BUILT' && currentBuild.result != 'ABORTED') {
                            currentBuild.result = 'SUCCESS'
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and was successful.\\n"
                        } else {
                            error "unrecognised build result"
                        }

                        Date date = new Date()
                        String date_string = date.format("yyyy/MM/dd HH:mm:ss")
                        body = "\${body}---\\n"
                        body = "\${body}\${date_string}"

                        node ('${node_label}') {
                            emailext replyTo: '${email_from}',
                            to: '${email_to}',
                            subject: "[arm02] \${currentBuild.result} \${currentBuild.fullDisplayName}",
                            body: body,
                            attachLog: true,
                            compressLog: true,
                            attachmentsPattern: "*_result.tar.bz2"
                        }

                        if(unstable_reason != "") {
                            error "\${unstable_reason}"
                        }
                    }
                } catch(err) {
                    if(currentBuild.result == 'NOT_BUILT') {
                        print "Build aborted. Build criteria not met."
                    }
                } finally {
                    cleanWs cleanWhenFailure: false, cleanWhenSuccess: true, deleteDirs: true
                }
            }
            """.stripIndent())
          }
        }
      }
    }
  }
}