blessed_build_auto.groovy 15 KB
Newer Older
1 2 3 4
// Copyright (C) 2018 Purism SPC
// SPDX-License-Identifier: GPL-3.0+
//
// Author: Guido Gunther <agx@sigxcpu.org>
5
// Editor: Arno Bauernöppel <arno.bauernoppel@puri.sm>
6 7
//
// Generate Pipeline jobs using to build Debian packages
Guido Gunther's avatar
Guido Gunther committed
8 9
// using the job-dsl plugin. This job is meant to be triggered
// when a new signed tag is pushed to git.
10 11
//
// https://github.com/jenkinsci/job-dsl-plugin
12
// API: https://jenkinsci.github.io/job-dsl-plugin/
13

Guido Gunther's avatar
Guido Gunther committed
14 15 16 17 18 19
@Grab('org.yaml:snakeyaml:1.17')
import org.yaml.snakeyaml.Yaml

// The Operating system we build for. This is the first part
// of the git branch: ${os}/${dist}
def os = 'debian'
20

Guido Gunther's avatar
Guido Gunther committed
21
// Default distribution to build for
22
def default_dists = ['amber-phone']
23

Guido Gunther's avatar
Guido Gunther committed
24
// Default architectures to build for
25 26
def default_archs = ['aarch64']

Guido Gunther's avatar
Guido Gunther committed
27
// Options to pass to git-buildpackage
28
def default_build_args = '--git-no-pristine-tar --git-pbuilder --git-ignore-new --git-ignore-branch --jobs=auto -nc --build=full -sa'
29

30 31
// the same but for sloppy builds
def sloppy_build_args = '--git-force-create --git-upstream-tree=SLOPPY'
Guido Gunther's avatar
Guido Gunther committed
32

33 34
def default_repo = "scratch"

Guido Gunther's avatar
Guido Gunther committed
35 36
// All jobs will be generated in this folder
def folder = 'debs'
37 38

// Artifacts
39 40 41 42
def artifacts = '*.deb,*.changes,*.dsc,*.xz,*.gz'

// Signature (keyid) for signing the packages before upload
def pkg_gpg_keyid = '1CBB2345A7F02749' //jenkins@arm02.puri.sm
Guido Gunther's avatar
Guido Gunther committed
43

44
// Who get's mail notifications
45
def email_to = 'librem5-builds@lists.community.puri.sm'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
46
def email_from = 'arno.bauernoppel@puri.sm'
47

Guido Gunther's avatar
Guido Gunther committed
48 49 50 51 52 53 54
// Load jobs from YAML file
Yaml yaml = new Yaml()
def jobsfile = (new File(__FILE__)).parent + '/jobs.yml'
println("Using jobsfile : ${jobsfile}")
def pkgs = yaml.load(new File(jobsfile).text)

pkgs.each { pkg ->
55 56 57
        params = pkg.value
        deb_build_opts = params.get('deb_build_options', '')
        deb_build_profiles = params.get('deb_build_profiles', '')
58
        default_archs.each { arch ->
59
          default_dists.each { dist ->
60 61 62 63
        name = folder + '/' + 'deb-' + pkg.key + '-' + dist + '-' + arch
        branch = params.get('branch', os + '/' + dist)
        aptly_repo = params.get('repo', default_repo)
        submodule_update = params.get('submodule_update', true)
64

65
        dist_arg = '--git-dist=' + dist
66

67 68 69 70 71 72 73
        if (submodule_update) {
            submodule_update_cmd = 'git submodule update --init --recursive'
        } else {
            submodule_update_cmd = ''
        }

        //Currently builds are only for aarch64
74
        dpkg_args = ''
75

76 77 78 79
        // Sloppy builds always build a new upstream tarball to save us from patch
        // maintenance:
        // https://honk.sigxcpu.org/projects/git-buildpackage/manual-html/gbp.special.sloppytarball.html
        if (params.get('sloppy', false)) {
80 81 82
            // Not much todo here, we just want to make sure we create a fresh tarball
            // since for sloppy builds upstrem tarballs change on each commit:
            // creates a new tarball anyway:
83
            origtgz_cmd = ''
84
            build_args = "${default_build_args} ${sloppy_build_args} --git-debian-branch=\${head_tag}"
85
        } else { // regular blessed build
86
            build_args = "${default_build_args}"
87
            origtgz_cmd = """
88
            git fetch origin pristine-tar || true
89 90
            gbp export-orig --pristine-tar || gbp export-orig --no-pristine-tar
            """
91 92
        }

93
        scm_tag_cmd = """commit="\$(git for-each-ref --sort=taggerdate --format '%(objectname)' refs/tags | tail -n 1)"; head_tag="\$(git describe --exact-match --tags \${commit})" 2> /dev/null || exit 128""".stripIndent()
94
        scm_verify_tag_cmd =  "/bin/false" // save default
95
        dev_mail = params.get('gpg-allowed-dev','')
96

97 98
        if(! dev_mail.equals('')) {
            dev_mail = "'"  + dev_mail + "'"
99
            //verify dev tag in output
100 101 102 103
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" 2>&1 | grep -F @ | perl -e "\\\\\$ret=0; while(<>){ \\\\\$_=~/Good signature/ and \\\\\$ret++; index(\\\\\$_, ${dev_mail}) != -1 and \\\\\$ret++; }; exit \\\\\$ret - 2;" || exit 64""".stripIndent()
        } else {
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
        }
104

105 106 107 108
        scm_checkout_cmd = """cd build
            ${submodule_update_cmd}
            git clean -dfx
            $scm_tag_cmd
109
            git checkout -f "\${head_tag}"
110 111
        """.stripIndent()

112
        build_cmd = """
113
            rm -f *.deb *.changes *.dsc *.upload *.tar.*
114 115 116 117
            ${scm_checkout_cmd}
            ${scm_verify_tag_cmd}

            #If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
118 119
            echo "Changelog: \$(dpkg-parsechangelog -S Distribution)"
            dpkg-parsechangelog -S Distribution | grep -qsE  '^(amber-phone(-staging)?|purple)\$' || exit 32
120
            #If it is purple change to amber-phone (temporary)
Guido Gunther's avatar
Guido Gunther committed
121
            sed -i  '1 s/ purple;/ amber-phone;/' debian/changelog
122

123 124 125
            # get the upstream tarball if necessary
            ${origtgz_cmd}

126 127 128 129
            export DEB_BUILD_OPTIONS='${deb_build_opts.join(' ')}'
            export DEB_BUILD_PROFILES='${deb_build_profiles.join(' ')}'
            echo "Building ${pkg.key} tag: \${head_tag}"
            gbp buildpackage ${[dist_arg, build_args, dpkg_args].join(' ')}
130
        """.stripIndent()
131

132
        upload_cmd = """
133 134 135
            #Workaround to cache secret key
            sudo /usr/bin/gpg --batch --pinentry-mode loopback --passphrase-file /home/jenkins/.secret_key -o /dev/null --decrypt /home/jenkins/.workaround.gpg

136
            [ \$(echo ./*.changes | wc -w) -ne 1 ] && exit 4
137

138 139 140 141
            changes_file=\$(ls ./*.changes)
            source_changes=\$(echo \${changes_file} | sed -e 's/_arm64/_source/')
            mergechanges -S \${changes_file} \${changes_file} > \${source_changes}
            sudo /usr/bin/debsign -k "${pkg_gpg_keyid}" "\${source_changes}" || exit 16
142

143 144
            echo "Used changes file: "
            cat "\${source_changes}"
145

146
            # finally upload the package
147
            dput -c /etc/dput.cf pureos-ftp "\${source_changes}" || exit 32;
148

149
            #Check with lintian for errors
150
            lintian -c "\${source_changes}" || exit 64
151 152
        """.stripIndent()

Arno Bauernoppel's avatar
Arno Bauernoppel committed
153
        node_label = "${os}-${arch}"
154

155
        pipelineJob(name) {
156

157
        triggers {
158
            scm('H H/6 * * *')
159
        }
160

Guido Gunther's avatar
Guido Gunther committed
161
        quietPeriod(30)
162 163 164 165 166 167

        definition {
          cps {
            sandbox()
            script("""
              node {
168 169 170
                properties properties: [
                    disableConcurrentBuilds()
                ]
171

172
                try {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
173
                    def unstable_reason = ""
Arno Bauernoppel's avatar
Arno Bauernoppel committed
174
                    def head_tag = ""
175

176
                    stage('Build Debian Package') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
177
                        node ('${node_label}') {
178 179 180 181 182 183 184 185 186 187
                            checkout([
                                changelog: true,
                                poll: true,
                                userRemoteConfigs: [[url: '${params.url}', refspec: '+refs/tags/*:refs/remotes/origin/tags/*']],
                                    \$class: 'GitSCM',
                                    branches: [[name: '**']],
                                    extensions: [[\$class: 'RelativeTargetDirectory',
                                    relativeTargetDir: 'build'],
                                ],
                            ])
188

189
                            def result = sh returnStatus: true, script: '''${build_cmd}'''
190

191 192 193 194 195 196
                            if(result != 0) {
                                if(result == 128) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "No tag on HEAD was found or tag was not signed. Aborting build."
                                    throw "not built"
                                } else if (result == 64) {
197
                                    currentBuild.result = 'NOT_BUILT'
198
                                    if(params.get('gpg-allowed-dev', false)) {
199
                                        print "Tag verification with key of " + params.get('gpg-allowed-dev', false) + " failed!"
200
                                    } else {
201
                                        print "Tag verification failed!"
202
                                    }
203
                                    throw "not built"
204 205 206 207 208
                                } else if (result == 32) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "Wrong distribution in changelog found."
                                    throw "not built"
                                } else {
209
                                    currentBuild.result = 'FAILURE'
210
                                    print "Script returned " + result + ". Aborting build."
211 212 213
                                }
                            }
                        }
214
                    }
215

216
                    stage('Save Artifacts') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
217
                        node ('${node_label}') {
218 219 220
                            archiveArtifacts '${artifacts}'
                        }
                    }
221

222
                    stage('Upload Debian Packages') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
223
                        node ('${node_label}') {
224 225 226 227
                            def result = sh returnStatus: true, script: '''${upload_cmd}'''
                            if(result != 0) {
                                if (result == 64) {
                                    currentBuild.result = 'UNSTABLE'
228
                                    unstable_reason += 'lintian failed'
229
                                    print "Lintian check of package failed."
230
                                } else if (result == 32) {
231
                                    currentBuild.result = 'FAILURE'
232
                                    print "Upload of package failed."
233
                                } else if (result == 16) {
234
                                    currentBuild.result = 'FAILURE'
235
                                    print "Corresponding .dsc file not found for .changes file."
236
                                } else if (result == 8) {
237
                                    currentBuild.result = 'FAILURE'
238
                                    print "No .changes file found."
239
                                } else if (result == 4) {
240
                                    currentBuild.result = 'FAILURE'
241
                                    print "More than one .changes file found."
242
                                } else {
243
                                    currentBuild.result = 'FAILURE'
244
                                    print "Script returned " + result + ". Aborting build."
245 246 247
                                }
                            }
                        }
248
                    }
249

250 251 252 253 254 255 256 257
                    stage('Test Package') {
                        node ('${node_label}') {
                            if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'SUCCESS') {
                                sh '[ -d "/tmp/$pkg.key" ] || mkdir "/tmp/$pkg.key"'
                                def result = sh returnStatus: true, script: 'sudo /usr/local/bin/docker_run_test.sh "$pkg.key" "${dist}"'
                                if(result != 0) {
                                    if(result == 128) {
                                        currentBuild.result = 'UNSTABLE'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
258 259 260 261 262
                                        if(unstable_reason == "") {
                                            unstable_reason = "tests failed"
                                        } else {
                                            unstable_reason = "\${unstable_reason} and tests failed"
                                        }
263
                                        print "Package tests failed."
264
                                    } else {
265
                                        currentBuild.result = 'FAILURE'
266
                                        print "Script returned " + result + ". Aborting build."
267 268 269 270 271
                                    }
                                }
                            }
                        }
                    }
272

273
                    stage('Postbuild') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
274
                        unstable_reason = (unstable_reason != "")? "(\${unstable_reason})" : ""
275

Arno Bauernoppel's avatar
Arno Bauernoppel committed
276 277
                        def dev_mail = sh script: "git tag -v '\${head_tag}' 2>&1 | grep -F @ | perl -pe 's/.*<([^<]+)>[^>]+\\\$/\\\$1/g'",
                                          returnStdout: true
278
                        def body = ""
279
                        if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE') {
280 281 282 283 284 285 286
                            def ending = ""
                            if(currentBuild.result == 'UNSTABLE') {
                                ending = "was unstable."
                            } else {
                                ending = "has failed."
                            }
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and \${ending}\\n"
287
                            body = "\${body}You can find the build logs amnd the test results attached.\\n"
288 289 290 291
                            body = "\${body}If you need further information e.g. the build environment or other runtime information, please contact:\\n"
                            body = "\${body}Build maintainer:     arno.bauernoppel@puri.sm\\n"
                            body = "\${body}Build mailing list:   https://lists.community.puri.sm/listinfo/librem5-builds\\n"
                            body = "\${body}The build maintainer: \${dev_mail}\\n"
292
                        } else if (currentBuild.result != 'NOT_BUILT' && currentBuild.result != 'ABORTED') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
293
                            currentBuild.result = 'SUCCESS'
294
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and was successful.\\n"
295 296
                        } else {
                            error "unrecognised build result"
297
                        }
298

Arno Bauernoppel's avatar
Arno Bauernoppel committed
299 300 301
                        Date date = new Date()
                        String date_string = date.format("yyyy/MM/dd HH:mm:ss")
                        body = "\${body}---\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
302
                        body = "\${body}\${date_string}"
303

304
                        node ('${node_label}') {
305 306
                            emailext replyTo: '${email_from}',
                            to: '${email_to}',
307
                            subject: "[arm02] \${currentBuild.result} \${currentBuild.fullDisplayName}",
Arno Bauernoppel's avatar
Arno Bauernoppel committed
308
                            body: body,
309
                            attachLog: true,
310 311
                            compressLog: true,
                            attachmentsPattern: "*_result.tar.bz2"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
312
                        }
313

314 315 316
                        if(unstable_reason != "") {
                            error "\${unstable_reason}"
                        }
Arno Bauernoppel's avatar
Arno Bauernoppel committed
317
                    }
318
                } catch(err) {
319 320 321 322 323
                    if(currentBuild.result == 'NOT_BUILT') {
                        print "Build aborted. Build criteria not met."
                    }
                } finally {
                    cleanWs cleanWhenFailure: false, cleanWhenSuccess: true, deleteDirs: true
324
                }
325
            }
326
            """.stripIndent())
Guido Gunther's avatar
Guido Gunther committed
327 328 329 330 331 332
          }
        }
      }
    }
  }
}