blessed_build_auto.groovy 13.1 KB
Newer Older
1 2 3 4
// Copyright (C) 2018 Purism SPC
// SPDX-License-Identifier: GPL-3.0+
//
// Author: Guido Gunther <agx@sigxcpu.org>
5
// Editor: Arno Bauernöppel <arno.bauernoppel@puri.sm>
6 7
//
// Generate Pipeline jobs using to build Debian packages
Guido Gunther's avatar
Guido Gunther committed
8 9
// using the job-dsl plugin. This job is meant to be triggered
// when a new signed tag is pushed to git.
10 11
//
// https://github.com/jenkinsci/job-dsl-plugin
12
// API: https://jenkinsci.github.io/job-dsl-plugin/
13

Guido Gunther's avatar
Guido Gunther committed
14 15 16 17 18 19
@Grab('org.yaml:snakeyaml:1.17')
import org.yaml.snakeyaml.Yaml

// The Operating system we build for. This is the first part
// of the git branch: ${os}/${dist}
def os = 'debian'
20

Guido Gunther's avatar
Guido Gunther committed
21
// Default distribution to build for
22
def default_dists = ['amber-phone-staging']
23

Guido Gunther's avatar
Guido Gunther committed
24
// Default architectures to build for
25 26
def default_archs = ['aarch64']

Guido Gunther's avatar
Guido Gunther committed
27
// Options to pass to git-buildpackage
28
def default_build_args = '--git-no-pristine-tar --git-pbuilder --git-ignore-new --git-ignore-branch -nc --build=full -sa'
29

Guido Gunther's avatar
Guido Gunther committed
30 31
// the same but for sloppy builds
def sloppy_build_args = '--git-force-create --git-upstream-tree=SLOPPY'
Guido Gunther's avatar
Guido Gunther committed
32

Guido Gunther's avatar
Guido Gunther committed
33 34
def default_repo = "scratch"

Guido Gunther's avatar
Guido Gunther committed
35 36
// All jobs will be generated in this folder
def folder = 'debs'
37 38

// Artifacts
39 40 41 42
def artifacts = '*.deb,*.changes,*.dsc,*.xz,*.gz'

// Signature (keyid) for signing the packages before upload
def pkg_gpg_keyid = '1CBB2345A7F02749' //jenkins@arm02.puri.sm
Guido Gunther's avatar
Guido Gunther committed
43

44
// Who get's mail notifications
45
def email_to = 'librem5-builds@lists.community.puri.sm'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
46
def email_from = 'arno.bauernoppel@puri.sm'
47

Guido Gunther's avatar
Guido Gunther committed
48 49 50 51 52
// Load jobs from YAML file
Yaml yaml = new Yaml()
def jobsfile = (new File(__FILE__)).parent + '/jobs.yml'
println("Using jobsfile : ${jobsfile}")
def pkgs = yaml.load(new File(jobsfile).text)
Guido Gunther's avatar
Guido Gunther committed
53
def uploads = '*.deb *.tar.* *.diff.* *.dsc *.changes *.buildinfo'
Guido Gunther's avatar
Guido Gunther committed
54 55

pkgs.each { pkg ->
56 57 58
        params = pkg.value
        deb_build_opts = params.get('deb_build_options', '')
        deb_build_profiles = params.get('deb_build_profiles', '')
59
        default_archs.each { arch ->
60
          default_dists.each { dist ->
61 62 63 64
        name = folder + '/' + 'deb-' + pkg.key + '-' + dist + '-' + arch
        branch = params.get('branch', os + '/' + dist)
        aptly_repo = params.get('repo', default_repo)
        submodule_update = params.get('submodule_update', true)
65

66
        dist_arg = '--git-dist=' + dist
67

68 69 70 71 72 73 74
        if (submodule_update) {
            submodule_update_cmd = 'git submodule update --init --recursive'
        } else {
            submodule_update_cmd = ''
        }

        //Currently builds are only for aarch64
75
        dpkg_args = ''
76

77 78 79 80
        // Sloppy builds always build a new upstream tarball to save us from patch
        // maintenance:
        // https://honk.sigxcpu.org/projects/git-buildpackage/manual-html/gbp.special.sloppytarball.html
        if (params.get('sloppy', false)) {
81 82 83
            // Not much todo here, we just want to make sure we create a fresh tarball
            // since for sloppy builds upstrem tarballs change on each commit:
            // creates a new tarball anyway:
84
            origtgz_cmd = ''
85
            build_args = "${default_build_args} ${sloppy_build_args} --git-debian-branch=\${head_tag}"
Guido Gunther's avatar
Guido Gunther committed
86
        } else { // regular blessed build
87
            build_args = "${default_build_args}"
88
            origtgz_cmd = """
89
            git fetch origin pristine-tar || true
90 91
            gbp export-orig --pristine-tar || gbp export-orig --no-pristine-tar
            """
92 93
        }

94
        scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
95

96 97 98
        scm_checkout_cmd = """cd build
            ${submodule_update_cmd}
            git clean -dfx
99
            git checkout -f
Guido Gunther's avatar
Guido Gunther committed
100 101
        """.stripIndent()

102
        build_cmd = """
103
            rm -f *.deb *.changes *.dsc *.upload *.tar.* *.buildinfo *.build
104
            ${scm_checkout_cmd}
105
            head_tag=\$(git describe --exact-match HEAD)
106 107 108
            ${scm_verify_tag_cmd}

            #If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
109 110
            echo "Changelog: \$(dpkg-parsechangelog -S Distribution)"
            dpkg-parsechangelog -S Distribution | grep -qsE  '^(amber-phone(-staging)?|purple)\$' || exit 32
111
            #If it is purple change to amber-phone (temporary)
Guido Gunther's avatar
Guido Gunther committed
112
            sed -i  '1 s/ purple;/ amber-phone;/' debian/changelog
113

114 115 116
            # get the upstream tarball if necessary
            ${origtgz_cmd}

117 118
            export DEB_BUILD_OPTIONS='${deb_build_opts.join(' ')}'
            export DEB_BUILD_PROFILES='${deb_build_profiles.join(' ')}'
119
            export GIT_PBUILDER_OPTIONS='--source-only-changes'
120 121
            echo "Building ${pkg.key} tag: \${head_tag}"
            gbp buildpackage ${[dist_arg, build_args, dpkg_args].join(' ')}
Guido Gunther's avatar
Guido Gunther committed
122
        """.stripIndent()
123

124
        upload_cmd = """
125 126 127
            #Workaround to cache secret key
            sudo /usr/bin/gpg --batch --pinentry-mode loopback --passphrase-file /home/jenkins/.secret_key -o /dev/null --decrypt /home/jenkins/.workaround.gpg

128 129
            changes_file="\$(ls ./*_source.changes)"
            [ \$(echo "\${changes_file}" | wc -w) -eq 1 ] || exit 4
130

131
            sudo /usr/bin/debsign -k "${pkg_gpg_keyid}" "\${changes_file}" || exit 16
132
            # finally upload the package
133
            dput -c /etc/dput.cf pureos-ftp "\${changes_file}" || exit 32;
Arno Bauernoppel's avatar
Arno Bauernoppel committed
134
            #Check with lintian for errors
135
            lintian --suppress-tags bad-distribution-in-changes-file -c "\${changes_file}" || exit 64
136 137
        """.stripIndent()

Arno Bauernoppel's avatar
Arno Bauernoppel committed
138
        node_label = "${os}-${arch}"
Guido Gunther's avatar
Guido Gunther committed
139
        stash = "${os}-${pkg.key}-${arch}"
140

141
        pipelineJob(name) {
142

Arno Bauernoppel's avatar
Arno Bauernoppel committed
143
        triggers {
144
            scm('H/5 * * * *')
Arno Bauernoppel's avatar
Arno Bauernoppel committed
145
        }
146

Guido Gunther's avatar
Guido Gunther committed
147
        quietPeriod(30)
148 149 150 151 152 153

        definition {
          cps {
            sandbox()
            script("""
              node {
Guido Gunther's avatar
Guido Gunther committed
154 155 156
                properties properties: [
                    disableConcurrentBuilds()
                ]
157

158
                try {
159
                    def lintian_result = ""
Arno Bauernoppel's avatar
Arno Bauernoppel committed
160
                    def head_tag = ""
161

162
                    stage('Build Debian Package') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
163
                        node ('${node_label}') {
164 165 166
                            checkout([
                                changelog: true,
                                poll: true,
167
                                userRemoteConfigs: [[url: '${params.url}', refspec: '+refs/tags/pureos/*:refs/remotes/origin/tags/pureos/*']],
168
                                    \$class: 'GitSCM',
169
                                    branches: [[name: '**/tags/pureos/**']],
170 171 172 173
                                    extensions: [[\$class: 'RelativeTargetDirectory',
                                    relativeTargetDir: 'build'],
                                ],
                            ])
174

175
                            def result = sh returnStatus: true, script: '''${build_cmd}'''
Guido Gunther's avatar
Guido Gunther committed
176
                            stash name: '${stash}', allowEmpty: false, includes: "${uploads.replaceAll(' ',',')}"
177

178 179 180 181 182 183
                            if(result != 0) {
                                if(result == 128) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "No tag on HEAD was found or tag was not signed. Aborting build."
                                    throw "not built"
                                } else if (result == 64) {
184
                                    currentBuild.result = 'NOT_BUILT'
185
                                    if(params.get('gpg-allowed-dev', false)) {
186
                                        print "Tag verification with key of " + params.get('gpg-allowed-dev', false) + " failed!"
187
                                    } else {
188
                                        print "Tag verification failed!"
189
                                    }
190
                                    throw "not built"
191 192 193 194 195
                                } else if (result == 32) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "Wrong distribution in changelog found."
                                    throw "not built"
                                } else {
196
                                    currentBuild.result = 'FAILURE'
197
                                    print "Script returned " + result + ". Aborting build."
198 199 200
                                }
                            }
                        }
201
                    }
202

203
                    stage('Save Artifacts') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
204
                        node ('${node_label}') {
Guido Gunther's avatar
Guido Gunther committed
205 206
                            unstash '${stash}'

207 208 209
                            archiveArtifacts '${artifacts}'
                        }
                    }
210

211
                    stage('Upload Debian Packages') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
212
                        node ('${node_label}') {
Guido Gunther's avatar
Guido Gunther committed
213 214
                            unstash '${stash}'

215 216 217
                            def result = sh returnStatus: true, script: '''${upload_cmd}'''
                            if(result != 0) {
                                if (result == 64) {
218
                                    lintian_result = 'lintian failed'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
219
                                    print "Lintian check of package failed."
Arno Bauernoppel's avatar
Arno Bauernoppel committed
220
                                } else if (result == 32) {
221
                                    currentBuild.result = 'FAILURE'
222
                                    print "Upload of package failed."
223
                                } else if (result == 16) {
224
                                    currentBuild.result = 'FAILURE'
225
                                    print "Corresponding .dsc file not found for .changes file."
226
                                } else if (result == 8) {
227
                                    currentBuild.result = 'FAILURE'
228
                                    print "No .changes file found."
229
                                } else if (result == 4) {
230
                                    currentBuild.result = 'FAILURE'
231
                                    print "More than one .changes file found."
232
                                } else {
233
                                    currentBuild.result = 'FAILURE'
234
                                    print "Script returned " + result + ". Aborting build."
235 236 237
                                }
                            }
                        }
238
                    }
239

240
                    stage('Postbuild') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
241
                        unstable_reason = (unstable_reason != "")? "(\${unstable_reason})" : ""
242

Arno Bauernoppel's avatar
Arno Bauernoppel committed
243 244
                        def dev_mail = sh script: "git tag -v '\${head_tag}' 2>&1 | grep -F @ | perl -pe 's/.*<([^<]+)>[^>]+\\\$/\\\$1/g'",
                                          returnStdout: true
Arno Bauernoppel's avatar
Arno Bauernoppel committed
245
                        def body = ""
Arno Bauernoppel's avatar
Arno Bauernoppel committed
246
                        if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
247 248 249 250 251 252 253
                            def ending = ""
                            if(currentBuild.result == 'UNSTABLE') {
                                ending = "was unstable."
                            } else {
                                ending = "has failed."
                            }
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and \${ending}\\n"
254
                            body = "\${body}You can find the build logs amnd the test results attached.\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
255 256 257 258
                            body = "\${body}If you need further information e.g. the build environment or other runtime information, please contact:\\n"
                            body = "\${body}Build maintainer:     arno.bauernoppel@puri.sm\\n"
                            body = "\${body}Build mailing list:   https://lists.community.puri.sm/listinfo/librem5-builds\\n"
                            body = "\${body}The build maintainer: \${dev_mail}\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
259
                        } else if (currentBuild.result != 'NOT_BUILT' && currentBuild.result != 'ABORTED') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
260
                            currentBuild.result = 'SUCCESS'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
261
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and was successful.\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
262 263
                        } else {
                            error "unrecognised build result"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
264
                        }
265

Arno Bauernoppel's avatar
Arno Bauernoppel committed
266 267 268
                        Date date = new Date()
                        String date_string = date.format("yyyy/MM/dd HH:mm:ss")
                        body = "\${body}---\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
269
                        body = "\${body}\${date_string}"
270

Arno Bauernoppel's avatar
Arno Bauernoppel committed
271
                        node ('${node_label}') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
272 273
                            emailext replyTo: '${email_from}',
                            to: '${email_to}',
Arno Bauernoppel's avatar
Arno Bauernoppel committed
274
                            subject: "[arm02] \${currentBuild.result} \${currentBuild.fullDisplayName}",
Arno Bauernoppel's avatar
Arno Bauernoppel committed
275
                            body: body,
276
                            attachLog: true,
Arno Bauernoppel's avatar
Arno Bauernoppel committed
277 278
                            compressLog: true,
                            attachmentsPattern: "*_result.tar.bz2"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
279
                        }
280

Arno Bauernoppel's avatar
Arno Bauernoppel committed
281 282 283
                        if(unstable_reason != "") {
                            error "\${unstable_reason}"
                        }
Arno Bauernoppel's avatar
Arno Bauernoppel committed
284
                    }
285
                } catch(err) {
286 287 288 289 290
                    if(currentBuild.result == 'NOT_BUILT') {
                        print "Build aborted. Build criteria not met."
                    }
                } finally {
                    cleanWs cleanWhenFailure: false, cleanWhenSuccess: true, deleteDirs: true
291
                }
292
            }
293
            """.stripIndent())
Guido Gunther's avatar
Guido Gunther committed
294 295 296 297 298 299
          }
        }
      }
    }
  }
}