blessed_build_auto.groovy 14.4 KB
Newer Older
1
2
3
4
// Copyright (C) 2018 Purism SPC
// SPDX-License-Identifier: GPL-3.0+
//
// Author: Guido Gunther <agx@sigxcpu.org>
5
// Editor: Arno Bauernöppel <arno.bauernoppel@puri.sm>
6
7
//
// Generate Pipeline jobs using to build Debian packages
Guido Gunther's avatar
Guido Gunther committed
8
9
// using the job-dsl plugin. This job is meant to be triggered
// when a new signed tag is pushed to git.
10
11
//
// https://github.com/jenkinsci/job-dsl-plugin
12
// API: https://jenkinsci.github.io/job-dsl-plugin/
13

Guido Gunther's avatar
Guido Gunther committed
14
15
16
17
18
19
@Grab('org.yaml:snakeyaml:1.17')
import org.yaml.snakeyaml.Yaml

// The Operating system we build for. This is the first part
// of the git branch: ${os}/${dist}
def os = 'debian'
20

Guido Gunther's avatar
Guido Gunther committed
21
// Default distribution to build for
22
def default_dists = ['amber-phone']
23

Guido Gunther's avatar
Guido Gunther committed
24
// Default architectures to build for
25
26
def default_archs = ['aarch64']

Guido Gunther's avatar
Guido Gunther committed
27
// Options to pass to git-buildpackage
28
def default_build_args = '--git-no-pristine-tar --git-pbuilder --git-ignore-new --git-ignore-branch --jobs=auto -nc --build=full -sa'
29

Guido Gunther's avatar
Guido Gunther committed
30
31
// the same but for sloppy builds
def sloppy_build_args = '--git-force-create --git-upstream-tree=SLOPPY'
Guido Gunther's avatar
Guido Gunther committed
32

Guido Gunther's avatar
Guido Gunther committed
33
34
def default_repo = "scratch"

Guido Gunther's avatar
Guido Gunther committed
35
36
// All jobs will be generated in this folder
def folder = 'debs'
37
38

// Artifacts
39
40
41
42
def artifacts = '*.deb,*.changes,*.dsc,*.xz,*.gz'

// Signature (keyid) for signing the packages before upload
def pkg_gpg_keyid = '1CBB2345A7F02749' //jenkins@arm02.puri.sm
Guido Gunther's avatar
Guido Gunther committed
43

44
// Who get's mail notifications
45
def email_to = 'librem5-builds@lists.community.puri.sm'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
46
def email_from = 'arno.bauernoppel@puri.sm'
47

Guido Gunther's avatar
Guido Gunther committed
48
49
50
51
52
53
54
// Load jobs from YAML file
Yaml yaml = new Yaml()
def jobsfile = (new File(__FILE__)).parent + '/jobs.yml'
println("Using jobsfile : ${jobsfile}")
def pkgs = yaml.load(new File(jobsfile).text)

pkgs.each { pkg ->
55
56
57
        params = pkg.value
        deb_build_opts = params.get('deb_build_options', '')
        deb_build_profiles = params.get('deb_build_profiles', '')
58
        default_archs.each { arch ->
59
          default_dists.each { dist ->
60
61
62
63
        name = folder + '/' + 'deb-' + pkg.key + '-' + dist + '-' + arch
        branch = params.get('branch', os + '/' + dist)
        aptly_repo = params.get('repo', default_repo)
        submodule_update = params.get('submodule_update', true)
64

65
        dist_arg = '--git-dist=' + dist
66

67
68
69
70
71
72
73
        if (submodule_update) {
            submodule_update_cmd = 'git submodule update --init --recursive'
        } else {
            submodule_update_cmd = ''
        }

        //Currently builds are only for aarch64
74
        dpkg_args = ''
75

76
77
78
79
        // Sloppy builds always build a new upstream tarball to save us from patch
        // maintenance:
        // https://honk.sigxcpu.org/projects/git-buildpackage/manual-html/gbp.special.sloppytarball.html
        if (params.get('sloppy', false)) {
80
81
82
            // Not much todo here, we just want to make sure we create a fresh tarball
            // since for sloppy builds upstrem tarballs change on each commit:
            // creates a new tarball anyway:
83
            origtgz_cmd = ''
84
            build_args = "${default_build_args} ${sloppy_build_args} --git-debian-branch=\${head_tag}"
85
86
        } else { // regular snapshot build
            build_args = "${default_build_args}"
87
88
89
            origtgz_cmd = """
            gbp export-orig --pristine-tar || gbp export-orig --no-pristine-tar
            """
90
91
        }

92
        scm_tag_cmd = """commit="\$(git for-each-ref --sort=taggerdate --format '%(objectname)' refs/tags | tail -n 1)"; head_tag="\$(git describe --exact-match --tags \${commit})" 2> /dev/null || exit 128""".stripIndent()
93
        scm_verify_tag_cmd =  "/bin/false" // save default
94
        dev_mail = params.get('gpg-allowed-dev','')
95

96
97
        if(! dev_mail.equals('')) {
            dev_mail = "'"  + dev_mail + "'"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
98
            //verify dev tag in output
99
100
101
102
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" 2>&1 | grep -F @ | perl -e "\\\\\$ret=0; while(<>){ \\\\\$_=~/Good signature/ and \\\\\$ret++; index(\\\\\$_, ${dev_mail}) != -1 and \\\\\$ret++; }; exit \\\\\$ret - 2;" || exit 64""".stripIndent()
        } else {
            scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
        }
103

104
105
106
107
        scm_checkout_cmd = """cd build
            ${submodule_update_cmd}
            git clean -dfx
            $scm_tag_cmd
108
            git checkout -f "\${head_tag}"
Guido Gunther's avatar
Guido Gunther committed
109
110
        """.stripIndent()

111
        build_cmd = """
112
            rm -f *.deb *.changes *.dsc *.upload *.orig.tar.*
113
114
115
116
            ${scm_checkout_cmd}
            ${scm_verify_tag_cmd}

            #If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
117
118
            echo "Changelog: \$(dpkg-parsechangelog -S Distribution)"
            dpkg-parsechangelog -S Distribution | grep -qsE  '^(amber-phone(-staging)?|purple)\$' || exit 32
119
            #If it is purple change to amber-phone (temporary)
Guido Gunther's avatar
Guido Gunther committed
120
            sed -i  '1 s/ purple;/ amber-phone;/' debian/changelog
121

122
123
124
            # get the upstream tarball if necessary
            ${origtgz_cmd}

125
126
127
128
            export DEB_BUILD_OPTIONS='${deb_build_opts.join(' ')}'
            export DEB_BUILD_PROFILES='${deb_build_profiles.join(' ')}'
            echo "Building ${pkg.key} tag: \${head_tag}"
            gbp buildpackage ${[dist_arg, build_args, dpkg_args].join(' ')}
Guido Gunther's avatar
Guido Gunther committed
129
        """.stripIndent()
130

131
        upload_cmd = """
132
133
134
            #Workaround to cache secret key
            sudo /usr/bin/gpg --batch --pinentry-mode loopback --passphrase-file /home/jenkins/.secret_key -o /dev/null --decrypt /home/jenkins/.workaround.gpg

135
            [ \$(echo ./*.changes | wc -w) -ne 1 ] && exit 4
136

137
138
139
140
            changes_file=\$(ls ./*.changes)
            source_changes=\$(echo \${changes_file} | sed -e 's/_arm64/_source/')
            mergechanges -S \${changes_file} \${changes_file} > \${source_changes}
            sudo /usr/bin/debsign -k "${pkg_gpg_keyid}" "\${source_changes}" || exit 16
141

142
143
            echo "Used changes file: "
            cat "\${source_changes}"
144

145
            # finally upload the package
Arno Bauernoppel's avatar
Arno Bauernoppel committed
146
147
148
            dput -c /etc/dput.cf pureos-ftp "\${source_changes}" || exit 32;
            
            #Check with lintian for errors
149
            lintian -c "\${source_changes}" || exit 64
150
151
        """.stripIndent()

Arno Bauernoppel's avatar
Arno Bauernoppel committed
152
        node_label = "${os}-${arch}"
153

154
        pipelineJob(name) {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
155
156
157
158
159
        
        triggers {
            scm('H H/6 * * *')
        }
        
Guido Gunther's avatar
Guido Gunther committed
160
        quietPeriod(30)
161
162
163
164
165
166

        definition {
          cps {
            sandbox()
            script("""
              node {
Guido Gunther's avatar
Guido Gunther committed
167
168
169
                properties properties: [
                    disableConcurrentBuilds()
                ]
170

171
                try {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
172
                    def unstable_reason = ""
Arno Bauernoppel's avatar
Arno Bauernoppel committed
173
                    def head_tag = ""
174

175
                    stage('Build Debian Package') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
176
                        node ('${node_label}') {
177
178
179
180
181
182
183
184
185
186
                            checkout([
                                changelog: true,
                                poll: true,
                                userRemoteConfigs: [[url: '${params.url}', refspec: '+refs/tags/*:refs/remotes/origin/tags/*']],
                                    \$class: 'GitSCM',
                                    branches: [[name: '**']],
                                    extensions: [[\$class: 'RelativeTargetDirectory',
                                    relativeTargetDir: 'build'],
                                ],
                            ])
187

188
                            def result = sh returnStatus: true, script: '''${build_cmd}'''
189

190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
                            if(result != 0) {
                                if(result == 128) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "No tag on HEAD was found or tag was not signed. Aborting build."
                                    throw "not built"
                                } else if (result == 64) {
                                    if(params.get('gpg-allowed-dev', false)) {
                                        error "Tag verification with key of " + params.get('gpg-allowed-dev', false) + " failed!"
                                    } else {
                                        error "Tag verification failed!"
                                    }
                                } else if (result == 32) {
                                    currentBuild.result = 'NOT_BUILT'
                                    print "Wrong distribution in changelog found."
                                    throw "not built"
                                } else {
                                    error "Script returned " + result + ". Aborting build."
                                }
                            }
                        }
210
                    }
211

212
                    stage('Save Artifacts') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
213
                        node ('${node_label}') {
214
215
216
                            archiveArtifacts '${artifacts}'
                        }
                    }
217

218
                    stage('Upload Debian Packages') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
219
                        node ('${node_label}') {
220
221
222
223
                            def result = sh returnStatus: true, script: '''${upload_cmd}'''
                            if(result != 0) {
                                if (result == 64) {
                                    currentBuild.result = 'UNSTABLE'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
224
                                    unstable_reason += 'lintian failed'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
225
                                    print "Lintian check of package failed."
Arno Bauernoppel's avatar
Arno Bauernoppel committed
226
227
                                } else if (result == 32) {
                                    error "Upload of package failed."
228
229
230
231
232
233
234
235
236
237
238
                                } else if (result == 16) {
                                    error "Corresponding .dsc file not found for .changes file."
                                } else if (result == 8) {
                                    error "No .changes file found."
                                } else if (result == 4) {
                                    error "More than one .changes file found."
                                } else {
                                    error "Script returned " + result + ". Aborting build."
                                }
                            }
                        }
239
                    }
240

Arno Bauernoppel's avatar
Arno Bauernoppel committed
241
242
243
244
245
246
247
248
                    stage('Test Package') {
                        node ('${node_label}') {
                            if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'SUCCESS') {
                                sh '[ -d "/tmp/$pkg.key" ] || mkdir "/tmp/$pkg.key"'
                                def result = sh returnStatus: true, script: 'sudo /usr/local/bin/docker_run_test.sh "$pkg.key" "${dist}"'
                                if(result != 0) {
                                    if(result == 128) {
                                        currentBuild.result = 'UNSTABLE'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
249
250
251
252
253
254
                                        if(unstable_reason == "") {
                                            unstable_reason = "tests failed"
                                        } else {
                                            unstable_reason = "\${unstable_reason} and tests failed"
                                        }
                                        
Arno Bauernoppel's avatar
Arno Bauernoppel committed
255
                                        print  "Package tests failed."
Arno Bauernoppel's avatar
Arno Bauernoppel committed
256
257
258
259
260
261
262
                                    } else {
                                        error "Script returned " + result + ". Aborting build."
                                    }
                                }
                            }
                        }
                    }
263

264
                    stage('Postbuild') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
265
                        unstable_reason = (unstable_reason != "")? "(\${unstable_reason})" : ""
266

Arno Bauernoppel's avatar
Arno Bauernoppel committed
267
268
                        def dev_mail = sh script: "git tag -v '\${head_tag}' 2>&1 | grep -F @ | perl -pe 's/.*<([^<]+)>[^>]+\\\$/\\\$1/g'",
                                          returnStdout: true
Arno Bauernoppel's avatar
Arno Bauernoppel committed
269
                        def body = ""
Arno Bauernoppel's avatar
Arno Bauernoppel committed
270
                        if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
271
272
273
274
275
276
277
                            def ending = ""
                            if(currentBuild.result == 'UNSTABLE') {
                                ending = "was unstable."
                            } else {
                                ending = "has failed."
                            }
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and \${ending}\\n"
278
                            body = "\${body}You can find the build logs amnd the test results attached.\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
279
280
281
282
                            body = "\${body}If you need further information e.g. the build environment or other runtime information, please contact:\\n"
                            body = "\${body}Build maintainer:     arno.bauernoppel@puri.sm\\n"
                            body = "\${body}Build mailing list:   https://lists.community.puri.sm/listinfo/librem5-builds\\n"
                            body = "\${body}The build maintainer: \${dev_mail}\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
283
                        } else if (currentBuild.result != 'NOT_BUILT' && currentBuild.result != 'ABORTED') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
284
                            currentBuild.result = 'SUCCESS'
Arno Bauernoppel's avatar
Arno Bauernoppel committed
285
                            body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and was successful.\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
286
287
                        } else {
                            error "unrecognised build result"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
288
                        }
289

Arno Bauernoppel's avatar
Arno Bauernoppel committed
290
291
292
                        Date date = new Date()
                        String date_string = date.format("yyyy/MM/dd HH:mm:ss")
                        body = "\${body}---\\n"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
293
                        body = "\${body}\${date_string}"
294

Arno Bauernoppel's avatar
Arno Bauernoppel committed
295
                        node ('${node_label}') {
Arno Bauernoppel's avatar
Arno Bauernoppel committed
296
297
                            emailext replyTo: '${email_from}',
                            to: '${email_to}',
Arno Bauernoppel's avatar
Arno Bauernoppel committed
298
                            subject: "[arm02] \${currentBuild.result} \${currentBuild.fullDisplayName}",
Arno Bauernoppel's avatar
Arno Bauernoppel committed
299
                            body: body,
300
                            attachLog: true,
Arno Bauernoppel's avatar
Arno Bauernoppel committed
301
302
                            compressLog: true,
                            attachmentsPattern: "*_result.tar.bz2"
Arno Bauernoppel's avatar
Arno Bauernoppel committed
303
                        }
Arno Bauernoppel's avatar
Arno Bauernoppel committed
304
305
306
307
                        
                        if(unstable_reason != "") {
                            error "\${unstable_reason}"
                        }
Arno Bauernoppel's avatar
Arno Bauernoppel committed
308
                    }
309
                } catch(err) {
310
311
312
313
314
                    if(currentBuild.result == 'NOT_BUILT') {
                        print "Build aborted. Build criteria not met."
                    }
                } finally {
                    cleanWs cleanWhenFailure: false, cleanWhenSuccess: true, deleteDirs: true
315
                }
316
            }
317
            """.stripIndent())
Guido Gunther's avatar
Guido Gunther committed
318
319
320
321
322
323
          }
        }
      }
    }
  }
}