Commit abc2cc83 authored by Guido Gunther's avatar Guido Gunther
Browse files

Merge branch 'fix-tag' into 'master'

Use the correct tag when building

Closes #25

See merge request Librem5/deb-build-jobs!283
parents 54891607 0cc204d0
......@@ -91,28 +91,18 @@ pkgs.each { pkg ->
"""
}
scm_tag_cmd = """commit="\$(git for-each-ref --sort=taggerdate --format '%(objectname)' refs/tags | tail -n 1)"; head_tag="\$(git describe --exact-match --tags \${commit})" 2> /dev/null || exit 128""".stripIndent()
scm_verify_tag_cmd = "/bin/false" // save default
dev_mail = params.get('gpg-allowed-dev','')
if(! dev_mail.equals('')) {
dev_mail = "'" + dev_mail + "'"
//verify dev tag in output
scm_verify_tag_cmd = """git tag -v "\${head_tag}" 2>&1 | grep -F @ | perl -e "\\\\\$ret=0; while(<>){ \\\\\$_=~/Good signature/ and \\\\\$ret++; index(\\\\\$_, ${dev_mail}) != -1 and \\\\\$ret++; }; exit \\\\\$ret - 2;" || exit 64""".stripIndent()
} else {
scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
}
scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
scm_checkout_cmd = """cd build
${submodule_update_cmd}
git clean -dfx
$scm_tag_cmd
git checkout -f "\${head_tag}"
git checkout -f
""".stripIndent()
build_cmd = """
rm -f *.deb *.changes *.dsc *.upload *.tar.* *.buildinfo *.build
${scm_checkout_cmd}
head_tag=\$(git describe --exact-match HEAD)
${scm_verify_tag_cmd}
#If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
......
// Copyright (C) 2018 Purism SPC
// SPDX-License-Identifier: GPL-3.0+
//
// Author: Guido Gunther <agx@sigxcpu.org>
// Editor: Arno Bauernöppel <arno.bauernoppel@puri.sm>
//
// Generate Pipeline jobs using to build Debian packages
// using the job-dsl plugin. This job is parameterized
// looking at tags found in git.
//
// https://github.com/jenkinsci/job-dsl-plugin
// API: https://jenkinsci.github.io/job-dsl-plugin/
@Grab('org.yaml:snakeyaml:1.17')
import org.yaml.snakeyaml.Yaml
// The Operating system we build for. This is the first part
// of the git branch: ${os}/${dist}
def os = 'debian'
// Default distribution to build for
def default_dists = ['amber-phone-staging']
// Default architectures to build for
def default_archs = ['aarch64']
// Options to pass to git-buildpackage
def default_build_args = '--git-no-pristine-tar --git-pbuilder --git-ignore-new --git-ignore-branch -nc --build=full -sa'
// the same but for sloppy builds
def sloppy_build_args = '--git-force-create --git-upstream-tree=SLOPPY'
// All jobs will be generated in this folder
def folder = 'manual_debs'
// Artifacts
def artifacts = '*.deb,*.changes,*.dsc,*.xz,*.gz'
// Signature (keyid) for signing the packages before upload
def pkg_gpg_keyid = '1CBB2345A7F02749' //jenkins@arm02.puri.sm
// Who get's mail notifications
def email_to = 'librem5-builds@lists.community.puri.sm'
def email_from = 'arno.bauernoppel@puri.sm'
// Load jobs from YAML file
Yaml yaml = new Yaml()
def jobsfile = (new File(__FILE__)).parent + '/jobs.yml'
println("Using jobsfile : ${jobsfile}")
def pkgs = yaml.load(new File(jobsfile).text)
def tags_map = [:]
def findTags(pkgs, tags_map) {
pkgs.each {
pkg ->
//Find all tags in the current repository
def outputBuffer = new StringBuffer();
def errorBuffer = new StringBuffer();
if(null != pkg.value) {
params = pkg.value
def stderr = new StringBuilder()
def stdout = new StringBuilder()
cmdline = "/usr/local/bin/fetch_tags.sh ${params.url}"
println "$cmdline"
proc = cmdline.execute()
proc.waitFor()
proc.waitForProcessOutput(outputBuffer, System.out)
proc.waitForProcessOutput(errorBuffer, System.err)
tags_map.put("$pkg.key","${outputBuffer}")
println "Err: ${errorBuffer}\nOut: ${outputBuffer}\n"
}
}
return tags_map
}
tags_map = findTags(pkgs, tags_map)
pkgs.each { pkg ->
params = pkg.value
deb_build_opts = params.get('deb_build_options', '')
deb_build_profiles = params.get('deb_build_profiles', '')
default_archs.each { arch ->
default_dists.each { dist ->
name = folder + '/' + 'deb-' + pkg.key + '-' + dist + '-' + arch
branch = params.get('branch', os + '/' + dist)
submodule_update = params.get('submodule_update', true)
tags_list = tags_map.get("$pkg.key")
if( tags_list == null) {
tags_list = [ "no_tags_found" ]
} else {
tags_list = tags_list.split("\n")
}
dist_arg = '--git-dist=' + dist
if (submodule_update) {
submodule_update_cmd = 'git submodule update --init --recursive'
} else {
submodule_update_cmd = ''
}
//Currently builds are only for aarch64
dpkg_args = ''
// Sloppy builds always build a new upstream tarball to save us from patch
// maintenance:
// https://honk.sigxcpu.org/projects/git-buildpackage/manual-html/gbp.special.sloppytarball.html
if (params.get('sloppy', false)) {
// Not much todo here, we just want to make sure we create a fresh tarball
// since for sloppy builds upstrem tarballs change on each commit:
// creates a new tarball anyway:
origtgz_cmd = ''
build_args = "${default_build_args} ${sloppy_build_args} --git-debian-branch=\${head_tag}"
} else { // regular blessed build
build_args = "${default_build_args}"
origtgz_cmd = """
git fetch origin pristine-tar || true
gbp export-orig --pristine-tar || gbp export-orig --no-pristine-tar
"""
}
scm_verify_tag_cmd = "exit 1" // save default
dev_mail = params.get('gpg-allowed-dev','')
if(! dev_mail.equals('')) {
dev_mail = "'" + dev_mail + "'"
//verify dev tag in output
scm_verify_tag_cmd = """git tag -v "\${head_tag}" 2>&1 | grep -F @ | perl -e "\\\\\$ret=0; while(<>){ \\\\\$_=~/Good signature/ and \\\\\$ret++; index(\\\\\$_, ${dev_mail}) != -1 and \\\\\$ret++; }; exit \\\\\$ret - 2;" || exit 64""".stripIndent()
} else {
scm_verify_tag_cmd = """git tag -v "\${head_tag}" || exit 64""".stripIndent()
}
scm_checkout_cmd = """cd build
${submodule_update_cmd}
git clean -dfx
git checkout -f "\${head_tag}"
""".stripIndent()
build_cmd = """
if [ -z "\${choosen_tag}" ]; then
echo "No tag given"
exit 1
fi
head_tag=\${choosen_tag#refs/tags/}
rm -f *.deb *.changes *.dsc *.upload *.tar.*
${scm_checkout_cmd}
${scm_verify_tag_cmd}
#If changelog does not contain the distribution amber-phone, amber-phone-staging or purple mark as not built
echo "Changelog: \$(dpkg-parsechangelog -S Distribution)"
dpkg-parsechangelog -S Distribution | grep -qsE '^(amber-phone(-staging)?|purple)\$' || exit 32
#If it is purple change to amber-phone (temporary)
sed -i '1 s/ purple;/ amber-phone;/' debian/changelog
# get the upstream tarball if necessary
${origtgz_cmd}
export DEB_BUILD_OPTIONS='${deb_build_opts.join(' ')}'
export DEB_BUILD_PROFILES='${deb_build_profiles.join(' ')}'
export GIT_PBUILDER_OPTIONS='--source-only-changes'
echo "Building ${pkg.key} tag: \${head_tag}"
gbp buildpackage ${[dist_arg, build_args, dpkg_args].join(' ')}
""".stripIndent()
upload_cmd = """
#Workaround to cache secret key
sudo /usr/bin/gpg --batch --pinentry-mode loopback --passphrase-file /home/jenkins/.secret_key -o /dev/null --decrypt /home/jenkins/.workaround.gpg
changes_file="\$(ls ./*_source.changes)"
[ \$(echo "\${changes_file}" | wc -w) -eq 1 ] || exit 4
sudo /usr/bin/debsign -k "${pkg_gpg_keyid}" "\${changes_file}" || exit 16
# finally upload the package
dput -c /etc/dput.cf pureos-ftp "\${changes_file}" || exit 32;
#Check with lintian for errors
lintian --suppress-tags bad-distribution-in-changes-file -c "\${changes_file}" || exit 64
""".stripIndent()
node_label = "${os}-${arch}"
pipelineJob(name) {
quietPeriod(30)
parameters {
booleanParam('do_upload', true, 'Upload package')
choiceParam('choosen_tag', tags_list.toList(), 'Tags of the repository')
}
definition {
cps {
sandbox()
script("""
node {
properties properties: [
disableConcurrentBuilds()
]
try {
def test_result = ""
def linitan_result = ""
def head_tag = ""
stage('Build Debian Package') {
node ('${node_label}') {
checkout([
changelog: true,
poll: true,
userRemoteConfigs: [[url: '${params.url}', refspec: '+refs/tags/*:refs/remotes/origin/tags/*']],
\$class: 'GitSCM',
branches: [[name: '**']],
extensions: [[\$class: 'RelativeTargetDirectory',
relativeTargetDir: 'build'],
],
])
def result = sh returnStatus: true, script: '''${build_cmd}'''
if(result != 0) {
if(result == 128) {
currentBuild.result = 'NOT_BUILT'
print "No tag on HEAD was found or tag was not signed. Aborting build."
throw "not built"
} else if (result == 64) {
currentBuild.result = 'NOT_BUILT'
if(params.get('gpg-allowed-dev', false)) {
print "Tag verification with key of " + params.get('gpg-allowed-dev', false) + " failed!"
} else {
print "Tag verification failed!"
}
throw "not built"
} else if (result == 32) {
currentBuild.result = 'NOT_BUILT'
print "Wrong distribution in changelog found."
throw "not built"
} else {
currentBuild.result = 'FAILURE'
print "Script returned " + result + ". Aborting build."
}
}
}
}
stage('Save Artifacts') {
node ('${node_label}') {
archiveArtifacts '${artifacts}'
}
}
stage('Upload Debian Packages') {
node ('${node_label}') {
if(params.do_upload == true) {
def result = sh returnStatus: true, script: '''${upload_cmd}'''
if(result != 0) {
if (result == 64) {
linitan_result = 'lintian failed'
print "Lintian check of package failed."
} else if (result == 32) {
currentBuild.result = 'FAILURE'
print "Upload of package failed."
} else if (result == 16) {
currentBuild.result = 'FAILURE'
print "Corresponding .dsc file not found for .changes file."
} else if (result == 8) {
currentBuild.result = 'FAILURE'
print "No .changes file found."
} else if (result == 4) {
currentBuild.result = 'FAILURE'
print "More than one .changes file found."
} else {
currentBuild.result = 'FAILURE'
print "Script returned " + result + ". Aborting build."
}
}
}
}
}
stage('Test Package') {
node ('${node_label}') {
if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'SUCCESS') {
sh '[ -d "/tmp/$pkg.key" ] || mkdir "/tmp/$pkg.key"'
def result = sh returnStatus: true, script: 'sudo /usr/local/bin/docker_run_test.sh "$pkg.key" "${dist}"'
if(result != 0) {
if(result == 128) {
test_result = "tests failed " + result
} else {
test_result = "unknown failure " + result
}
}
}
}
}
stage('Postbuild') {
unstable_reason = (unstable_reason != "")? "(\${unstable_reason})" : ""
def dev_mail = sh script: "git tag -v '\${head_tag}' 2>&1 | grep -F @ | perl -pe 's/.*<([^<]+)>[^>]+\\\$/\\\$1/g'",
returnStdout: true
def body = ""
if(currentBuild.result == 'UNSTABLE' || currentBuild.result == 'FAILURE') {
def ending = ""
if(currentBuild.result == 'UNSTABLE') {
ending = "was unstable."
} else {
ending = "has failed."
}
body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and \${ending}\\n"
body = "\${body}You can find the build logs amnd the test results attached.\\n"
body = "\${body}If you need further information e.g. the build environment or other runtime information, please contact:\\n"
body = "\${body}Build maintainer: arno.bauernoppel@puri.sm\\n"
body = "\${body}Build mailing list: https://lists.community.puri.sm/listinfo/librem5-builds\\n"
body = "\${body}The build maintainer: \${dev_mail}\\n"
} else if (currentBuild.result != 'NOT_BUILT' && currentBuild.result != 'ABORTED') {
currentBuild.result = 'SUCCESS'
body = "\${currentBuild.fullDisplayName} took \${currentBuild.durationString} and was successful.\\n"
} else {
error "unrecognised build result"
}
Date date = new Date()
String date_string = date.format("yyyy/MM/dd HH:mm:ss")
body = "\${body}---\\n"
body = "\${body}\${date_string}"
node ('${node_label}') {
emailext replyTo: '${email_from}',
to: '${email_to}',
subject: "[arm02] \${currentBuild.result} \${currentBuild.fullDisplayName}",
body: body,
attachLog: true,
compressLog: true,
attachmentsPattern: "*_result.tar.bz2"
}
if(unstable_reason != "") {
error "\${unstable_reason}"
}
}
} catch(err) {
if(currentBuild.result == 'NOT_BUILT') {
print "Build aborted. Build criteria not met."
}
} finally {
cleanWs cleanWhenFailure: false, cleanWhenSuccess: true, deleteDirs: true
}
}
""".stripIndent())
}
}
}
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment