diff options
Diffstat (limited to 'src/CI')
-rwxr-xr-x | src/CI/travis/after_deploy | 7 | ||||
-rwxr-xr-x | src/CI/travis/before_deploy | 17 | ||||
-rwxr-xr-x | src/CI/travis/before_install_darwin | 14 | ||||
-rwxr-xr-x | src/CI/travis/before_install_linux | 38 | ||||
-rwxr-xr-x | src/CI/travis/deploy | 74 | ||||
-rw-r--r-- | src/CI/travis/generateDocumentationAndDeploy.sh.in | 97 | ||||
-rwxr-xr-x | src/CI/travis/inside_docker.sh | 26 | ||||
-rw-r--r-- | src/CI/travis/jobs_running_cnt.py | 42 | ||||
-rw-r--r-- | src/CI/travis/lib.sh | 304 | ||||
-rwxr-xr-x | src/CI/travis/make_darwin | 2 | ||||
-rwxr-xr-x | src/CI/travis/make_linux | 42 | ||||
-rwxr-xr-x | src/CI/travis/make_linux_qemu | 13 | ||||
-rwxr-xr-x | src/CI/travis/setup_qemu_for_arm.sh | 87 |
13 files changed, 669 insertions, 94 deletions
diff --git a/src/CI/travis/after_deploy b/src/CI/travis/after_deploy new file mode 100755 index 0000000..66705aa --- /dev/null +++ b/src/CI/travis/after_deploy @@ -0,0 +1,7 @@ +#!/bin/sh -e + +. CI/travis/lib.sh + +should_trigger_next_builds "$TRAVIS_BRANCH" || exit 0 + +trigger_adi_build "libad9361-iio" "$TRAVIS_BRANCH" diff --git a/src/CI/travis/before_deploy b/src/CI/travis/before_deploy index 1164dcf..9d3fd05 100755 --- a/src/CI/travis/before_deploy +++ b/src/CI/travis/before_deploy @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/sh -e + +. CI/travis/lib.sh # Don't prepare a deploy on a Coverity build if [ "x${COVERITY_SCAN_PROJECT_NAME}" != "x" ] ; then exit 0; fi @@ -18,8 +20,11 @@ else fi pwd +if [ -z "${LDIST}" -a -f "build/.LDIST" ] ; then + export LDIST="-$(cat build/.LDIST)" +fi if [ -z "${LDIST}" ] ; then - LDIST=-$(lsb_release -c | awk '{print $NF}') + export LDIST="-$(get_ldist)" fi check_file() @@ -27,7 +32,7 @@ check_file() temp="" for i in $(find ./ -name CMakeCache.txt) do -hit=$(find $(dirname ${i}) -maxdepth 1 -name "libiio*.$1" | grep -v -- ${LDIST}) +hit=$(find $(dirname ${i}) -maxdepth 1 -name "libiio*.$1" -a ! -name "*${LDIST}*") if [ "$(echo ${hit} | wc -w)" -gt "1" ] ; then echo "I am confused - more than 2 $1 files!" echo $hit @@ -52,7 +57,7 @@ if [ -n "${temp}" ] ; then deploy=$(expr ${deploy} + 1) if [ -z "${TARGET_DEB}" ] ; then export TARGET_DEB=$(echo ${temp} | \ - sed -e 's:^./.*/::' -e 's:.deb$::')${LDIST}.deb + sed -e 's:^./.*/::' -e 's:-Linux::' -e 's:.deb$::')${LDIST}.deb fi echo "deploying ${temp} to nightly $TARGET_DEB" if [ -z "${RELEASE_PKG_FILE_DEB}" ] ; then @@ -72,7 +77,7 @@ if [ -n "${temp}" ] ; then deploy=$(expr ${deploy} + 1) if [ -z "${TARGET_RPM}" ] ; then export TARGET_RPM=$(echo ${temp} | \ - sed -e 's:^./.*/::' -e 's:.rpm$::')${LDIST}.rpm + sed -e 's:^./.*/::' -e 's:-Linux::' -e 's:.rpm$::')${LDIST}.rpm fi echo "deploying ${temp} to nightly $TARGET_RPM" if [ -z "${RELEASE_PKG_FILE_RPM}" ] ; then @@ -137,7 +142,7 @@ if [ -n "${temp}" ] ; then ) export TARGET_TGZ=$(echo ${temp} | \ - sed -e 's:^./.*/::' -e 's:.tar.gz$::')${LDIST}.tar.gz; + sed -e 's:^./.*/::' -e 's:-Linux::' -e 's:-Darwin::' -e 's:.tar.gz$::')${LDIST}.tar.gz; fi echo "deploying ${temp} to $TARGET_TGZ" if [ -z "${RELEASE_PKG_FILE_TGZ}" ] ; then diff --git a/src/CI/travis/before_install_darwin b/src/CI/travis/before_install_darwin index 68da765..f8645b3 100755 --- a/src/CI/travis/before_install_darwin +++ b/src/CI/travis/before_install_darwin @@ -1,11 +1,5 @@ -#!/bin/sh +#!/bin/sh -e -brew update -if ! brew ls --version cmake &>/dev/null; then - brew install cmake -else - brew upgrade cmake -fi -brew install doxygen -brew install --build-from-source libusb -brew install libxml2 +. CI/travis/lib.sh + +brew_install_or_upgrade cmake doxygen libusb libxml2 diff --git a/src/CI/travis/before_install_linux b/src/CI/travis/before_install_linux index 76ee7db..b25a781 100755 --- a/src/CI/travis/before_install_linux +++ b/src/CI/travis/before_install_linux @@ -1,8 +1,34 @@ -#!/bin/sh +#!/bin/sh -e -sudo apt-get -qq update -sudo apt-get install -y cmake doxygen libaio-dev libavahi-client-dev libavahi-common-dev libusb-1.0-0-dev libxml2-dev rpm tar bzip2 gzip -if [ `sudo apt-cache search libserialport-dev | wc -l` -gt 0 ] ; then - sudo apt-get install -y libserialport-dev -fi +. CI/travis/lib.sh +handle_centos() { + # FIXME: see about adding `libserialport-dev` from EPEL ; maybe libusb-1.0.0-devel... + yum -y groupinstall 'Development Tools' + yum -y install cmake libxml2-devel libusb1-devel doxygen libaio-devel \ + avahi-devel bzip2 gzip rpm rpm-build +} + +handle_centos_docker() { + prepare_docker_image "centos:centos${OS_VERSION}" +} + +handle_ubuntu_docker() { + prepare_docker_image "ubuntu:${OS_VERSION}" +} + +handle_default() { + sudo apt-get -qq update + sudo apt-get install -y cmake graphviz libaio-dev libavahi-client-dev libavahi-common-dev libusb-1.0-0-dev libxml2-dev rpm tar bzip2 gzip flex bison git + if [ -n "${GH_DOC_TOKEN}" ] ; then + sudo apt-get install -y doxygen + fi + if [ `sudo apt-cache search libserialport-dev | wc -l` -gt 0 ] ; then + sudo apt-get install -y libserialport-dev + fi +} + +OS_TYPE=${1:-default} +OS_VERSION=${2} + +handle_${OS_TYPE} diff --git a/src/CI/travis/deploy b/src/CI/travis/deploy index 18941f7..63a3f60 100755 --- a/src/CI/travis/deploy +++ b/src/CI/travis/deploy @@ -1,70 +1,10 @@ -#!/bin/sh +#!/bin/sh -e -cd $TRAVIS_BUILD_DIR +. CI/travis/lib.sh -send() -{ -if [ "$#" -ne 3 ] ; then - echo "skipping deployment of something" - echo "send called with $@" - return -fi - -if [ "x$1" = "x" ] ; then - echo no file to send - return -fi - -if [ ! -r "$1" ] ; then - echo "file $1 is not readable" - ls -l $1 - return -fi - -if [ $BRANCH_PULL ] ; then - branch=$BRANCH_PULL -else - branch=$BRANCH -fi - -FROM=$1 -TO=${branch}_$2 -LATE=${branch}_latest_libiio${LDIST}$3 -GLOB=${DEPLOY_TO}/${branch}_libiio-* - -echo attemting to deploy $FROM to $TO -echo and ${branch}_libiio${LDIST}$3 -ssh -V - -echo "cd ${DEPLOY_TO}" > script$3 -if curl -m 10 -s -I -f -o /dev/null http://swdownloads.analog.com/cse/travis_builds/${TO} ; then - echo "rm ${TO}" >> script$3 -fi -echo "put ${FROM} ${TO}" >> script$3 -echo "ls -l ${TO}" >> script$3 -if curl -m 10 -s -I -f -o /dev/null http://swdownloads.analog.com/cse/travis_builds/${LATE} ; then - echo "rm ${LATE}" >> script$3 -fi -echo "symlink ${TO} ${LATE}" >> script$3 -echo "ls -l ${LATE}" >> script$3 -echo "bye" >> script$3 - -sftp ${EXTRA_SSH} -b script$3 ${SSHUSER}@${SSHHOST} - -# limit things to a few files, so things don't grow forever -if [ "${LDIST}" = "-precise" -a "$3" = ".deb" ] ; then - for files in $(ssh ${EXTRA_SSH} ${SSHUSER}@${SSHHOST} \ - "ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}') - do - ssh ${EXTRA_SSH} ${SSHUSER}@${SSHHOST} \ - "rm ${DEPLOY_TO}/${files}" - done -fi -} - -# from to suffix -send ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb -send ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm -send ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz -send ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg +# libname from to suffix +upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb +upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm +upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz +upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg diff --git a/src/CI/travis/generateDocumentationAndDeploy.sh.in b/src/CI/travis/generateDocumentationAndDeploy.sh.in new file mode 100644 index 0000000..079913a --- /dev/null +++ b/src/CI/travis/generateDocumentationAndDeploy.sh.in @@ -0,0 +1,97 @@ +#!/bin/sh +set -x +################################################################################ +# Title : generateDocumentationAndDeploy.sh +# Date created : 16Nov2018 +# Original Author: "Jeroen de Bruijn" +# based on https://gist.github.com/vidavidorra/548ffbcdae99d752da02 +# +# Preconditions: +# - Packages doxygen graphviz must be installed. +# - An gh-pages branch should already exist. See below for mor info on how to +# create a gh-pages branch. +# +# Required global variables: +# - GH_DOC_TOKEN : Secure token to the github repository. +# +# This script will generate Doxygen documentation and push the documentation to +# the gh-pages branch of a repository specified by $TRAVIS_REPO_SLUG +# Before this script is used there should already be a gh-pages branch in the +# repository. +# +# This file is processed by CMAKE to get the version in the commit message +# +################################################################################ + +##### Setup this script and get the current gh-pages branch. +echo 'Setting up the script...' +GH_REPO_NAME=$(echo $TRAVIS_REPO_SLUG | awk -F/ '{print $2}') + +# Exit with nonzero exit code if anything fails +set -e + +# by the time this script is run, we should have already made the docs +cd $TRAVIS_BUILD_DIR/build +#docs should be in the $TRAVIS_BUILD_DIR/build/html directory +if [ ! -d "html" ] || [ ! -f "./html/index.html" ]; then + echo '' >&2 + echo 'Warning: No documentation (html) files have been found!' >&2 + echo 'Warning: Not going to push the documentation to GitHub!' >&2 + exit 0 +fi + +if [ -z "${TRAVIS_TAG}" ] ; then + echo 'Warning: Not a tag' >&2 + echo 'Warning: Not going to push the documentation to GitHub!' >&2 + exit 0 +fi + +# Get the current gh-pages branch +git clone -b gh-pages https://git@github.com/$TRAVIS_REPO_SLUG +cd $GH_REPO_NAME + +# Remove everything currently in the gh-pages branch. +# GitHub is smart enough to know which files have changed and which files have +# stayed the same and will only update the changed files. So the gh-pages branch +# can be safely cleaned, and it is sure that everything pushed later is the new +# documentation. +rm -rf * + +#copy the files over +cp -a ../html/* ./ + +##### Configure git. +# Set the push default to simple i.e. push only the current branch. +git config --global push.default simple +# Pretend to be an user called Travis CI. +git config user.name "Autogenerated by Travis CI" +git config user.email "robin.getz@analog.com" + +# Need to create a .nojekyll file to allow filenames starting with an underscore +# to be seen on the gh-pages site. Therefore creating an empty .nojekyll file. +# Presumably this is only needed when the SHORT_NAMES option in Doxygen is set +# to NO, which it is by default. So creating the file just in case. +if [ ! -f ".nojekyll" ] ; then + touch .nojekyll +fi + +################################################################################ +##### Upload the documentation to the gh-pages branch of the repository. ##### + +echo 'Uploading documentation to the gh-pages branch...' +# Add everything in this directory (the Doxygen code documentation) to the +# gh-pages branch. +# +# GitHub is smart enough to know which files have changed and which files have +# stayed the same and will only update the changed files. +git add --all + +# Commit the added files with a title and description containing the Travis CI +# build number and the GitHub commit reference that issued this build. +git commit -m "Deploy autogenerated docs for ${GH_REPO_NAME} v@LIBIIO_VERSION_MAJOR@.@LIBIIO_VERSION_MINOR@-g@LIBIIO_VERSION_GIT@" --sign + +# Force push to the remote gh-pages branch. +# The ouput is redirected to /dev/null to hide any sensitive credential data +# that might otherwise be exposed. +git push --force "https://${GH_DOC_TOKEN}@github.com/${TRAVIS_REPO_SLUG}" > /dev/null 2>&1 + diff --git a/src/CI/travis/inside_docker.sh b/src/CI/travis/inside_docker.sh new file mode 100755 index 0000000..7aa9522 --- /dev/null +++ b/src/CI/travis/inside_docker.sh @@ -0,0 +1,26 @@ +#!/bin/sh -e + +LIBNAME="$1" +OS_TYPE="$2" + +export INSIDE_DOCKER="1" +export TRAVIS_BUILD_DIR="/$LIBNAME" + +cd /$LIBNAME + +if [ -d "/$LIBNAME/CI" ] ; then + CI="/$LIBNAME/CI" +elif [ -d "/$LIBNAME/ci" ] ; then + CI="/$LIBNAME/ci" +else + echo "No CI/ci directory present" + exit 1 +fi + +$CI/travis/before_install_linux "$OS_TYPE" + +$CI/travis/make_linux "$OS_TYPE" + +# need to find this out inside the container +. $CI/travis/lib.sh +echo "$(get_ldist)" > /${LIBNAME}/build/.LDIST diff --git a/src/CI/travis/jobs_running_cnt.py b/src/CI/travis/jobs_running_cnt.py new file mode 100644 index 0000000..e5f4121 --- /dev/null +++ b/src/CI/travis/jobs_running_cnt.py @@ -0,0 +1,42 @@ +#!/usr/bin/python + +import os +import sys +import urllib2 +import json + +# This is pretty constant, but allow it to be overriden via env-var +url = os.getenv('TRAVIS_API_URL', 'https://api.travis-ci.org') + +if (not url.lower().startswith("https://")): + print (0) + sys.exit(0) + +ci_token = os.getenv('TRAVIS_API_TOKEN') +build_id = os.getenv('TRAVIS_BUILD_ID') + +headers = { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Travis-API-Version': "3", + 'Authorization': "token {0}".format(ci_token) +} + +# Codacy's bandit linter may complain that we haven't validated +# this URL for permitted schemes; we have validated this a few lines above +req = urllib2.Request("{0}/build/{1}/jobs".format(url, build_id), + headers=headers) + +response = urllib2.urlopen(req).read() +json_r = json.loads(response.decode('utf-8')) + +jobs_running = 0 +for job in json_r['jobs']: + # bump number of jobs higher, so nothing triggers + if (job['state'] in [ 'canceled', 'failed' ]): + jobs_running += 99 + break + if (job['state'] in [ 'started', 'created', 'queued', 'received' ]): + jobs_running += 1 + +print (jobs_running) diff --git a/src/CI/travis/lib.sh b/src/CI/travis/lib.sh new file mode 100644 index 0000000..a071cfc --- /dev/null +++ b/src/CI/travis/lib.sh @@ -0,0 +1,304 @@ +#!/bin/sh -e + +export TRAVIS_API_URL="https://api.travis-ci.org" +LOCAL_BUILD_DIR=${LOCAL_BUILD_DIR:-build} + +COMMON_SCRIPTS="jobs_running_cnt.py inside_docker.sh" + +echo_red() { printf "\033[1;31m$*\033[m\n"; } +echo_green() { printf "\033[1;32m$*\033[m\n"; } +echo_blue() { printf "\033[1;34m$*\033[m\n"; } + +get_script_path() { + local script="$1" + + [ -n "$script" ] || return 1 + + if [ -f "CI/travis/$script" ] ; then + echo "CI/travis/$script" + elif [ -f "ci/travis/$script" ] ; then + echo "ci/travis/$script" + elif [ -f "${LOCAL_BUILD_DIR}/$script" ] ; then + echo "${LOCAL_BUILD_DIR}/$script" + else + return 1 + fi +} + +pipeline_branch() { + local branch=$1 + + [ -n "$branch" ] || return 1 + + # master is a always a pipeline branch + [ "$branch" = "master" ] && return 0 + + set +x + # Check if branch name is 20XX_RY where: + # XX - 14 to 99 /* wooh, that's a lot of years */ + # Y - 1 to 9 /* wooh, that's a lot of releases per year */ + for year in $(seq 2014 2099) ; do + for rel_num in $(seq 1 9) ; do + [ "$branch" = "${year}_R${rel_num}" ] && \ + return 0 + done + done + + return 1 +} + +should_trigger_next_builds() { + local branch="$1" + + [ -z "${COVERITY_SCAN_PROJECT_NAME}" ] || return 1 + + # These Travis-CI vars have to be non-empty + [ -n "$TRAVIS_PULL_REQUEST" ] || return 1 + [ -n "$branch" ] || return 1 + set +x + [ -n "$TRAVIS_API_TOKEN" ] || return 1 + + # Has to be a non-pull-request + [ "$TRAVIS_PULL_REQUEST" = "false" ] || return 1 + + pipeline_branch "$branch" || return 1 + + local python_script="$(get_script_path jobs_running_cnt.py)" + if [ -z "$python_script" ] ; then + echo "Could not find 'jobs_running_cnt.py'" + return 1 + fi + + local jobs_cnt=$(python $python_script) + + # Trigger next job if we are the last job running + [ "$jobs_cnt" = "1" ] +} + +trigger_build() { + local repo_slug="$1" + local branch="$2" + + [ -n "$repo_slug" ] || return 1 + [ -n "$branch" ] || return 1 + + local body="{ + \"request\": { + \"branch\":\"$branch\" + } + }" + + # Turn off tracing here (shortly) + set +x + curl -s -X POST \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -H "Travis-API-Version: 3" \ + -H "Authorization: token $TRAVIS_API_TOKEN" \ + -d "$body" \ + https://api.travis-ci.org/repo/$repo_slug/requests +} + +trigger_adi_build() { + local adi_repo="$1" + local branch="$2" + + [ -n "$adi_repo" ] || return 1 + trigger_build "analogdevicesinc%2F$adi_repo" "$branch" +} + +command_exists() { + local cmd=$1 + [ -n "$cmd" ] || return 1 + type "$cmd" >/dev/null 2>&1 +} + +get_ldist() { + case "$(uname)" in + Linux*) + if [ ! -f /etc/os-release ] ; then + if [ -f /etc/centos-release ] ; then + echo "centos-$(sed -e 's/CentOS release //' -e 's/(.*)$//' \ + -e 's/ //g' /etc/centos-release)-$(uname -m)" + return 0 + fi + ls /etc/*elease + [ -z "${OSTYPE}" ] || { + echo "${OSTYPE}-unknown" + return 0 + } + echo "linux-unknown" + return 0 + fi + . /etc/os-release + if ! command_exists dpkg ; then + echo $ID-$VERSION_ID-$(uname -m) + else + echo $ID-$VERSION_ID-$(dpkg --print-architecture) + fi + ;; + Darwin*) + echo "darwin-$(sw_vers -productVersion)" + ;; + *) + echo "$(uname)-unknown" + ;; + esac + return 0 +} + +__brew_install_or_upgrade() { + brew install $1 || \ + brew upgrade $1 || \ + brew ls --version $1 +} + +brew_install_or_upgrade() { + while [ -n "$1" ] ; do + __brew_install_or_upgrade "$1" || return 1 + shift + done +} + +sftp_cmd_pipe() { + sftp ${EXTRA_SSH} ${SSHUSER}@${SSHHOST} +} + +sftp_rm_artifact() { + local artifact="$1" + sftp_cmd_pipe <<-EOF + cd ${DEPLOY_TO} + rm ${artifact} + EOF +} + +sftp_upload() { + local FROM="$1" + local TO="$2" + local LATE="$3" + + sftp_cmd_pipe <<-EOF + cd ${DEPLOY_TO} + + put ${FROM} ${TO} + ls -l ${TO} + + symlink ${TO} ${LATE} + ls -l ${LATE} + bye + EOF +} + +upload_file_to_swdownloads() { + + if [ "$#" -ne 4 ] ; then + echo "skipping deployment of something" + echo "send called with $@" + return 0 + fi + + local LIBNAME=$1 + local FROM=$2 + local FNAME=$3 + local EXT=$4 + + if [ -z "$FROM" ] ; then + echo no file to send + return 1 + fi + + if [ ! -r "$FROM" ] ; then + echo "file $FROM is not readable" + return 1 + fi + + if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then + local branch="$TRAVIS_PULL_REQUEST_BRANCH" + else + local branch="$TRAVIS_BRANCH" + fi + + local TO=${branch}_${FNAME} + local LATE=${branch}_latest_${LIBNAME}${LDIST}${EXT} + local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-* + + echo attemting to deploy $FROM to $TO + echo and ${branch}_${LIBNAME}${LDIST}${EXT} + ssh -V + + for rmf in ${TO} ${LATE} ; do + sftp_rm_artifact ${rmf} || \ + echo_blue "Could not delete ${rmf}" + done + + sftp_upload "${FROM}" "${TO}" "${LATE}" || { + echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'" + return 1 + } + + # limit things to a few files, so things don't grow forever + if [ "${EXT}" = ".deb" ] ; then + for files in $(ssh ${EXTRA_SSH} ${SSHUSER}@${SSHHOST} \ + "ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}') + do + ssh ${EXTRA_SSH} ${SSHUSER}@${SSHHOST} \ + "rm ${DEPLOY_TO}/${files}" || \ + return 1 + done + fi + + return 0 +} + +prepare_docker_image() { + local DOCKER_IMAGE="$1" + sudo apt-get -qq update + echo 'DOCKER_OPTS="-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock -s devicemapper"' | sudo tee /etc/default/docker > /dev/null + sudo service docker restart + sudo docker pull "$DOCKER_IMAGE" +} + +run_docker_script() { + local DOCKER_SCRIPT="$(get_script_path $1)" + local DOCKER_IMAGE="$2" + local OS_TYPE="$3" + local MOUNTPOINT="${4:-docker_build_dir}" + sudo docker run --rm=true \ + -v "$(pwd):/${MOUNTPOINT}:rw" \ + $DOCKER_IMAGE \ + /bin/bash -e "/${MOUNTPOINT}/${DOCKER_SCRIPT}" "${MOUNTPOINT}" "${OS_TYPE}" +} + +ensure_command_exists() { + local cmd="$1" + local package="$2" + [ -n "$cmd" ] || return 1 + [ -n "$package" ] || package="$cmd" + ! command_exists "$cmd" || return 0 + # go through known package managers + for pacman in apt-get brew yum ; do + command_exists $pacman || continue + $pacman install -y $package || { + # Try an update if install doesn't work the first time + $pacman -y update && \ + $pacman install -y $package + } + return $? + done + return 1 +} + +ensure_command_exists sudo + +# Other scripts will download lib.sh [this script] and lib.sh will +# in turn download the other scripts it needs. +# This gives way more flexibility when changing things, as they propagate +for script in $COMMON_SCRIPTS ; do + [ ! -f "CI/travis/$script" ] || continue + [ ! -f "ci/travis/$script" ] || continue + [ ! -f "${LOCAL_BUILD_DIR}/$script" ] || continue + mkdir -p ${LOCAL_BUILD_DIR} + ensure_command_exists wget + wget https://raw.githubusercontent.com/analogdevicesinc/libiio/master/CI/travis/$script \ + -O $LOCAL_BUILD_DIR/$script +done diff --git a/src/CI/travis/make_darwin b/src/CI/travis/make_darwin index 4fe4e44..aff1af9 100755 --- a/src/CI/travis/make_darwin +++ b/src/CI/travis/make_darwin @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e if [ "x${COVERITY_SCAN_PROJECT_NAME}" != "x" ] ; then exit 0; fi diff --git a/src/CI/travis/make_linux b/src/CI/travis/make_linux index 688fabd..0e1e5c9 100755 --- a/src/CI/travis/make_linux +++ b/src/CI/travis/make_linux @@ -1,8 +1,42 @@ -#!/bin/sh +#!/bin/sh -e if [ "x${COVERITY_SCAN_PROJECT_NAME}" != "x" ] ; then exit 0; fi -cd $TRAVIS_BUILD_DIR/build -cmake -DENABLE_PACKAGING=ON -DDEB_DETECT_DEPENDENCIES=ON .. -make && make package +. CI/travis/lib.sh + +handle_default() { + mkdir -p build + cd build + cmake -DENABLE_PACKAGING=ON -DDEB_DETECT_DEPENDENCIES=ON .. + make && make package + if [ -n "${GH_DOC_TOKEN}" ] && \ + [ -f "./generateDocumentationAndDeploy.sh" ] ; then + sh generateDocumentationAndDeploy.sh + fi + cd .. +} + +handle_centos() { + mkdir -p build + cd build + cmake -DENABLE_PACKAGING=ON .. + make && make package + cd .. +} + +handle_centos_docker() { + run_docker_script inside_docker.sh \ + "centos:centos${OS_VERSION}" "centos" +} + +handle_ubuntu_docker() { + run_docker_script inside_docker.sh \ + "ubuntu:${OS_VERSION}" +} + +LIBNAME="$1" +OS_TYPE=${2:-default} +OS_VERSION="$3" + +handle_${OS_TYPE} diff --git a/src/CI/travis/make_linux_qemu b/src/CI/travis/make_linux_qemu new file mode 100755 index 0000000..b1d101d --- /dev/null +++ b/src/CI/travis/make_linux_qemu @@ -0,0 +1,13 @@ +#!/bin/bash +set -ex + +CHROOT_DIR=/tmp/arm-chroot + +sudo chroot ${CHROOT_DIR} bash -c "cd ${TRAVIS_BUILD_DIR} && ./CI/travis/make_linux libiio" +sudo chroot ${CHROOT_DIR} bash -c ". ${TRAVIS_BUILD_DIR}/CI/travis/lib.sh && get_ldist > ${TRAVIS_BUILD_DIR}/build/.LDIST" + +#move the artifacts back to the non-qemu place +sudo rsync -av ${CHROOT_DIR}/${TRAVIS_BUILD_DIR}/ ${TRAVIS_BUILD_DIR}/ +#make sure the normal travis user can read them +sudo chown -R ${USER} ${TRAVIS_BUILD_DIR} + diff --git a/src/CI/travis/setup_qemu_for_arm.sh b/src/CI/travis/setup_qemu_for_arm.sh new file mode 100755 index 0000000..2904e94 --- /dev/null +++ b/src/CI/travis/setup_qemu_for_arm.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Based on a test script from avsm/ocaml repo https://github.com/avsm/ocaml +set -ex + +if [ $# -ne 1 ] ; then + echo Must include debian distribution ie wheezy, jessie, stretch, or buster + exit -1 +fi +if [[ ! $1 =~ ^wheezy|jessie|stretch|buster$ ]] ; then + echo Must include debian distribution ie wheezy, jessie, stretch, or buster + exit -1 +fi + +CHROOT_DIR=/tmp/arm-chroot +MIRROR=http://archive.raspbian.org/raspbian +# wheezy = 7 (2013-05-04); jessie = 8 (2015-04-26); stretch = 9 (2017-06-17); buster = 10 (2019-03-12) +VERSION=$1 +echo building for ${VERSION} +CHROOT_ARCH=armhf + +# Host dependencies +sudo apt-get update +sudo apt-get install -qq -y qemu-user-static binfmt-support sbuild wget debian-archive-keyring ubuntu-keyring gnupg libudev1 libudev-dev liblzma-dev libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev +# need a newer version of qemu :( +QEMU_DIR=/tmp/qemu +sudo mkdir -p ${QEMU_DIR} +sudo chown ${USER} /tmp/qemu +cd ${QEMU_DIR} +wget https://download.qemu.org/qemu-3.1.0.tar.xz +tar xf qemu-3.1.0.tar.xz +cd qemu-3.1.0 +mkdir build +mkdir install +cd build +../configure --prefix=${QEMU_DIR}/install --disable-bsd-user --disable-guest-agent --disable-strip --disable-werror --disable-gcrypt --disable-debug-info --disable-debug-tcg --disable-docs --disable-tcg-interpreter --enable-attr --disable-brlapi --disable-linux-aio --disable-bzip2 --disable-bluez --disable-cap-ng --disable-curl --disable-fdt --disable-glusterfs --disable-gnutls --disable-nettle --disable-gtk --disable-rdma --disable-libiscsi --disable-vnc-jpeg --disable-kvm --disable-lzo --disable-curses --disable-libnfs --disable-numa --disable-opengl --disable-vnc-png --disable-rbd --disable-vnc-sasl --disable-sdl --disable-seccomp --disable-smartcard --disable-snappy --disable-spice --disable-libssh2 --disable-libusb --disable-usb-redir --disable-vde --disable-vhost-net --disable-virglrenderer --disable-virtfs --disable-vnc --disable-vte --disable-xen --disable-xen-pci-passthrough --disable-xfsctl --enable-linux-user --disable-system --disable-blobs --disable-tools --target-list=arm-linux-user --static --disable-pie +make -j $(nproc) +make install +ls -lR ${QEMU_DIR}/install +cd /tmp + +# per https://wiki.ubuntu.com/DebootstrapChroot +if [[ "$VERSION" =~ ^wheezy|jessie$ ]] ; then + sudo apt-get install -qq -y debootstrap + +elif [[ "$VERSION" =~ ^stretch$ ]] ; then + sudo add-apt-repository -r "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse " + sudo apt-get install -qq -y -t $(lsb_release -cs)-updates debootstrap +elif [[ "$VERSION" =~ ^buster$ ]] ; then + wget http://http.us.debian.org/debian/pool/main/d/debootstrap/debootstrap_1.0.111_all.deb -O /tmp/debootstrap_1.0.111_all.deb + sudo dpkg --install /tmp/debootstrap_1.0.111_all.deb +fi + +# Create chrooted environment +sudo mkdir ${CHROOT_DIR} +sudo debootstrap --foreign --no-check-gpg --include=fakeroot,build-essential --arch=${CHROOT_ARCH} ${VERSION} ${CHROOT_DIR} ${MIRROR} +sudo cp ${QEMU_DIR}/install/bin/qemu-arm ${CHROOT_DIR}/usr/bin/qemu-arm-static +${CHROOT_DIR}/usr/bin/qemu-arm-static -version +sudo chroot ${CHROOT_DIR} ./debootstrap/debootstrap --second-stage +sudo sbuild-createchroot --arch=${CHROOT_ARCH} --foreign --setup-only ${VERSION} ${CHROOT_DIR} ${MIRROR} + +# Create file with environment variables which will be used inside chrooted +# environment +echo "export ARCH=${ARCH}" > envvars.sh +echo "export TRAVIS_BUILD_DIR=${TRAVIS_BUILD_DIR}" >> envvars.sh +chmod a+x envvars.sh + +# Install dependencies inside chroot +sudo chroot ${CHROOT_DIR} dpkg --add-architecture ${CHROOT_ARCH} +sudo chroot ${CHROOT_DIR} dpkg --remove-architecture amd64 +sudo chroot ${CHROOT_DIR} apt-get update +sudo chroot ${CHROOT_DIR} apt-get --allow-unauthenticated install -qq -y locales +sudo chroot ${CHROOT_DIR} locale +sudo chroot ${CHROOT_DIR} bash -c "echo en_US.UTF-8 UTF-8 > /etc/locale.gen" +sudo chroot ${CHROOT_DIR} locale-gen +#sudo chroot ${CHROOT_DIR} bash -c "echo -e 'LANG=\"en_US.UTF-8\"\\nLANGUAGE=\"en_US:en\"\\n' > /etc/default/locale" +sudo chroot ${CHROOT_DIR} apt-get --allow-unauthenticated install -qq -y build-essential git m4 sudo python cmake + +# Create build dir and copy travis build files to our chroot environment +sudo mkdir -p ${CHROOT_DIR}/${TRAVIS_BUILD_DIR} +sudo rsync -av ${TRAVIS_BUILD_DIR}/ ${CHROOT_DIR}/${TRAVIS_BUILD_DIR}/ + +# Indicate chroot environment has been set up +sudo touch ${CHROOT_DIR}/.chroot_is_done + +# Call standard before_install_linux in chroot environment +sudo chroot ${CHROOT_DIR} bash -c "cd ${TRAVIS_BUILD_DIR} && pwd && ./CI/travis/before_install_linux" + |