From 657343a602ec5eddae6074656bfd0e8a91aaa8e8 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Sun, 10 Sep 2017 16:44:28 +0200 Subject: [PATCH 1/4] travis-ci: move Travis CI code into dedicated scripts Most of the Travis CI commands are in the '.travis.yml'. The yml format does not support functions and therefore code duplication is necessary to run commands across all builds. To fix this, add a library for common CI functions. Move all Travis CI code into dedicated scripts and make them call the library first. Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- .travis.yml | 88 ++++---------------------------------- ci/install-dependencies.sh | 43 +++++++++++++++++++ ci/lib-travisci.sh | 5 +++ ci/print-test-failures.sh | 18 ++++++++ ci/run-build.sh | 8 ++++ ci/run-linux32-docker.sh | 23 ++++++++++ ci/run-static-analysis.sh | 8 ++++ ci/run-tests.sh | 10 +++++ ci/run-windows-build.sh | 2 + ci/test-documentation.sh | 4 +- 10 files changed, 128 insertions(+), 81 deletions(-) create mode 100755 ci/install-dependencies.sh create mode 100755 ci/lib-travisci.sh create mode 100755 ci/print-test-failures.sh create mode 100755 ci/run-build.sh create mode 100755 ci/run-linux32-docker.sh create mode 100755 ci/run-static-analysis.sh create mode 100755 ci/run-tests.sh diff --git a/.travis.yml b/.travis.yml index 278943d14a..fead995edd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,23 +61,8 @@ matrix: services: - docker before_install: - - docker pull daald/ubuntu32:xenial before_script: - script: - - > - docker run - --interactive - --env DEVELOPER - --env DEFAULT_TEST_TARGET - --env GIT_PROVE_OPTS - --env GIT_TEST_OPTS - --env GIT_TEST_CLONE_2GB - --volume "${PWD}:/usr/src/git" - daald/ubuntu32:xenial - /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) - # Use the following command to debug the docker build locally: - # $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial - # root@container:/# /usr/src/git/ci/run-linux32-build.sh + script: ci/run-linux32-docker.sh - env: Static Analysis os: linux compiler: @@ -86,9 +71,8 @@ matrix: packages: - coccinelle before_install: - script: - # "before_script" that builds Git is inherited from base job - - make coccicheck + # "before_script" that builds Git is inherited from base job + script: ci/run-static-analysis.sh after_failure: - env: Documentation os: linux @@ -99,70 +83,14 @@ matrix: - asciidoc - xmlto before_install: - before_script: gem install asciidoctor + before_script: script: ci/test-documentation.sh after_failure: -before_install: - - > - case "${TRAVIS_OS_NAME:-linux}" in - linux) - export GIT_TEST_HTTPD=YesPlease - - mkdir --parents custom/p4 - pushd custom/p4 - wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4d - wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4 - chmod u+x p4d - chmod u+x p4 - export PATH="$(pwd):$PATH" - popd - mkdir --parents custom/git-lfs - pushd custom/git-lfs - wget --quiet https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz - tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . - export PATH="$(pwd):$PATH" - popd - ;; - osx) - brew update --quiet - # Uncomment this if you want to run perf tests: - # brew install gnu-time - brew install git-lfs gettext - brew link --force gettext - brew install caskroom/cask/perforce - ;; - esac; - echo "$(tput setaf 6)Perforce Server Version$(tput sgr0)"; - p4d -V | grep Rev.; - echo "$(tput setaf 6)Perforce Client Version$(tput sgr0)"; - p4 -V | grep Rev.; - echo "$(tput setaf 6)Git-LFS Version$(tput sgr0)"; - git-lfs version; - -before_script: make --jobs=2 - -script: - - > - mkdir -p $HOME/travis-cache; - ln -s $HOME/travis-cache/.prove t/.prove; - make --quiet test; - -after_failure: - - > - : '<-- Click here to see detailed test output! '; - for TEST_EXIT in t/test-results/*.exit; - do - if [ "$(cat "$TEST_EXIT")" != "0" ]; - then - TEST_OUT="${TEST_EXIT%exit}out"; - echo "------------------------------------------------------------------------"; - echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)"; - echo "------------------------------------------------------------------------"; - cat "${TEST_OUT}"; - fi; - done; +before_install: ci/install-dependencies.sh +before_script: ci/run-build.sh +script: ci/run-tests.sh +after_failure: ci/print-test-failures.sh notifications: email: false diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh new file mode 100755 index 0000000000..067e6eb702 --- /dev/null +++ b/ci/install-dependencies.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Install dependencies required to build and test Git on Linux and macOS +# + +. ${0%/*}/lib-travisci.sh + +case "${TRAVIS_OS_NAME:-linux}" in + linux) + export GIT_TEST_HTTPD=YesPlease + + mkdir --parents custom/p4 + pushd custom/p4 + wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4d + wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4 + chmod u+x p4d + chmod u+x p4 + export PATH="$(pwd):$PATH" + popd + mkdir --parents custom/git-lfs + pushd custom/git-lfs + wget --quiet https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz + tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . + export PATH="$(pwd):$PATH" + popd + ;; + osx) + brew update --quiet + # Uncomment this if you want to run perf tests: + # brew install gnu-time + brew install git-lfs gettext + brew link --force gettext + brew install caskroom/cask/perforce + ;; +esac + +echo "$(tput setaf 6)Perforce Server Version$(tput sgr0)" +p4d -V | grep Rev. +echo "$(tput setaf 6)Perforce Client Version$(tput sgr0)" +p4 -V | grep Rev. +echo "$(tput setaf 6)Git-LFS Version$(tput sgr0)" +git-lfs version diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh new file mode 100755 index 0000000000..44d6ba2dd2 --- /dev/null +++ b/ci/lib-travisci.sh @@ -0,0 +1,5 @@ +# Library of functions shared by all CI scripts + +# Set 'exit on error' for all CI scripts to let the caller know that +# something went wrong +set -e diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh new file mode 100755 index 0000000000..8583e61839 --- /dev/null +++ b/ci/print-test-failures.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# +# Print output of failing tests +# + +. ${0%/*}/lib-travisci.sh + +for TEST_EXIT in t/test-results/*.exit + do + if [ "$(cat "$TEST_EXIT")" != "0" ] + then + TEST_OUT="${TEST_EXIT%exit}out" + echo "------------------------------------------------------------------------" + echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)" + echo "------------------------------------------------------------------------" + cat "${TEST_OUT}" + fi +done diff --git a/ci/run-build.sh b/ci/run-build.sh new file mode 100755 index 0000000000..4f940d1032 --- /dev/null +++ b/ci/run-build.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# +# Build Git +# + +. ${0%/*}/lib-travisci.sh + +make --jobs=2 diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh new file mode 100755 index 0000000000..0edf63acfa --- /dev/null +++ b/ci/run-linux32-docker.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# +# Download and run Docker image to build and test 32-bit Git +# + +. ${0%/*}/lib-travisci.sh + +docker pull daald/ubuntu32:xenial + +# Use the following command to debug the docker build locally: +# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial +# root@container:/# /usr/src/git/ci/run-linux32-build.sh + +docker run \ + --interactive \ + --env DEVELOPER \ + --env DEFAULT_TEST_TARGET \ + --env GIT_PROVE_OPTS \ + --env GIT_TEST_OPTS \ + --env GIT_TEST_CLONE_2GB \ + --volume "${PWD}:/usr/src/git" \ + daald/ubuntu32:xenial \ + /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) diff --git a/ci/run-static-analysis.sh b/ci/run-static-analysis.sh new file mode 100755 index 0000000000..68dd0f080e --- /dev/null +++ b/ci/run-static-analysis.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# +# Perform various static code analysis checks +# + +. ${0%/*}/lib-travisci.sh + +make coccicheck diff --git a/ci/run-tests.sh b/ci/run-tests.sh new file mode 100755 index 0000000000..f0c743de94 --- /dev/null +++ b/ci/run-tests.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Test Git +# + +. ${0%/*}/lib-travisci.sh + +mkdir -p $HOME/travis-cache +ln -s $HOME/travis-cache/.prove t/.prove +make --quiet test diff --git a/ci/run-windows-build.sh b/ci/run-windows-build.sh index 2d98f6b2f9..8757b3a97c 100755 --- a/ci/run-windows-build.sh +++ b/ci/run-windows-build.sh @@ -6,6 +6,8 @@ # supported) and a commit hash. # +. ${0%/*}/lib-travisci.sh + test $# -ne 2 && echo "Unexpected number of parameters" && exit 1 test -z "$GFW_CI_TOKEN" && echo "GFW_CI_TOKEN not defined" && exit diff --git a/ci/test-documentation.sh b/ci/test-documentation.sh index 6214e6acb4..7a0a848e83 100755 --- a/ci/test-documentation.sh +++ b/ci/test-documentation.sh @@ -3,7 +3,9 @@ # Perform sanity checks on documentation and build it. # -set -e +. ${0%/*}/lib-travisci.sh + +gem install asciidoctor make check-builtins make check-docs From 09f5e9746ce8be640683f9f5f1ffb54a34d4dc90 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Sun, 10 Sep 2017 16:44:29 +0200 Subject: [PATCH 2/4] travis-ci: skip a branch build if equal tag is present If we push a branch and a tag pointing to the HEAD of this branch, then Travis CI would run the build twice. This wastes resources and slows the testing. Add a function to detect this situation and skip the build the branch if appropriate. Invoke this function on every build. Helped-by: Junio C Hamano Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- ci/lib-travisci.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index 44d6ba2dd2..9c4ae9bdd0 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -1,5 +1,28 @@ # Library of functions shared by all CI scripts +skip_branch_tip_with_tag () { + # Sometimes, a branch is pushed at the same time the tag that points + # at the same commit as the tip of the branch is pushed, and building + # both at the same time is a waste. + # + # Travis gives a tagname e.g. v2.14.0 in $TRAVIS_BRANCH when + # the build is triggered by a push to a tag. Let's see if + # $TRAVIS_BRANCH is exactly at a tag, and if so, if it is + # different from $TRAVIS_BRANCH. That way, we can tell if + # we are building the tip of a branch that is tagged and + # we can skip the build because we won't be skipping a build + # of a tag. + + if TAG=$(git describe --exact-match "$TRAVIS_BRANCH" 2>/dev/null) && + $TAG != $TRAVIS_BRANCH + then + echo "Tip of $TRAVIS_BRANCH is exactly at $TAG" + exit 0 + fi +} + # Set 'exit on error' for all CI scripts to let the caller know that # something went wrong set -e + +skip_branch_tip_with_tag From f67242c10ddf24a14b5085b1effcbe818aa007e6 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 11 Sep 2017 10:18:29 +0900 Subject: [PATCH 3/4] travis: dedent a few scripts that are indented overly deeply Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 53 ++++++++++++++++++++------------------ ci/print-test-failures.sh | 18 ++++++------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 067e6eb702..a29246af35 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -5,33 +5,36 @@ . ${0%/*}/lib-travisci.sh -case "${TRAVIS_OS_NAME:-linux}" in - linux) - export GIT_TEST_HTTPD=YesPlease +P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION +LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION - mkdir --parents custom/p4 - pushd custom/p4 - wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4d - wget --quiet http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION/bin.linux26x86_64/p4 - chmod u+x p4d - chmod u+x p4 - export PATH="$(pwd):$PATH" - popd - mkdir --parents custom/git-lfs - pushd custom/git-lfs - wget --quiet https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz - tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . - export PATH="$(pwd):$PATH" - popd +case "${TRAVIS_OS_NAME:-linux}" in +linux) + export GIT_TEST_HTTPD=YesPlease + + mkdir --parents custom/p4 + pushd custom/p4 + wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" + wget --quiet "$P4WHENCE/bin.linux26x86_64/p4" + chmod u+x p4d + chmod u+x p4 + export PATH="$(pwd):$PATH" + popd + mkdir --parents custom/git-lfs + pushd custom/git-lfs + wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . + export PATH="$(pwd):$PATH" + popd ;; - osx) - brew update --quiet - # Uncomment this if you want to run perf tests: - # brew install gnu-time - brew install git-lfs gettext - brew link --force gettext - brew install caskroom/cask/perforce +osx) + brew update --quiet + # Uncomment this if you want to run perf tests: + # brew install gnu-time + brew install git-lfs gettext + brew link --force gettext + brew install caskroom/cask/perforce ;; esac diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index 8583e61839..8c8973cbf3 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -6,13 +6,13 @@ . ${0%/*}/lib-travisci.sh for TEST_EXIT in t/test-results/*.exit - do - if [ "$(cat "$TEST_EXIT")" != "0" ] - then - TEST_OUT="${TEST_EXIT%exit}out" - echo "------------------------------------------------------------------------" - echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)" - echo "------------------------------------------------------------------------" - cat "${TEST_OUT}" - fi +do + if [ "$(cat "$TEST_EXIT")" != "0" ] + then + TEST_OUT="${TEST_EXIT%exit}out" + echo "------------------------------------------------------------------------" + echo "$(tput setaf 1)${TEST_OUT}...$(tput sgr0)" + echo "------------------------------------------------------------------------" + cat "${TEST_OUT}" + fi done From 8376eb4a8f4f5e048d6be3b54f9e13577f53eafa Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Thu, 21 Sep 2017 22:48:30 +0200 Subject: [PATCH 4/4] travis-ci: fix "skip_branch_tip_with_tag()" string comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 09f5e97 ("travis-ci: skip a branch build if equal tag is present", 2017-09-17) introduced the "skip_branch_tip_with_tag" function with a broken string comparison. Fix it! Reported-by: SZEDER Gábor Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- ci/lib-travisci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/lib-travisci.sh b/ci/lib-travisci.sh index 9c4ae9bdd0..b3ed0a0dda 100755 --- a/ci/lib-travisci.sh +++ b/ci/lib-travisci.sh @@ -14,7 +14,7 @@ skip_branch_tip_with_tag () { # of a tag. if TAG=$(git describe --exact-match "$TRAVIS_BRANCH" 2>/dev/null) && - $TAG != $TRAVIS_BRANCH + test "$TAG" != "$TRAVIS_BRANCH" then echo "Tip of $TRAVIS_BRANCH is exactly at $TAG" exit 0