From d2fae19e0ffa682e8781dbb649519bc118de1c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SZEDER=20G=C3=A1bor?= Date: Thu, 2 Apr 2020 20:04:00 +0700 Subject: [PATCH 1/6] ci: make MAKEFLAGS available inside the Docker container in the Linux32 job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once upon a time we ran 'make --jobs=2 ...' to build Git, its documentation, or to apply Coccinelle semantic patches. Then commit eaa62291ff (ci: inherit --jobs via MAKEFLAGS in run-build-and-tests, 2019-01-27) came along, and started using the MAKEFLAGS environment variable to centralize setting the number of parallel jobs in 'ci/libs.sh'. Alas, it forgot to update 'ci/run-linux32-docker.sh' to make MAKEFLAGS available inside the Docker container running the 32 bit Linux job, and, consequently, since then that job builds Git sequentially, and it ignores any Makefile knobs that we might set in MAKEFLAGS (though we don't set any for the 32 bit Linux job at the moment). So update the 'docker run' invocation in 'ci/run-linux32-docker.sh' to make MAKEFLAGS available inside the Docker container as well. Set CC=gcc for the 32 bit Linux job, because that's the compiler installed in the 32 bit Linux Docker image that we use (Travis CI nowadays sets CC=clang by default, but clang is not installed in this image). Signed-off-by: SZEDER Gábor Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- ci/lib.sh | 3 +++ ci/run-linux32-docker.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index a90d0dc0fd..8048335281 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -192,6 +192,9 @@ osx-clang|osx-gcc) GIT_TEST_GETTEXT_POISON) export GIT_TEST_GETTEXT_POISON=true ;; +Linux32) + CC=gcc + ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index 751acfcf8a..ebb18fa747 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -20,6 +20,7 @@ docker run \ --env GIT_PROVE_OPTS \ --env GIT_TEST_OPTS \ --env GIT_TEST_CLONE_2GB \ + --env MAKEFLAGS \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ From ffce2ebdd9ad14d1ba315592157c03ce7ebd6212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 4 Apr 2020 08:08:46 +0700 Subject: [PATCH 2/6] ci/lib-docker: preserve required environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're using "su -m" to preserve environment variables in the shell run by "su". But, that options will be ignored while "-l" (aka "--login") is specified in util-linux and busybox's su. In a later patch this script will be reused for checking Git for Linux with musl libc on Alpine Linux, Alpine Linux uses "su" from busybox. Since we don't have interest in all environment variables, pass only those necessary variables to the inner script. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- ci/run-linux32-build.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index e3a193adbc..7f985615c2 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -51,10 +51,17 @@ else fi # Build and test -linux32 --32bit i386 su -m -l $CI_USER -c ' +linux32 --32bit i386 su -m -l $CI_USER -c " set -ex + export DEVELOPER='$DEVELOPER' + export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' + export GIT_PROVE_OPTS='$GIT_PROVE_OPTS' + export GIT_TEST_OPTS='$GIT_TEST_OPTS' + export GIT_TEST_CLONE_2GB='$GIT_TEST_CLONE_2GB' + export MAKEFLAGS='$MAKEFLAGS' + export cache_dir='$cache_dir' cd /usr/src/git - test -n "$cache_dir" && ln -s "$cache_dir/.prove" t/.prove + test -n '$cache_dir' && ln -s '$cache_dir/.prove' t/.prove make make test -' +" From 2bd1e2d273b69516a3c161671a39263589ccc9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 4 Apr 2020 08:08:47 +0700 Subject: [PATCH 3/6] ci/linux32: parameterise command to switch arch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, the remaining of this command will be re-used for the CI job for linux with musl libc. Allow customisation of the emulator, now. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- ci/run-linux32-build.sh | 13 +++++++++++-- ci/run-linux32-docker.sh | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index 7f985615c2..44bb332f64 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -14,8 +14,17 @@ then exit 1 fi +case "$jobname" in +Linux32) + switch_cmd="linux32 --32bit i386" + ;; +*) + exit 1 + ;; +esac + # Update packages to the latest available versions -linux32 --32bit i386 sh -c ' +command $switch_cmd sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext python >/dev/null @@ -51,7 +60,7 @@ else fi # Build and test -linux32 --32bit i386 su -m -l $CI_USER -c " +command $switch_cmd su -m -l $CI_USER -c " set -ex export DEVELOPER='$DEVELOPER' export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index ebb18fa747..54186b6aa7 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -9,6 +9,7 @@ docker pull daald/ubuntu32:xenial # Use the following command to debug the docker build locally: # $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial +# root@container:/# export jobname= # root@container:/# /usr/src/git/ci/run-linux32-build.sh container_cache_dir=/tmp/travis-cache @@ -21,6 +22,7 @@ docker run \ --env GIT_TEST_OPTS \ --env GIT_TEST_CLONE_2GB \ --env MAKEFLAGS \ + --env jobname \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ From 5a33f541ddb52e3f45db87db7c3dd40945b9a564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 4 Apr 2020 08:08:48 +0700 Subject: [PATCH 4/6] ci: refactor docker runner script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will support alpine check in docker later in this series. While we're at it, tell people to run as root in podman, if podman is used as drop-in replacement for docker, because podman will map host-user to container's root, therefore, mapping their permission. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- .travis.yml | 2 +- azure-pipelines.yml | 4 ++-- ...n-linux32-build.sh => run-docker-build.sh} | 6 ++--- ci/{run-linux32-docker.sh => run-docker.sh} | 22 ++++++++++++++----- 4 files changed, 22 insertions(+), 12 deletions(-) rename ci/{run-linux32-build.sh => run-docker-build.sh} (93%) rename ci/{run-linux32-docker.sh => run-docker.sh} (53%) diff --git a/.travis.yml b/.travis.yml index fc5730b085..069aeeff3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ matrix: services: - docker before_install: - script: ci/run-linux32-docker.sh + script: ci/run-docker.sh - env: jobname=StaticAnalysis os: linux compiler: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 675c3a43c9..11413f66f8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -489,14 +489,14 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 res=0 - sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS="$MAKEFLAGS" bash -lxc ci/run-linux32-docker.sh || res=1 + sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS="$MAKEFLAGS" jobname=Linux32 bash -lxc ci/run-docker.sh || res=1 sudo chmod a+r t/out/TEST-*.xml test ! -d t/failed-test-artifacts || sudo chmod a+r t/failed-test-artifacts test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || res=1 exit $res - displayName: 'ci/run-linux32-docker.sh' + displayName: 'jobname=Linux32 ci/run-docker.sh' env: GITFILESHAREPWD: $(gitfileshare.pwd) - task: PublishTestResults@2 diff --git a/ci/run-linux32-build.sh b/ci/run-docker-build.sh similarity index 93% rename from ci/run-linux32-build.sh rename to ci/run-docker-build.sh index 44bb332f64..a05b48c559 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-docker-build.sh @@ -1,16 +1,16 @@ #!/bin/sh # -# Build and test Git in a 32-bit environment +# Build and test Git inside container # # Usage: -# run-linux32-build.sh +# run-docker-build.sh # set -ex if test $# -ne 1 || test -z "$1" then - echo >&2 "usage: run-linux32-build.sh " + echo >&2 "usage: run-docker-build.sh " exit 1 fi diff --git a/ci/run-linux32-docker.sh b/ci/run-docker.sh similarity index 53% rename from ci/run-linux32-docker.sh rename to ci/run-docker.sh index 54186b6aa7..3881f99b53 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-docker.sh @@ -1,16 +1,26 @@ #!/bin/sh # -# Download and run Docker image to build and test 32-bit Git +# Download and run Docker image to build and test Git # . ${0%/*}/lib.sh -docker pull daald/ubuntu32:xenial +case "$jobname" in +Linux32) + CI_CONTAINER="daald/ubuntu32:xenial" + ;; +*) + exit 1 + ;; +esac + +docker pull "$CI_CONTAINER" # Use the following command to debug the docker build locally: -# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial +# must be 0 if podman is used as drop-in replacement for docker +# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/sh "$CI_CONTAINER" # root@container:/# export jobname= -# root@container:/# /usr/src/git/ci/run-linux32-build.sh +# root@container:/# /usr/src/git/ci/run-docker-build.sh container_cache_dir=/tmp/travis-cache @@ -26,8 +36,8 @@ docker run \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ - daald/ubuntu32:xenial \ - /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) + "$CI_CONTAINER" \ + /usr/src/git/ci/run-docker-build.sh $(id -u $USER) check_unignored_build_artifacts From c3bc449eb1b8b458e812c07eb8d0cb643f3d1b36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 4 Apr 2020 08:08:49 +0700 Subject: [PATCH 5/6] ci/linux32: libify install-dependencies step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, we will add new Travis Job for linux-musl. Most of other code in this file could be reuse for that job. Move the code to install dependencies to a common script. Should we add new CI system that can run directly in container, we can reuse this script for installation step. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- ci/install-docker-dependencies.sh | 14 ++++++++++++++ ci/run-docker-build.sh | 7 +------ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100755 ci/install-docker-dependencies.sh diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh new file mode 100755 index 0000000000..a104c61d29 --- /dev/null +++ b/ci/install-docker-dependencies.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Install dependencies required to build and test Git inside container +# + +case "$jobname" in +Linux32) + linux32 --32bit i386 sh -c ' + apt update >/dev/null && + apt install -y build-essential libcurl4-openssl-dev \ + libssl-dev libexpat-dev gettext python >/dev/null + ' + ;; +esac diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index a05b48c559..4a153492ba 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -23,12 +23,7 @@ Linux32) ;; esac -# Update packages to the latest available versions -command $switch_cmd sh -c ' - apt update >/dev/null && - apt install -y build-essential libcurl4-openssl-dev libssl-dev \ - libexpat-dev gettext python >/dev/null -' +"${0%/*}/install-docker-dependencies.sh" # If this script runs inside a docker container, then all commands are # usually executed as root. Consequently, the host user might not be From e0f8690dee7ec9373657eb2ecef9c8d88d0ece08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Sat, 4 Apr 2020 08:08:50 +0700 Subject: [PATCH 6/6] travis: build and test on Linux with musl libc and busybox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- .travis.yml | 8 ++++++++ ci/install-docker-dependencies.sh | 4 ++++ ci/lib.sh | 5 +++++ ci/run-docker-build.sh | 4 ++++ ci/run-docker.sh | 3 +++ 5 files changed, 24 insertions(+) diff --git a/.travis.yml b/.travis.yml index 069aeeff3c..0cfc3c3428 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,14 @@ matrix: - docker before_install: script: ci/run-docker.sh + - env: jobname=linux-musl + os: linux + compiler: + addons: + services: + - docker + before_install: + script: ci/run-docker.sh - env: jobname=StaticAnalysis os: linux compiler: diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index a104c61d29..26a6689766 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -11,4 +11,8 @@ Linux32) libssl-dev libexpat-dev gettext python >/dev/null ' ;; +linux-musl) + apk add --update build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + ;; esac diff --git a/ci/lib.sh b/ci/lib.sh index 8048335281..e9a5c51425 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -195,6 +195,11 @@ GIT_TEST_GETTEXT_POISON) Linux32) CC=gcc ;; +linux-musl) + CC=gcc + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" + MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" + ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index 4a153492ba..8d47a5fda3 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -18,6 +18,10 @@ case "$jobname" in Linux32) switch_cmd="linux32 --32bit i386" ;; +linux-musl) + switch_cmd= + useradd () { adduser -D "$@"; } + ;; *) exit 1 ;; diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 3881f99b53..37fa372052 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -9,6 +9,9 @@ case "$jobname" in Linux32) CI_CONTAINER="daald/ubuntu32:xenial" ;; +linux-musl) + CI_CONTAINER=alpine + ;; *) exit 1 ;;