Merge branch 'docker-ci-image/use-own-image-for-pipeline-jobs' into 'master'

Speed up CI pipeline

See merge request !536

.gitlab-ci.yml

-image: golang:1.7
-
 stages:
+- prepare
 - prebuild
 - test
 - build
 - package
 - release
 
+variables:
+  CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-ci-multi-runner:ci-1.7
+
+image: $CI_IMAGE
+
 before_script:
 - source ci/prepare
 
 .docker: &docker
-  before_script:
-  - source ci/prepare
-  - wget -q https://get.docker.com/builds/Linux/x86_64/docker-1.13.1.tgz -O /tmp/docker.tar.gz
-  - tar -xzf /tmp/docker.tar.gz -C /tmp/
-  - cp /tmp/docker/docker* /usr/bin
-  - chmod +x /usr/bin/docker*
-  services: ["docker:dind"]
+  services:
+  - docker:dind
   variables:
+    CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-ci-multi-runner:ci-1.7
     DOCKER_DRIVER: overlay
     DOCKER_HOST: tcp://docker:2375
   tags:
   - docker
 
+#
+# Prepare stage
+#
+
+prepare ci image:
+  <<: *docker
+  stage: prepare
+  image: docker:git
+  before_script: []
+  script:
+  - source ./ci/build_ci_image
+  only:
+  - /\Adocker-ci-image\/.*/@gitlab-org/gitlab-ci-multi-runner
+
 #
 # Prebuild stage
 #
@@ -32,8 +46,6 @@ bindata:
   <<: *docker
   stage: prebuild
   script:
-  - apt-get install -yqqq xz-utils
-  - make deps
   - make docker
   artifacts:
     paths:
@@ -62,7 +74,6 @@ static code analysis:
   stage: test
   script:
   - source ci/touch_make_dependencies
-  - make deps
   - make static_code_analysis
   tags:
   - docker
@@ -71,7 +82,6 @@ unit tests:
   stage: test
   script:
   - source ci/touch_make_dependencies
-  - make deps
   - make test
   tags:
   - docker
@@ -109,12 +119,12 @@ integration windows:
 # Build stage
 #
 
-binaries:
+.binaries: &binaries
   stage: build
   script:
   - source ci/touch_make_dependencies
-  - make deps
-  - make build
+  - export platforms=$(echo $CI_JOB_NAME | sed 's|binaries ||')
+  - make build BUILD_PLATFORMS="-osarch='$platforms'"
   artifacts:
     paths:
     - out/binaries/
@@ -122,16 +132,24 @@ binaries:
   tags:
   - docker
 
+binaries darwin/386 darwin/amd64: *binaries
+binaries freebsd/386 freebsd/amd64 freebsd/arm: *binaries
+binaries linux/386 linux/amd64 linux/arm: *binaries
+binaries windows/386 windows/amd64: *binaries
+
 #
 # Package stage
 #
 
-packages:
+.packages: &packages
   stage: package
   script:
+  - mkdir -p out/docker
+  - touch out/docker/prebuilt-x86_64.tar.xz
+  - touch out/docker/prebuilt-arm.tar.xz
+  - touch executors/docker/bindata.go
   - source ci/touch_make_dependencies
-  - apt-get install -yqqq ruby ruby-dev python-pip dpkg-sig createrepo rpm
-  - make package
+  - make ${CI_JOB_NAME}
   artifacts:
     paths:
     - out/deb/
@@ -140,6 +158,9 @@ packages:
   tags:
   - docker
 
+package-deb: *packages
+package-rpm: *packages
+
 #
 # Release stage
 #
@@ -147,11 +168,8 @@ packages:
 .release_docker_images: &release_docker_images
   <<: *docker
   stage: release
-  image: docker:git
-  before_script:
-  - source ci/touch_make_dependencies
-  - apk -U add make bash
   script:
+  - source ci/touch_make_dependencies
   - make release_docker_images
   tags:
   - release
@@ -159,8 +177,6 @@ packages:
 .release: &release
   stage: release
   before_script:
-  - apt-get update -yqqq >/dev/null
-  - apt-get install -yqqq ruby ruby-dev >/dev/null
   - source ci/touch_make_dependencies
   tags:
   - release

Makefile

@@ -243,14 +243,14 @@ package-prepare:
 	chmod 755 packaging/root/usr/share/gitlab-runner/
 	chmod 755 packaging/root/usr/share/gitlab-runner/*
 
-package-deb:
+package-deb: package-deps package-prepare
 	# Building Debian compatible packages...
 	make package-deb-fpm ARCH=amd64 PACKAGE_ARCH=amd64
 	make package-deb-fpm ARCH=386 PACKAGE_ARCH=i386
 	make package-deb-fpm ARCH=arm PACKAGE_ARCH=armel
 	make package-deb-fpm ARCH=arm PACKAGE_ARCH=armhf
 
-package-rpm:
+package-rpm: package-deps package-prepare
 	# Building RedHat compatible packages...
 	make package-rpm-fpm ARCH=amd64 PACKAGE_ARCH=amd64
 	make package-rpm-fpm ARCH=386 PACKAGE_ARCH=i686
@@ -310,7 +310,7 @@ packagecloud: packagecloud-deps packagecloud-deb packagecloud-rpm
 
 packagecloud-deps:
 	# Installing packagecloud dependencies...
-	gem install package_cloud
+	gem install package_cloud --no-ri --no-rdoc
 
 packagecloud-deb:
 	# Sending Debian compatible packages...

ci/build_ci_image

+#!/bin/bash
+
+set -eo pipefail
+
+registry_credentials_available() {
+    echo "Checking registry credentials"
+    [[ -n "${CI_REGISTRY_USER}" ]] && [[ -n "${CI_REGISTRY_PASSWORD}" ]] && return 0
+    return 1
+}
+
+login() {
+    echo "Logging into registry"
+    registry_credentials_available || return 0
+    docker login --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+}
+
+logout() {
+    echo "Logging out of registry"
+    docker logout ${CI_REGISTRY}
+}
+
+pull () {
+    echo "Pulling base image"
+    registry_credentials_available || return 0
+    docker pull ${CI_IMAGE} || echo "${CI_IMAGE} image is not available. Will not use cache."
+}
+
+push () {
+    echo "Pushing image"
+    registry_credentials_available || return 0
+    docker push ${CI_IMAGE}
+}
+
+
+build() {
+    echo "Building image"
+    docker build \
+           --cache-from ${CI_IMAGE} \
+           -t ${CI_IMAGE} \
+           -f ./dockerfiles/ci/Dockerfile \
+           .
+}
+
+login
+pull
+build
+push
+logout
\ No newline at end of file

ci/prepare

@@ -3,13 +3,3 @@ GODIR=$GOPATH/src/gitlab.com/gitlab-org/gitlab-ci-multi-runner
 mkdir -p "$(dirname "$GODIR")"
 ln -sfv "$(pwd -P)" "$GODIR"
 cd "$GODIR"
-
-apt-get update -yqqq >/dev/null
-apt-get install -yqqq locales >/dev/null
-
-# Set default locale for the environment
-echo "en_US UTF-8" > /etc/locale.gen
-locale-gen en_US.UTF-8
-export LANG=en_US.UTF-8
-export LANGUAGE=en_US:en
-export LC_ALL=en_US.UTF-8

dockerfiles/ci/Dockerfile

+FROM golang:1.7
+
+# Install required dependencies
+RUN apt-get update -yq
+RUN apt-get install -yq locales make xz-utils \
+                        ruby ruby-dev python-pip \
+                        dpkg-sig createrepo rpm
+
+# Set default locale for the environment
+RUN echo "en_US UTF-8" > /etc/locale.gen; \
+    locale-gen en_US.UTF-8
+
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+# Install docker client
+RUN wget -q https://get.docker.com/builds/Linux/x86_64/docker-1.13.1.tgz -O /tmp/docker.tar.gz; \
+    tar -xzf /tmp/docker.tar.gz -C /tmp/; \
+    cp /tmp/docker/docker* /usr/bin; \
+    chmod +x /usr/bin/docker*; \
+    rm -rf /tmp/*
+
+COPY Makefile /tmp/
+RUN cd /tmp; \
+    make deps package-deps packagecloud-deps
\ No newline at end of file