Add ServicesTmpfs options to Docker runner configuration.

This will mount the specified path, (along with the specified mount options) using the native Docker tmpfs support in all the container services.

common/config.go

@@ -49,32 +49,33 @@ func (p DockerPullPolicy) Get() (DockerPullPolicy, error) {
 
 type DockerConfig struct {
 	docker_helpers.DockerCredentials
-	Hostname               string           `toml:"hostname,omitempty" json:"hostname" long:"hostname" env:"DOCKER_HOSTNAME" description:"Custom container hostname"`
-	Image                  string           `toml:"image" json:"image" long:"image" env:"DOCKER_IMAGE" description:"Docker image to be used"`
-	CPUSetCPUs             string           `toml:"cpuset_cpus,omitempty" json:"cpuset_cpus" long:"cpuset-cpus" env:"DOCKER_CPUSET_CPUS" description:"String value containing the cgroups CpusetCpus to use"`
-	CPUS                   string           `toml:"cpus,omitempty" json:"cpus" long:"cpus" env:"DOCKER_CPUS" description:"Number of CPUs"`
-	DNS                    []string         `toml:"dns,omitempty" json:"dns" long:"dns" env:"DOCKER_DNS" description:"A list of DNS servers for the container to use"`
-	DNSSearch              []string         `toml:"dns_search,omitempty" json:"dns_search" long:"dns-search" env:"DOCKER_DNS_SEARCH" description:"A list of DNS search domains"`
-	Privileged             bool             `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"DOCKER_PRIVILEGED" description:"Give extended privileges to container"`
-	UsernsMode             string           `toml:"userns_mode,omitempty" json:"userns_mode" long:"userns" env:"DOCKER_USERNS_MODE" description:"User namespace to use"`
-	CapAdd                 []string         `toml:"cap_add" json:"cap_add" long:"cap-add" env:"DOCKER_CAP_ADD" description:"Add Linux capabilities"`
-	CapDrop                []string         `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"DOCKER_CAP_DROP" description:"Drop Linux capabilities"`
-	SecurityOpt            []string         `toml:"security_opt" json:"security_opt" long:"security-opt" env:"DOCKER_SECURITY_OPT" description:"Security Options"`
-	Devices                []string         `toml:"devices" json:"devices" long:"devices" env:"DOCKER_DEVICES" description:"Add a host device to the container"`
-	DisableCache           bool             `toml:"disable_cache,omitzero" json:"disable_cache" long:"disable-cache" env:"DOCKER_DISABLE_CACHE" description:"Disable all container caching"`
-	Volumes                []string         `toml:"volumes,omitempty" json:"volumes" long:"volumes" env:"DOCKER_VOLUMES" description:"Bind mount a volumes"`
-	VolumeDriver           string           `toml:"volume_driver,omitempty" json:"volume_driver" long:"volume-driver" env:"DOCKER_VOLUME_DRIVER" description:"Volume driver to be used"`
-	CacheDir               string           `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"DOCKER_CACHE_DIR" description:"Directory where to store caches"`
-	ExtraHosts             []string         `toml:"extra_hosts,omitempty" json:"extra_hosts" long:"extra-hosts" env:"DOCKER_EXTRA_HOSTS" description:"Add a custom host-to-IP mapping"`
-	VolumesFrom            []string         `toml:"volumes_from,omitempty" json:"volumes_from" long:"volumes-from" env:"DOCKER_VOLUMES_FROM" description:"A list of volumes to inherit from another container"`
-	NetworkMode            string           `toml:"network_mode,omitempty" json:"network_mode" long:"network-mode" env:"DOCKER_NETWORK_MODE" description:"Add container to a custom network"`
-	Links                  []string         `toml:"links,omitempty" json:"links" long:"links" env:"DOCKER_LINKS" description:"Add link to another container"`
-	Services               []string         `toml:"services,omitempty" json:"services" long:"services" env:"DOCKER_SERVICES" description:"Add service that is started with container"`
-	WaitForServicesTimeout int              `toml:"wait_for_services_timeout,omitzero" json:"wait_for_services_timeout" long:"wait-for-services-timeout" env:"DOCKER_WAIT_FOR_SERVICES_TIMEOUT" description:"How long to wait for service startup"`
-	AllowedImages          []string         `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"DOCKER_ALLOWED_IMAGES" description:"Whitelist allowed images"`
-	AllowedServices        []string         `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"DOCKER_ALLOWED_SERVICES" description:"Whitelist allowed services"`
-	PullPolicy             DockerPullPolicy `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"DOCKER_PULL_POLICY" description:"Image pull policy: never, if-not-present, always"`
-	ShmSize                int64            `toml:"shm_size,omitempty" json:"shm_size" long:"shm-size" env:"DOCKER_SHM_SIZE" description:"Shared memory size for docker images (in bytes)"`
+	Hostname               string            `toml:"hostname,omitempty" json:"hostname" long:"hostname" env:"DOCKER_HOSTNAME" description:"Custom container hostname"`
+	Image                  string            `toml:"image" json:"image" long:"image" env:"DOCKER_IMAGE" description:"Docker image to be used"`
+	CPUSetCPUs             string            `toml:"cpuset_cpus,omitempty" json:"cpuset_cpus" long:"cpuset-cpus" env:"DOCKER_CPUSET_CPUS" description:"String value containing the cgroups CpusetCpus to use"`
+	CPUS                   string            `toml:"cpus,omitempty" json:"cpus" long:"cpus" env:"DOCKER_CPUS" description:"Number of CPUs"`
+	DNS                    []string          `toml:"dns,omitempty" json:"dns" long:"dns" env:"DOCKER_DNS" description:"A list of DNS servers for the container to use"`
+	DNSSearch              []string          `toml:"dns_search,omitempty" json:"dns_search" long:"dns-search" env:"DOCKER_DNS_SEARCH" description:"A list of DNS search domains"`
+	Privileged             bool              `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"DOCKER_PRIVILEGED" description:"Give extended privileges to container"`
+	UsernsMode             string            `toml:"userns_mode,omitempty" json:"userns_mode" long:"userns" env:"DOCKER_USERNS_MODE" description:"User namespace to use"`
+	CapAdd                 []string          `toml:"cap_add" json:"cap_add" long:"cap-add" env:"DOCKER_CAP_ADD" description:"Add Linux capabilities"`
+	CapDrop                []string          `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"DOCKER_CAP_DROP" description:"Drop Linux capabilities"`
+	SecurityOpt            []string          `toml:"security_opt" json:"security_opt" long:"security-opt" env:"DOCKER_SECURITY_OPT" description:"Security Options"`
+	Devices                []string          `toml:"devices" json:"devices" long:"devices" env:"DOCKER_DEVICES" description:"Add a host device to the container"`
+	DisableCache           bool              `toml:"disable_cache,omitzero" json:"disable_cache" long:"disable-cache" env:"DOCKER_DISABLE_CACHE" description:"Disable all container caching"`
+	Volumes                []string          `toml:"volumes,omitempty" json:"volumes" long:"volumes" env:"DOCKER_VOLUMES" description:"Bind mount a volumes"`
+	VolumeDriver           string            `toml:"volume_driver,omitempty" json:"volume_driver" long:"volume-driver" env:"DOCKER_VOLUME_DRIVER" description:"Volume driver to be used"`
+	CacheDir               string            `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"DOCKER_CACHE_DIR" description:"Directory where to store caches"`
+	ExtraHosts             []string          `toml:"extra_hosts,omitempty" json:"extra_hosts" long:"extra-hosts" env:"DOCKER_EXTRA_HOSTS" description:"Add a custom host-to-IP mapping"`
+	VolumesFrom            []string          `toml:"volumes_from,omitempty" json:"volumes_from" long:"volumes-from" env:"DOCKER_VOLUMES_FROM" description:"A list of volumes to inherit from another container"`
+	NetworkMode            string            `toml:"network_mode,omitempty" json:"network_mode" long:"network-mode" env:"DOCKER_NETWORK_MODE" description:"Add container to a custom network"`
+	Links                  []string          `toml:"links,omitempty" json:"links" long:"links" env:"DOCKER_LINKS" description:"Add link to another container"`
+	Services               []string          `toml:"services,omitempty" json:"services" long:"services" env:"DOCKER_SERVICES" description:"Add service that is started with container"`
+	WaitForServicesTimeout int               `toml:"wait_for_services_timeout,omitzero" json:"wait_for_services_timeout" long:"wait-for-services-timeout" env:"DOCKER_WAIT_FOR_SERVICES_TIMEOUT" description:"How long to wait for service startup"`
+	AllowedImages          []string          `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"DOCKER_ALLOWED_IMAGES" description:"Whitelist allowed images"`
+	AllowedServices        []string          `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"DOCKER_ALLOWED_SERVICES" description:"Whitelist allowed services"`
+	PullPolicy             DockerPullPolicy  `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"DOCKER_PULL_POLICY" description:"Image pull policy: never, if-not-present, always"`
+	ShmSize                int64             `toml:"shm_size,omitempty" json:"shm_size" long:"shm-size" env:"DOCKER_SHM_SIZE" description:"Shared memory size for docker images (in bytes)"`
+	ServicesTmpfs          map[string]string `toml:"services_tmpfs,omitempty" json:"services_tmpfs" long:"services-tmpfs" env:"DOCKER_SERVICES_TMPFS" description:"A toml table/json object with the format key=values. When set this will mount the specified path in the key as a tmpfs volume in all the service containers, using the options specified as key. For the supported options, see the documentation for the unix 'mount' command"`
 }
 
 type DockerMachine struct {

docs/executors/docker.md

@@ -196,6 +196,19 @@ distinguish which variable should go where.
 >
 Secure variables are only passed to the build container.
 
+## Mounting a directory in RAM
+
+You can mount a path inside all the services containers in RAM using tmpfs. This can speed up the time required to test if there is a lot of I/O related work, such as with databases.
+If you use the `services_tmpfs` option in the runner configuration, you c  an specify multiple paths, each with its own options. See the [docker reference](https://docs.docker.com/engine/reference/commandline/run/#mount-tmpfs-tmpfs) for details.
+This is an example `config.toml` to mount the data directory for the official Mysql container in RAM.
+
+```
+[runners.docker]
+  [runners.docker.services_tmpfs]
+      "/var/lib/mysql" = "rw,noexec"
+
+```
+
 ## Build directory in service
 
 Since version 1.5 GitLab Runner mounts a `/builds` directory to all shared services.

executors/docker/executor_docker.go

@@ -591,6 +591,7 @@ func (s *executor) createService(service, version, image string, serviceDefiniti
 		Binds:         s.binds,
 		ShmSize:       s.Config.Docker.ShmSize,
 		VolumesFrom:   s.volumesFrom,
+		Tmpfs:         s.Config.Docker.ServicesTmpfs,
 		LogConfig: container.LogConfig{
 			Type: "json-file",
 		},

executors/docker/executor_docker_test.go

@@ -835,14 +835,13 @@ func (c *dockerConfigurationTestFakeDockerClient) ContainerCreate(ctx context.Co
 	return container.ContainerCreateCreatedBody{ID: "abc"}, nil
 }
 
-func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) {
+func prepareTestDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) (*dockerConfigurationTestFakeDockerClient, *executor) {
 	c := &dockerConfigurationTestFakeDockerClient{
 		cce: cce,
 		t:   t,
 	}
-	defer c.AssertExpectations(t)
 
-	e := executor{}
+	e := &executor{}
 	e.client = c
 	e.Config.Docker = dockerConfig
 	e.Build = &common.Build{
@@ -861,6 +860,14 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc
 		Return([]types.NetworkResource{}, nil).Once()
 	c.On("ContainerRemove", mock.Anything, mock.Anything, mock.Anything).
 		Return(nil).Once()
+
+	return c, e
+}
+
+func testDockerConfigurationWithJobContainer(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) {
+	c, e := prepareTestDockerConfiguration(t, dockerConfig, cce)
+	defer c.AssertExpectations(t)
+
 	c.On("ContainerInspect", mock.Anything, "abc").
 		Return(types.ContainerJSON{}, nil).Once()
 
@@ -868,6 +875,17 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc
 	assert.NoError(t, err, "Should create container without errors")
 }
 
+func testDockerConfigurationWithServiceContainer(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) {
+	c, e := prepareTestDockerConfiguration(t, dockerConfig, cce)
+	defer c.AssertExpectations(t)
+
+	c.On("ContainerStart", mock.Anything, "abc", mock.Anything).
+		Return(nil).Once()
+
+	_, err := e.createService("build", "latest", "alpine", common.Image{Command: []string{"/bin/sh"}})
+	assert.NoError(t, err, "Should create service container without errors")
+}
+
 func TestDockerCPUSSetting(t *testing.T) {
 	examples := []struct {
 		cpus     string
@@ -890,7 +908,7 @@ func TestDockerCPUSSetting(t *testing.T) {
 				assert.Equal(t, int64(example.nanocpus), hostConfig.NanoCPUs)
 			}
 
-			testDockerConfiguration(t, dockerConfig, cce)
+			testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
 		})
 	}
 }
@@ -904,9 +922,22 @@ func TestDockerCPUSetCPUsSetting(t *testing.T) {
 		assert.Equal(t, "1-3,5", hostConfig.CpusetCpus)
 	}
 
-	testDockerConfiguration(t, dockerConfig, cce)
+	testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
 }
 
+func TestDockerServicesTmpfsSetting(t *testing.T) {
+	dockerConfig := &common.DockerConfig{
+		ServicesTmpfs: map[string]string{
+			"/tmpfs": "rw,noexec",
+		},
+	}
+
+	cce := func(t *testing.T, config *container.Config, hostConfig *container.HostConfig) {
+		require.NotEmpty(t, hostConfig.Tmpfs)
+	}
+
+	testDockerConfigurationWithServiceContainer(t, dockerConfig, cce)
+}
 func TestDockerUserNSSetting(t *testing.T) {
 	dockerConfig := &common.DockerConfig{
 		UsernsMode: "host",
@@ -916,7 +947,8 @@ func TestDockerUserNSSetting(t *testing.T) {
 		assert.Equal(t, container.UsernsMode("host"), hostConfig.UsernsMode)
 	}
 
-	testDockerConfiguration(t, dockerConfig, cce)
+	testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
+
 }
 
 func init() {