Skip to content
Snippets Groups Projects
Commit d66f3d6f authored by Carlo Mion's avatar Carlo Mion
Browse files

Add ServicesTmpfs options to Docker runner configuration. 

This will mount the specified path, (along with the specified mount options) using the native Docker tmpfs support in all the container services.
parent 6a81cebe
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
common/config.go
+ 27
− 26
View file @ d66f3d6f
...@@ -49,32 +49,33 @@ func (p DockerPullPolicy) Get() (DockerPullPolicy, error) { ...@@ -49,32 +49,33 @@ func (p DockerPullPolicy) Get() (DockerPullPolicy, error) {
type DockerConfig struct { type DockerConfig struct {
docker_helpers.DockerCredentials docker_helpers.DockerCredentials
Hostname string `toml:"hostname,omitempty" json:"hostname" long:"hostname" env:"DOCKER_HOSTNAME" description:"Custom container hostname"` Hostname string `toml:"hostname,omitempty" json:"hostname" long:"hostname" env:"DOCKER_HOSTNAME" description:"Custom container hostname"`
Image string `toml:"image" json:"image" long:"image" env:"DOCKER_IMAGE" description:"Docker image to be used"` Image string `toml:"image" json:"image" long:"image" env:"DOCKER_IMAGE" description:"Docker image to be used"`
CPUSetCPUs string `toml:"cpuset_cpus,omitempty" json:"cpuset_cpus" long:"cpuset-cpus" env:"DOCKER_CPUSET_CPUS" description:"String value containing the cgroups CpusetCpus to use"` CPUSetCPUs string `toml:"cpuset_cpus,omitempty" json:"cpuset_cpus" long:"cpuset-cpus" env:"DOCKER_CPUSET_CPUS" description:"String value containing the cgroups CpusetCpus to use"`
CPUS string `toml:"cpus,omitempty" json:"cpus" long:"cpus" env:"DOCKER_CPUS" description:"Number of CPUs"` CPUS string `toml:"cpus,omitempty" json:"cpus" long:"cpus" env:"DOCKER_CPUS" description:"Number of CPUs"`
DNS []string `toml:"dns,omitempty" json:"dns" long:"dns" env:"DOCKER_DNS" description:"A list of DNS servers for the container to use"` DNS []string `toml:"dns,omitempty" json:"dns" long:"dns" env:"DOCKER_DNS" description:"A list of DNS servers for the container to use"`
DNSSearch []string `toml:"dns_search,omitempty" json:"dns_search" long:"dns-search" env:"DOCKER_DNS_SEARCH" description:"A list of DNS search domains"` DNSSearch []string `toml:"dns_search,omitempty" json:"dns_search" long:"dns-search" env:"DOCKER_DNS_SEARCH" description:"A list of DNS search domains"`
Privileged bool `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"DOCKER_PRIVILEGED" description:"Give extended privileges to container"` Privileged bool `toml:"privileged,omitzero" json:"privileged" long:"privileged" env:"DOCKER_PRIVILEGED" description:"Give extended privileges to container"`
UsernsMode string `toml:"userns_mode,omitempty" json:"userns_mode" long:"userns" env:"DOCKER_USERNS_MODE" description:"User namespace to use"` UsernsMode string `toml:"userns_mode,omitempty" json:"userns_mode" long:"userns" env:"DOCKER_USERNS_MODE" description:"User namespace to use"`
CapAdd []string `toml:"cap_add" json:"cap_add" long:"cap-add" env:"DOCKER_CAP_ADD" description:"Add Linux capabilities"` CapAdd []string `toml:"cap_add" json:"cap_add" long:"cap-add" env:"DOCKER_CAP_ADD" description:"Add Linux capabilities"`
CapDrop []string `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"DOCKER_CAP_DROP" description:"Drop Linux capabilities"` CapDrop []string `toml:"cap_drop" json:"cap_drop" long:"cap-drop" env:"DOCKER_CAP_DROP" description:"Drop Linux capabilities"`
SecurityOpt []string `toml:"security_opt" json:"security_opt" long:"security-opt" env:"DOCKER_SECURITY_OPT" description:"Security Options"` SecurityOpt []string `toml:"security_opt" json:"security_opt" long:"security-opt" env:"DOCKER_SECURITY_OPT" description:"Security Options"`
Devices []string `toml:"devices" json:"devices" long:"devices" env:"DOCKER_DEVICES" description:"Add a host device to the container"` Devices []string `toml:"devices" json:"devices" long:"devices" env:"DOCKER_DEVICES" description:"Add a host device to the container"`
DisableCache bool `toml:"disable_cache,omitzero" json:"disable_cache" long:"disable-cache" env:"DOCKER_DISABLE_CACHE" description:"Disable all container caching"` DisableCache bool `toml:"disable_cache,omitzero" json:"disable_cache" long:"disable-cache" env:"DOCKER_DISABLE_CACHE" description:"Disable all container caching"`
Volumes []string `toml:"volumes,omitempty" json:"volumes" long:"volumes" env:"DOCKER_VOLUMES" description:"Bind mount a volumes"` Volumes []string `toml:"volumes,omitempty" json:"volumes" long:"volumes" env:"DOCKER_VOLUMES" description:"Bind mount a volumes"`
VolumeDriver string `toml:"volume_driver,omitempty" json:"volume_driver" long:"volume-driver" env:"DOCKER_VOLUME_DRIVER" description:"Volume driver to be used"` VolumeDriver string `toml:"volume_driver,omitempty" json:"volume_driver" long:"volume-driver" env:"DOCKER_VOLUME_DRIVER" description:"Volume driver to be used"`
CacheDir string `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"DOCKER_CACHE_DIR" description:"Directory where to store caches"` CacheDir string `toml:"cache_dir,omitempty" json:"cache_dir" long:"cache-dir" env:"DOCKER_CACHE_DIR" description:"Directory where to store caches"`
ExtraHosts []string `toml:"extra_hosts,omitempty" json:"extra_hosts" long:"extra-hosts" env:"DOCKER_EXTRA_HOSTS" description:"Add a custom host-to-IP mapping"` ExtraHosts []string `toml:"extra_hosts,omitempty" json:"extra_hosts" long:"extra-hosts" env:"DOCKER_EXTRA_HOSTS" description:"Add a custom host-to-IP mapping"`
VolumesFrom []string `toml:"volumes_from,omitempty" json:"volumes_from" long:"volumes-from" env:"DOCKER_VOLUMES_FROM" description:"A list of volumes to inherit from another container"` VolumesFrom []string `toml:"volumes_from,omitempty" json:"volumes_from" long:"volumes-from" env:"DOCKER_VOLUMES_FROM" description:"A list of volumes to inherit from another container"`
NetworkMode string `toml:"network_mode,omitempty" json:"network_mode" long:"network-mode" env:"DOCKER_NETWORK_MODE" description:"Add container to a custom network"` NetworkMode string `toml:"network_mode,omitempty" json:"network_mode" long:"network-mode" env:"DOCKER_NETWORK_MODE" description:"Add container to a custom network"`
Links []string `toml:"links,omitempty" json:"links" long:"links" env:"DOCKER_LINKS" description:"Add link to another container"` Links []string `toml:"links,omitempty" json:"links" long:"links" env:"DOCKER_LINKS" description:"Add link to another container"`
Services []string `toml:"services,omitempty" json:"services" long:"services" env:"DOCKER_SERVICES" description:"Add service that is started with container"` Services []string `toml:"services,omitempty" json:"services" long:"services" env:"DOCKER_SERVICES" description:"Add service that is started with container"`
WaitForServicesTimeout int `toml:"wait_for_services_timeout,omitzero" json:"wait_for_services_timeout" long:"wait-for-services-timeout" env:"DOCKER_WAIT_FOR_SERVICES_TIMEOUT" description:"How long to wait for service startup"` WaitForServicesTimeout int `toml:"wait_for_services_timeout,omitzero" json:"wait_for_services_timeout" long:"wait-for-services-timeout" env:"DOCKER_WAIT_FOR_SERVICES_TIMEOUT" description:"How long to wait for service startup"`
AllowedImages []string `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"DOCKER_ALLOWED_IMAGES" description:"Whitelist allowed images"` AllowedImages []string `toml:"allowed_images,omitempty" json:"allowed_images" long:"allowed-images" env:"DOCKER_ALLOWED_IMAGES" description:"Whitelist allowed images"`
AllowedServices []string `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"DOCKER_ALLOWED_SERVICES" description:"Whitelist allowed services"` AllowedServices []string `toml:"allowed_services,omitempty" json:"allowed_services" long:"allowed-services" env:"DOCKER_ALLOWED_SERVICES" description:"Whitelist allowed services"`
PullPolicy DockerPullPolicy `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"DOCKER_PULL_POLICY" description:"Image pull policy: never, if-not-present, always"` PullPolicy DockerPullPolicy `toml:"pull_policy,omitempty" json:"pull_policy" long:"pull-policy" env:"DOCKER_PULL_POLICY" description:"Image pull policy: never, if-not-present, always"`
ShmSize int64 `toml:"shm_size,omitempty" json:"shm_size" long:"shm-size" env:"DOCKER_SHM_SIZE" description:"Shared memory size for docker images (in bytes)"` ShmSize int64 `toml:"shm_size,omitempty" json:"shm_size" long:"shm-size" env:"DOCKER_SHM_SIZE" description:"Shared memory size for docker images (in bytes)"`
ServicesTmpfs map[string]string `toml:"services_tmpfs,omitempty" json:"services_tmpfs" long:"services-tmpfs" env:"DOCKER_SERVICES_TMPFS" description:"A toml table/json object with the format key=values. When set this will mount the specified path in the key as a tmpfs volume in all the service containers, using the options specified as key. For the supported options, see the documentation for the unix 'mount' command"`
} }
type DockerMachine struct { type DockerMachine struct {
......
docs/executors/docker.md
+ 13
− 0
View file @ d66f3d6f
...@@ -196,6 +196,19 @@ distinguish which variable should go where. ...@@ -196,6 +196,19 @@ distinguish which variable should go where.
> >
Secure variables are only passed to the build container. Secure variables are only passed to the build container.
## Mounting a directory in RAM
You can mount a path inside all the services containers in RAM using tmpfs. This can speed up the time required to test if there is a lot of I/O related work, such as with databases.
If you use the `services_tmpfs` option in the runner configuration, you c an specify multiple paths, each with its own options. See the [docker reference](https://docs.docker.com/engine/reference/commandline/run/#mount-tmpfs-tmpfs) for details.
This is an example `config.toml` to mount the data directory for the official Mysql container in RAM.
```
[runners.docker]
[runners.docker.services_tmpfs]
"/var/lib/mysql" = "rw,noexec"
```
## Build directory in service ## Build directory in service
Since version 1.5 GitLab Runner mounts a `/builds` directory to all shared services. Since version 1.5 GitLab Runner mounts a `/builds` directory to all shared services.
......
executors/docker/executor_docker.go
+ 1
− 0
View file @ d66f3d6f
...@@ -591,6 +591,7 @@ func (s *executor) createService(service, version, image string, serviceDefiniti ...@@ -591,6 +591,7 @@ func (s *executor) createService(service, version, image string, serviceDefiniti
Binds: s.binds, Binds: s.binds,
ShmSize: s.Config.Docker.ShmSize, ShmSize: s.Config.Docker.ShmSize,
VolumesFrom: s.volumesFrom, VolumesFrom: s.volumesFrom,
Tmpfs: s.Config.Docker.ServicesTmpfs,
LogConfig: container.LogConfig{ LogConfig: container.LogConfig{
Type: "json-file", Type: "json-file",
}, },
......
executors/docker/executor_docker_test.go
+ 38
− 6
View file @ d66f3d6f
...@@ -835,14 +835,13 @@ func (c *dockerConfigurationTestFakeDockerClient) ContainerCreate(ctx context.Co ...@@ -835,14 +835,13 @@ func (c *dockerConfigurationTestFakeDockerClient) ContainerCreate(ctx context.Co
return container.ContainerCreateCreatedBody{ID: "abc"}, nil return container.ContainerCreateCreatedBody{ID: "abc"}, nil
} }
func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) { func prepareTestDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) (*dockerConfigurationTestFakeDockerClient, *executor) {
c := &dockerConfigurationTestFakeDockerClient{ c := &dockerConfigurationTestFakeDockerClient{
cce: cce, cce: cce,
t: t, t: t,
} }
defer c.AssertExpectations(t)
e := executor{} e := &executor{}
e.client = c e.client = c
e.Config.Docker = dockerConfig e.Config.Docker = dockerConfig
e.Build = &common.Build{ e.Build = &common.Build{
...@@ -861,6 +860,14 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc ...@@ -861,6 +860,14 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc
Return([]types.NetworkResource{}, nil).Once() Return([]types.NetworkResource{}, nil).Once()
c.On("ContainerRemove", mock.Anything, mock.Anything, mock.Anything). c.On("ContainerRemove", mock.Anything, mock.Anything, mock.Anything).
Return(nil).Once() Return(nil).Once()
return c, e
}
func testDockerConfigurationWithJobContainer(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) {
c, e := prepareTestDockerConfiguration(t, dockerConfig, cce)
defer c.AssertExpectations(t)
c.On("ContainerInspect", mock.Anything, "abc"). c.On("ContainerInspect", mock.Anything, "abc").
Return(types.ContainerJSON{}, nil).Once() Return(types.ContainerJSON{}, nil).Once()
...@@ -868,6 +875,17 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc ...@@ -868,6 +875,17 @@ func testDockerConfiguration(t *testing.T, dockerConfig *common.DockerConfig, cc
assert.NoError(t, err, "Should create container without errors") assert.NoError(t, err, "Should create container without errors")
} }
func testDockerConfigurationWithServiceContainer(t *testing.T, dockerConfig *common.DockerConfig, cce containerConfigExpectations) {
c, e := prepareTestDockerConfiguration(t, dockerConfig, cce)
defer c.AssertExpectations(t)
c.On("ContainerStart", mock.Anything, "abc", mock.Anything).
Return(nil).Once()
_, err := e.createService("build", "latest", "alpine", common.Image{Command: []string{"/bin/sh"}})
assert.NoError(t, err, "Should create service container without errors")
}
func TestDockerCPUSSetting(t *testing.T) { func TestDockerCPUSSetting(t *testing.T) {
examples := []struct { examples := []struct {
cpus string cpus string
...@@ -890,7 +908,7 @@ func TestDockerCPUSSetting(t *testing.T) { ...@@ -890,7 +908,7 @@ func TestDockerCPUSSetting(t *testing.T) {
assert.Equal(t, int64(example.nanocpus), hostConfig.NanoCPUs) assert.Equal(t, int64(example.nanocpus), hostConfig.NanoCPUs)
} }
testDockerConfiguration(t, dockerConfig, cce) testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
}) })
} }
} }
...@@ -904,9 +922,22 @@ func TestDockerCPUSetCPUsSetting(t *testing.T) { ...@@ -904,9 +922,22 @@ func TestDockerCPUSetCPUsSetting(t *testing.T) {
assert.Equal(t, "1-3,5", hostConfig.CpusetCpus) assert.Equal(t, "1-3,5", hostConfig.CpusetCpus)
} }
testDockerConfiguration(t, dockerConfig, cce) testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
} }
func TestDockerServicesTmpfsSetting(t *testing.T) {
dockerConfig := &common.DockerConfig{
ServicesTmpfs: map[string]string{
"/tmpfs": "rw,noexec",
},
}
cce := func(t *testing.T, config *container.Config, hostConfig *container.HostConfig) {
require.NotEmpty(t, hostConfig.Tmpfs)
}
testDockerConfigurationWithServiceContainer(t, dockerConfig, cce)
}
func TestDockerUserNSSetting(t *testing.T) { func TestDockerUserNSSetting(t *testing.T) {
dockerConfig := &common.DockerConfig{ dockerConfig := &common.DockerConfig{
UsernsMode: "host", UsernsMode: "host",
...@@ -916,7 +947,8 @@ func TestDockerUserNSSetting(t *testing.T) { ...@@ -916,7 +947,8 @@ func TestDockerUserNSSetting(t *testing.T) {
assert.Equal(t, container.UsernsMode("host"), hostConfig.UsernsMode) assert.Equal(t, container.UsernsMode("host"), hostConfig.UsernsMode)
} }
testDockerConfiguration(t, dockerConfig, cce) testDockerConfigurationWithJobContainer(t, dockerConfig, cce)
} }
func init() { func init() {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment