diff --git a/cache/cache.go b/cache/cache.go index 2b40ff269ddd868ab3a9dea63270a11e5338db06..4d30cf64c57b1a31cce923cb667d5b10b6b2a466 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -26,11 +26,11 @@ func generateObjectName(build *common.Build, config *common.CacheConfig, key str } runnerSegment := "" - if !config.Shared { + if !config.GetShared() { runnerSegment = path.Join("runner", build.Runner.ShortDescription()) } - return path.Join(config.Path, runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key) + return path.Join(config.GetPath(), runnerSegment, "project", strconv.Itoa(build.JobInfo.ProjectID), key) } func onAdapter(build *common.Build, key string, handler func(adapter Adapter) *url.URL) *url.URL { diff --git a/cache/gcs/credentials_resolver.go b/cache/gcs/credentials_resolver.go index b0d513225ddd6f46d336a027616c01e67c06fff2..52cbabe24c3468c6f309cdf42c442bf0c2d70f11 100644 --- a/cache/gcs/credentials_resolver.go +++ b/cache/gcs/credentials_resolver.go @@ -65,7 +65,7 @@ func (cr *defaultCredentialsResolver) readCredentialsFromFile() error { } func (cr *defaultCredentialsResolver) readCredentialsFromConfig() error { - if cr.config.CacheGCSCredentials == nil { + if cr.config.AccessID == "" || cr.config.PrivateKey == "" { return fmt.Errorf("GCS config present, but credentials are not configured") } diff --git a/cache/gcs/credentials_resolver_test.go b/cache/gcs/credentials_resolver_test.go index cae8ac75903fa2f95d5dee5aa27eb7570d5a71e9..c9c9e931de9aed8d1d566f83f8193b6b588d1fb6 100644 --- a/cache/gcs/credentials_resolver_test.go +++ b/cache/gcs/credentials_resolver_test.go @@ -63,7 +63,7 @@ func prepareStubbedCredentialsFile(t *testing.T, testCase credentialsResolverTes func getCredentialsConfig(accessID string, privateKey string) *common.CacheGCSConfig { return &common.CacheGCSConfig{ - CacheGCSCredentials: &common.CacheGCSCredentials{ + CacheGCSCredentials: common.CacheGCSCredentials{ AccessID: accessID, PrivateKey: privateKey, }, diff --git a/cache/s3/adapter.go b/cache/s3/adapter.go index ca54884ef0ddcc57922e9eb0b3c3f42b0436c900..5928b8780afa7d56adc66d84d4a7eedc187f27d0 100644 --- a/cache/s3/adapter.go +++ b/cache/s3/adapter.go @@ -72,12 +72,12 @@ var deprecatedConfigHandler = func(config *common.CacheConfig) *common.CacheConf logrus.Warningln("Runner uses S3 caching with deprecated configuration format. Support for deprecated format will be removed in GitLab Runner 12.0") config.S3 = &common.CacheS3Config{ - ServerAddress: config.ServerAddress, - AccessKey: config.AccessKey, - SecretKey: config.SecretKey, - BucketName: config.BucketName, - BucketLocation: config.BucketLocation, - Insecure: config.Insecure, + ServerAddress: config.GetServerAddress(), + AccessKey: config.GetAccessKey(), + SecretKey: config.GetSecretKey(), + BucketName: config.GetBucketName(), + BucketLocation: config.GetBucketLocation(), + Insecure: config.GetInsecure(), } return config diff --git a/cache/s3/adapter_test.go b/cache/s3/adapter_test.go index ee1f846ed055521af6d499ec66cfba3783dae91d..a3597a6dad35c1df369bb0e2ff3017f0837957c8 100644 --- a/cache/s3/adapter_test.go +++ b/cache/s3/adapter_test.go @@ -132,7 +132,8 @@ func TestS3DeprecatedConfigFormatDetection(t *testing.T) { url := adapter.GetDownloadURL() assert.NotNil(t, url) - message, err := hook.LastEntry().String() + entries := hook.AllEntries() + message, err := entries[0].String() require.NoError(t, err) assert.Contains(t, message, "Runner uses S3 caching with deprecated configuration format") } diff --git a/commands/register.go b/commands/register.go index 9d1c1f651fb558f931b74b7498bfccc85c6d800c..436cb043920572e3add32d19674c4a5e0614a12b 100644 --- a/commands/register.go +++ b/commands/register.go @@ -230,6 +230,45 @@ func (s *RegisterCommand) askExecutorOptions() { } } +// DEPRECATED +// TODO: Remove in 12.0 +// +// Writes cache configuration section using new syntax, even if +// old CLI options/env variables were used. +func (s *RegisterCommand) prepareCache() { + cache := s.RunnerConfig.Cache + + // Called to log deprecated usage, if old cli options/env variables are used + cache.Path = cache.GetPath() + cache.Shared = cache.GetShared() + + // Called to assign values and log deprecated usage, if old env variables are used + setStringIfUnset(&cache.S3.ServerAddress, cache.GetServerAddress()) + setStringIfUnset(&cache.S3.AccessKey, cache.GetAccessKey()) + setStringIfUnset(&cache.S3.SecretKey, cache.GetSecretKey()) + setStringIfUnset(&cache.S3.BucketName, cache.GetBucketName()) + setStringIfUnset(&cache.S3.BucketLocation, cache.GetBucketLocation()) + setBoolIfUnset(&cache.S3.Insecure, cache.GetInsecure()) +} + +// TODO: Remove in 12.0 +func setStringIfUnset(setting *string, newSetting string) { + if *setting != "" { + return + } + + *setting = newSetting +} + +// TODO: Remove in 12.0 +func setBoolIfUnset(setting *bool, newSetting bool) { + if *setting { + return + } + + *setting = newSetting +} + func (s *RegisterCommand) Execute(context *cli.Context) { userModeWarning(true) @@ -270,6 +309,7 @@ func (s *RegisterCommand) Execute(context *cli.Context) { s.askExecutor() s.askExecutorOptions() s.addRunner(&s.RunnerConfig) + s.prepareCache() s.saveConfig() log.Printf("Runner registered successfully. Feel free to start it, but if it's running already the config should be automatically reloaded!") @@ -280,8 +320,8 @@ func getHostname() string { return hostname } -func init() { - common.RegisterCommand2("register", "register a new runner", &RegisterCommand{ +func newRegisterCommand() *RegisterCommand { + return &RegisterCommand{ RunnerConfig: common.RunnerConfig{ Name: getHostname(), RunnerSettings: common.RunnerSettings{ @@ -297,5 +337,9 @@ func init() { Locked: true, Paused: false, network: network.NewGitLabClient(), - }) + } +} + +func init() { + common.RegisterCommand2("register", "register a new runner", newRegisterCommand()) } diff --git a/commands/register_test.go b/commands/register_test.go index 257b1ff30a1b92592c964a8042b5f6c4ee19d332..9b84df16581ec48b89156f3ae747d100b27f4f16 100644 --- a/commands/register_test.go +++ b/commands/register_test.go @@ -1,11 +1,18 @@ package commands import ( + "bytes" "flag" + "io/ioutil" + "os" "testing" + "github.com/sirupsen/logrus/hooks/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/urfave/cli" + "gitlab.com/ayufan/golang-cli-helpers" "gitlab.com/gitlab-org/gitlab-runner/common" ) @@ -56,3 +63,129 @@ func TestRegisterCustomMappedDockerCacheVolume(t *testing.T) { assert.Equal(t, 1, len(s.Docker.Volumes)) assert.Equal(t, "/my/cache:/cache", s.Docker.Volumes[0]) } + +func getLogrusOutput(t *testing.T, hook *test.Hook) string { + buf := &bytes.Buffer{} + for _, entry := range hook.AllEntries() { + message, err := entry.String() + require.NoError(t, err) + buf.WriteString(message) + } + + return buf.String() +} + +func mockEnv(t *testing.T, key string, value string) func() { + err := os.Setenv(key, value) + require.NoError(t, err, "Variable %q not set properly", key) + + return func() { + err := os.Unsetenv(key) + assert.NoError(t, err, "Variable %q not unset properly", key) + } +} + +func testRegisterCommandRun(t *testing.T, network common.Network, args ...string) { + cmd := newRegisterCommand() + cmd.network = network + + app := cli.NewApp() + app.Commands = []cli.Command{ + { + Name: "register", + Action: cmd.Execute, + Flags: clihelpers.GetFlagsFromStruct(cmd), + }, + } + + args = append([]string{"binary", "register"}, args...) + app.Run(args) +} + +func testRegisterCommandDeprecatedOptions(t *testing.T, args ...string) (string, string) { + hook := test.NewGlobal() + + network := new(common.MockNetwork) + defer network.AssertExpectations(t) + + network.On("RegisterRunner", mock.Anything, mock.Anything).Once().Return(&common.RegisterRunnerResponse{ + Token: "test-runner-token", + }) + + configFile, err := ioutil.TempFile("", "config.toml") + require.NoError(t, err) + + configFile.Close() + defer os.Remove(configFile.Name()) + + arguments := []string{ + "-n", + "--config", configFile.Name(), + "--url", "http://gitlab.example.com/", + "--registration-token", "test-registration-token", + "--executor", "shell", + "--cache-type", "s3", + } + arguments = append(arguments, args...) + + testRegisterCommandRun(t, network, arguments...) + + content, err := ioutil.ReadFile(configFile.Name()) + require.NoError(t, err) + + return string(content), getLogrusOutput(t, hook) +} + +// TODO: Remove in 12.0 +func TestRegisterCacheDeprecatedOptions_CLIOptions(t *testing.T) { + content, output := testRegisterCommandDeprecatedOptions( + t, + "--cache-s3-cache-path", "test_path", + "--cache-cache-shared", + ) + + assert.Contains(t, content, ` + [runners.cache] + Type = "s3" + Path = "test_path" + Shared = true +`) + + assert.Contains(t, output, "'--cache-s3-cache-path' command line option and `$S3_CACHE_PATH` environment variables are deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-path' or '$CACHE_PATH' instead") + assert.Contains(t, output, "'--cache-cache-shared' command line is deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-shared' instead") +} + +// TODO: Remove in 12.0 +func TestRegisterCacheDeprecatedOptions_EnvVariables(t *testing.T) { + defer mockEnv(t, "S3_CACHE_PATH", "test_path")() + defer mockEnv(t, "S3_SERVER_ADDRESS", "server_address")() + defer mockEnv(t, "S3_ACCESS_KEY", "access_key")() + defer mockEnv(t, "S3_SECRET_KEY", "secret_key")() + defer mockEnv(t, "S3_BUCKET_NAME", "bucket_name")() + defer mockEnv(t, "S3_BUCKET_LOCATION", "bucket_location")() + defer mockEnv(t, "S3_CACHE_INSECURE", "1")() + + content, output := testRegisterCommandDeprecatedOptions(t) + + assert.Contains(t, content, ` + [runners.cache] + Type = "s3" + Path = "test_path" + [runners.cache.s3] + ServerAddress = "server_address" + AccessKey = "access_key" + SecretKey = "secret_key" + BucketName = "bucket_name" + BucketLocation = "bucket_location" + Insecure = true + [runners.cache.gcs] +`) + + assert.Contains(t, output, "'--cache-s3-cache-path' command line option and `$S3_CACHE_PATH` environment variables are deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-path' or '$CACHE_PATH' instead") + assert.Contains(t, output, "S3_SERVER_ADDRESS environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_SERVER_ADDRESS instead") + assert.Contains(t, output, "S3_ACCESS_KEY environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_ACCESS_KEY instead") + assert.Contains(t, output, "S3_SECRET_KEY environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_SECRET_KEY instead") + assert.Contains(t, output, "S3_BUCKET_NAME environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_BUCKET_NAME instead") + assert.Contains(t, output, "S3_BUCKET_LOCATION environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_BUCKET_LOCATION instead") + assert.Contains(t, output, "S3_CACHE_INSECURE environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_INSECURE instead") +} diff --git a/common/config.go b/common/config.go index 203276175a52fcc1980d05ff8bfb4d07e216b1d7..b0069883df03ca91999c7da61074bf6b3c3eae6e 100644 --- a/common/config.go +++ b/common/config.go @@ -9,6 +9,7 @@ import ( "math/big" "os" "path/filepath" + "strconv" "time" "github.com/BurntSushi/toml" @@ -219,14 +220,14 @@ type RunnerCredentials struct { } type CacheGCSCredentials struct { - AccessID string `toml:"AccessID,omitempty" long:"access-id" env:"CACHE_GCS_GOOGLE_ACCESS_ID" description:"ID of GCP Service Account used to access the storage"` + AccessID string `toml:"AccessID,omitempty" long:"access-id" env:"CACHE_GCS_ACCESS_ID" description:"ID of GCP Service Account used to access the storage"` PrivateKey string `toml:"PrivateKey,omitempty" long:"private-key" env:"CACHE_GCS_PRIVATE_KEY" description:"Private key used to sign GCS requests"` } type CacheGCSConfig struct { - *CacheGCSCredentials + CacheGCSCredentials CredentialsFile string `toml:"CredentialsFile,omitempty" long:"credentials-file" env:"GOOGLE_APPLICATION_CREDENTIALS" description:"File with GCP credentials, containing AccessID and PrivateKey"` - BucketName string `toml:"BucketName" long:"bucket-name" env:"CACHE_GCS_BUCKET_NAME" description:"Name of the bucket where cache will be stored"` + BucketName string `toml:"BucketName,omitempty" long:"bucket-name" env:"CACHE_GCS_BUCKET_NAME" description:"Name of the bucket where cache will be stored"` } type CacheS3Config struct { @@ -235,24 +236,26 @@ type CacheS3Config struct { SecretKey string `toml:"SecretKey,omitempty" long:"secret-key" env:"CACHE_S3_SECRET_KEY" description:"S3 Secret Key"` BucketName string `toml:"BucketName,omitempty" long:"bucket-name" env:"CACHE_S3_BUCKET_NAME" description:"Name of the bucket where cache will be stored"` BucketLocation string `toml:"BucketLocation,omitempty" long:"bucket-location" env:"CACHE_S3_BUCKET_LOCATION" description:"Name of S3 region"` - Insecure bool `toml:"Insecure,omitempty" long:"insecure" env:"CACHE_S3_CACHE_INSECURE" description:"Use insecure mode (without https)"` + Insecure bool `toml:"Insecure,omitempty" long:"insecure" env:"CACHE_S3_INSECURE" description:"Use insecure mode (without https)"` } type CacheConfig struct { Type string `toml:"Type,omitempty" long:"type" env:"CACHE_TYPE" description:"Select caching method"` - Path string `toml:"Path,omitempty" long:"s3-cache-path" env:"S3_CACHE_PATH" description:"Name of the path to prepend to the cache URL"` - Shared bool `toml:"Shared,omitempty" long:"cache-shared" env:"CACHE_SHARED" description:"Enable cache sharing between runners."` + Path string `toml:"Path,omitempty" long:"path" env:"CACHE_PATH" description:"Name of the path to prepend to the cache URL"` + Shared bool `toml:"Shared,omitempty" long:"shared" env:"CACHE_SHARED" description:"Enable cache sharing between runners."` S3 *CacheS3Config `toml:"s3,omitempty" json:"s3" namespace:"s3"` GCS *CacheGCSConfig `toml:"gcs,omitempty" json:"gcs" namespace:"gcs"` // TODO: Remove in 12.0 - ServerAddress string `toml:"ServerAddress,omitempty" long:"s3-server-address" env:"S3_SERVER_ADDRESS" description:"A host:port to the used S3-compatible server"` // DEPRECATED - AccessKey string `toml:"AccessKey,omitempty" long:"s3-access-key" env:"S3_ACCESS_KEY" description:"S3 Access Key"` // DEPRECATED - SecretKey string `toml:"SecretKey,omitempty" long:"s3-secret-key" env:"S3_SECRET_KEY" description:"S3 Secret Key"` // DEPRECATED - BucketName string `toml:"BucketName,omitempty" long:"s3-bucket-name" env:"S3_BUCKET_NAME" description:"Name of the bucket where cache will be stored"` // DEPRECATED - BucketLocation string `toml:"BucketLocation,omitempty" long:"s3-bucket-location" env:"S3_BUCKET_LOCATION" description:"Name of S3 region"` // DEPRECATED - Insecure bool `toml:"Insecure,omitempty" long:"s3-insecure" env:"S3_CACHE_INSECURE" description:"Use insecure mode (without https)"` // DEPRECATED + S3CachePath string `toml:"-" long:"s3-cache-path" env:"S3_CACHE_PATH" description:"Name of the path to prepend to the cache URL. DEPRECATED"` // DEPRECATED + CacheShared bool `toml:"-" long:"cache-shared" description:"Enable cache sharing between runners. DEPRECATED"` // DEPRECATED + ServerAddress string `toml:"ServerAddress,omitempty" description:"A host:port to the used S3-compatible server DEPRECATED"` // DEPRECATED + AccessKey string `toml:"AccessKey,omitempty" description:"S3 Access Key DEPRECATED"` // DEPRECATED + SecretKey string `toml:"SecretKey,omitempty" description:"S3 Secret Key DEPRECATED"` // DEPRECATED + BucketName string `toml:"BucketName,omitempty" description:"Name of the bucket where cache will be stored DEPRECATED"` // DEPRECATED + BucketLocation string `toml:"BucketLocation,omitempty" description:"Name of S3 region DEPRECATED"` // DEPRECATED + Insecure bool `toml:"Insecure,omitempty" description:"Use insecure mode (without https) DEPRECATED"` // DEPRECATED } type RunnerSettings struct { @@ -310,10 +313,130 @@ type Config struct { Loaded bool `toml:"-"` } +func getDeprecatedStringSetting(setting string, tomlField string, envVariable string, tomlReplacement string, envReplacement string) string { + if setting != "" { + log.Warningf("%s setting is deprecated and will be removed in GitLab Runner 12.0. Please use %s instead", tomlField, tomlReplacement) + return setting + } + + value := os.Getenv(envVariable) + if value != "" { + log.Warningf("%s environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use %s instead", envVariable, envReplacement) + } + + return value +} + +func getDeprecatedBoolSetting(setting bool, tomlField string, envVariable string, tomlReplacement string, envReplacement string) bool { + if setting { + log.Warningf("%s setting is deprecated and will be removed in GitLab Runner 12.0. Please use %s instead", tomlField, tomlReplacement) + return setting + } + + value, _ := strconv.ParseBool(os.Getenv(envVariable)) + if value { + log.Warningf("%s environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use %s instead", envVariable, envReplacement) + } + + return value +} + func (c *CacheS3Config) ShouldUseIAMCredentials() bool { return c.ServerAddress == "" || c.AccessKey == "" || c.SecretKey == "" } +func (c *CacheConfig) GetPath() string { + if c.Path != "" { + return c.Path + } + + // TODO: Remove in 12.0 + if c.S3CachePath != "" { + log.Warning("'--cache-s3-cache-path' command line option and `$S3_CACHE_PATH` environment variables are deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-path' or '$CACHE_PATH' instead") + } + + return c.S3CachePath +} + +func (c *CacheConfig) GetShared() bool { + if c.Shared { + return c.Shared + } + + // TODO: Remove in 12.0 + if c.CacheShared { + log.Warning("'--cache-cache-shared' command line is deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-shared' instead") + } + + return c.CacheShared +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetServerAddress() string { + return getDeprecatedStringSetting( + c.ServerAddress, + "[runners.cache] ServerAddress", + "S3_SERVER_ADDRESS", + "[runners.cache.s3] ServerAddress", + "CACHE_S3_SERVER_ADDRESS") +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetAccessKey() string { + return getDeprecatedStringSetting( + c.AccessKey, + "[runners.cache] AccessKey", + "S3_ACCESS_KEY", + "[runners.cache.s3] AccessKey", + "CACHE_S3_ACCESS_KEY") +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetSecretKey() string { + return getDeprecatedStringSetting( + c.SecretKey, + "[runners.cache] SecretKey", + "S3_SECRET_KEY", + "[runners.cache.s3] SecretKey", + "CACHE_S3_SECRET_KEY") +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetBucketName() string { + return getDeprecatedStringSetting( + c.BucketName, + "[runners.cache] BucketName", + "S3_BUCKET_NAME", + "[runners.cache.s3] BucketName", + "CACHE_S3_BUCKET_NAME") +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetBucketLocation() string { + return getDeprecatedStringSetting( + c.BucketLocation, + "[runners.cache] BucketLocation", + "S3_BUCKET_LOCATION", + "[runners.cache.s3] BucketLocation", + "CACHE_S3_BUCKET_LOCATION") +} + +// DEPRECATED +// TODO: Remove in 12.0 +func (c *CacheConfig) GetInsecure() bool { + return getDeprecatedBoolSetting( + c.Insecure, + "[runners.cache] Insecure", + "S3_CACHE_INSECURE", + "[runners.cache.s3] Insecure", + "CACHE_S3_INSECURE") +} + func (c *SessionServer) GetSessionTimeout() time.Duration { if c.SessionTimeout > 0 { return time.Duration(c.SessionTimeout) * time.Second diff --git a/common/config_test.go b/common/config_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d60bd85454301d41e8e9f1c2d619a3fca78976bf --- /dev/null +++ b/common/config_test.go @@ -0,0 +1,144 @@ +package common + +import ( + "bytes" + "os" + "testing" + + "github.com/BurntSushi/toml" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli" + "gitlab.com/ayufan/golang-cli-helpers" +) + +func getLogrusOutput(t *testing.T, hook *test.Hook) string { + buf := &bytes.Buffer{} + for _, entry := range hook.AllEntries() { + message, err := entry.String() + require.NoError(t, err) + buf.WriteString(message) + } + + return buf.String() +} + +// TODO: Remove in 12.0 +func TestCacheConfig_DeprecatedSupport_TOML(t *testing.T) { + hook := test.NewGlobal() + content := ` +[cache] + ServerAddress = "server_address" + AccessKey = "access_key" + SecretKey = "secret_key" + BucketName = "bucket_name" + BucketLocation = "bucket_location" + Insecure = true +` + test := struct { + Cache CacheConfig `toml:"cache,omitempty"` + }{} + + _, err := toml.Decode(content, &test) + require.NoError(t, err) + + assert.Equal(t, "server_address", test.Cache.GetServerAddress()) + assert.Equal(t, "access_key", test.Cache.GetAccessKey()) + assert.Equal(t, "secret_key", test.Cache.GetSecretKey()) + assert.Equal(t, "bucket_name", test.Cache.GetBucketName()) + assert.Equal(t, "bucket_location", test.Cache.GetBucketLocation()) + assert.True(t, test.Cache.GetInsecure()) + + output := getLogrusOutput(t, hook) + assert.Contains(t, output, "[runners.cache] ServerAddress setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] ServerAddress instead") + assert.Contains(t, output, "[runners.cache] AccessKey setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] AccessKey instead") + assert.Contains(t, output, "[runners.cache] SecretKey setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] SecretKey instead") + assert.Contains(t, output, "[runners.cache] BucketName setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] BucketName instead") + assert.Contains(t, output, "[runners.cache] BucketLocation setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] BucketLocation instead") + assert.Contains(t, output, "[runners.cache] Insecure setting is deprecated and will be removed in GitLab Runner 12.0. Please use [runners.cache.s3] Insecure instead") +} + +type cacheConfigDeprecatedCommand struct { + Cache CacheConfig `namespace:"cache"` + + test func(c *cacheConfigDeprecatedCommand) +} + +func (c *cacheConfigDeprecatedCommand) Execute(cliCtx *cli.Context) { + c.test(c) +} + +func testCacheConfigDeprecatedSupportRunCommand(test func(c *cacheConfigDeprecatedCommand), args ...string) { + cmd := &cacheConfigDeprecatedCommand{test: test} + + app := cli.NewApp() + app.Commands = []cli.Command{ + { + Name: "test", + Action: cmd.Execute, + Flags: clihelpers.GetFlagsFromStruct(cmd), + }, + } + + args = append([]string{"binary", "test"}, args...) + app.Run(args) +} + +// TODO: Remove in 12.0 +func TestCacheConfig_DeprecatedSupport_CliOptions(t *testing.T) { + hook := test.NewGlobal() + test := func(c *cacheConfigDeprecatedCommand) { + assert.Equal(t, "test_path", c.Cache.GetPath()) + assert.True(t, c.Cache.GetShared()) + } + + testCacheConfigDeprecatedSupportRunCommand(test, "--cache-s3-cache-path", "test_path", "--cache-cache-shared") + + output := getLogrusOutput(t, hook) + assert.Contains(t, output, "'--cache-s3-cache-path' command line option and `$S3_CACHE_PATH` environment variables are deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-path' or '$CACHE_PATH' instead") + assert.Contains(t, output, "'--cache-cache-shared' command line is deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-shared' instead") +} + +func mockEnv(t *testing.T, key string, value string) func() { + err := os.Setenv(key, value) + require.NoError(t, err, "Variable %q not set properly", key) + + return func() { + err := os.Unsetenv(key) + assert.NoError(t, err, "Variable %q not unset properly", key) + } +} + +// TODO: Remove in 12.0 +func TestCacheConfig_DeprecatedSupport_EnvVariables(t *testing.T) { + hook := test.NewGlobal() + test := func(c *cacheConfigDeprecatedCommand) { + assert.Equal(t, "test_path", c.Cache.GetPath()) + assert.Equal(t, "server_address", c.Cache.GetServerAddress()) + assert.Equal(t, "access_key", c.Cache.GetAccessKey()) + assert.Equal(t, "secret_key", c.Cache.GetSecretKey()) + assert.Equal(t, "bucket_name", c.Cache.GetBucketName()) + assert.Equal(t, "bucket_location", c.Cache.GetBucketLocation()) + assert.True(t, c.Cache.GetInsecure()) + } + + defer mockEnv(t, "S3_CACHE_PATH", "test_path")() + defer mockEnv(t, "S3_SERVER_ADDRESS", "server_address")() + defer mockEnv(t, "S3_ACCESS_KEY", "access_key")() + defer mockEnv(t, "S3_SECRET_KEY", "secret_key")() + defer mockEnv(t, "S3_BUCKET_NAME", "bucket_name")() + defer mockEnv(t, "S3_BUCKET_LOCATION", "bucket_location")() + defer mockEnv(t, "S3_CACHE_INSECURE", "1")() + + testCacheConfigDeprecatedSupportRunCommand(test) + + output := getLogrusOutput(t, hook) + assert.Contains(t, output, "'--cache-s3-cache-path' command line option and `$S3_CACHE_PATH` environment variables are deprecated and will be removed in GitLab Runner 12.0. Please use '--cache-path' or '$CACHE_PATH' instead") + assert.Contains(t, output, "S3_SERVER_ADDRESS environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_SERVER_ADDRESS instead") + assert.Contains(t, output, "S3_ACCESS_KEY environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_ACCESS_KEY instead") + assert.Contains(t, output, "S3_SECRET_KEY environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_SECRET_KEY instead") + assert.Contains(t, output, "S3_BUCKET_NAME environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_BUCKET_NAME instead") + assert.Contains(t, output, "S3_BUCKET_LOCATION environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_BUCKET_LOCATION instead") + assert.Contains(t, output, "S3_CACHE_INSECURE environment variables is deprecated and will be removed in GitLab Runner 12.0. Please use CACHE_S3_INSECURE instead") +} diff --git a/docs/configuration/advanced-configuration.md b/docs/configuration/advanced-configuration.md index aea1bc10e96e7cc10d43609b126773e68862a24e..c128739f27f0d7bf9ca7af52b8b10748be02a4a0 100644 --- a/docs/configuration/advanced-configuration.md +++ b/docs/configuration/advanced-configuration.md @@ -512,6 +512,25 @@ With GitLab Runner 11.3.0, the configuration parameters related to S3 were moved The old format of the configuration with S3 configured directly in `[runners.cache]` section is still supported, but is deprecated with GitLab Runner 11.3.0. **This support will be removed in GitLab Runner 12.0.0**. +Bellow is a table containing a summary of `config.toml`, cli options and ENV variables deprecations: + +| Setting | TOML field | CLI option for `register` | ENV for `register` | deprecated TOML field | deprecated CLI option | deprecated ENV | +|---------------------|------------------------------------------|--------------------------------|-----------------------------------|-------------------------------------|-------------------------|-----------------------| +| Type | `[runners.cache] -> Type` | `--cache-type` | `$CACHE_TYPE` | | | | +| Path | `[runners.cache] -> Path` | `--cache-path` | `$CACHE_PATH` | | `--cache-s3-cache-path` | `$S3_CACHE_PATH` | +| Shared | `[runners.cache] -> Shared` | `--cache-shared` | `$CACHE_SHARED` | | `--cache-cache-shared` | | +| S3.ServerAddress | `[runners.cache.s3] -> ServerAddress` | `--cache-s3-server-address` | `$CACHE_S3_SERVER_ADDRESS` | `[runners.cache] -> ServerAddress` | | `$S3_SERVER_ADDRESS` | +| S3.AccessKey | `[runners.cache.s3] -> AccessKey` | `--cache-s3-access-key` | `$CACHE_S3_ACCESS_KEY` | `[runners.cache] -> AccessKey` | | `$S3_ACCESS_KEY` | +| S3.SecretKey | `[runners.cache.s3] -> SecretKey` | `--cache-s3-secret-key` | `$CACHE_S3_SECRET_KEY` | `[runners.cache] -> SecretKey` | | `$S3_SECRET_KEY` | +| S3.BucketName | `[runners.cache.s3] -> BucketName` | `--cache-s3-bucket-name` | `$CACHE_S3_BUCKET_NAME` | `[runners.cache] -> BucketName` | | `$S3_BUCKET_NAME` | +| S3.BucketLocation | `[runners.cache.s3] -> BucketLocation` | `--cache-s3-bucket-location` | `$CACHE_S3_BUCKET_LOCATION` | `[runners.cache] -> BucketLocation` | | `$S3_BUCKET_LOCATION` | +| S3.Insecure | `[runners.cache.s3] -> Insecure` | `--cache-s3-insecure` | `$CACHE_S3_INSECURE` | `[runners.cache] -> Insecure` | | `$S3_INSECURE` | +| GCS.AccessID | `[runners.cache.gcs] -> AccessID` | `--cache-gcs-access-id` | `$CACHE_GCS_ACCESS_ID` | | | | +| GCS.PrivateKey | `[runners.cache.gcs] -> PrivateKey` | `--cache-gcs-private-key` | `$CACHE_GCS_PRIVATE_KEY` | | | | +| GCS.CredentialsFile | `[runners.cache.gcs] -> CredentialsFile` | `--cache-gcs-credentials-file` | `$GOOGLE_APPLICATION_CREDENTIALS` | | | | +| GCS.BucketName | `[runners.cache.gcs] -> BucketName` | `--cache-gcs-bucket-name` | `$CACHE_GCS_BUCKET_NAME` | | | | + + ### The `[runners.cache.s3]` section NOTE: **Note:**