diff --git a/.travis.yml b/.travis.yml
index 3dc29262df0cca5f0ecef965818a1cd561b65f85..f5554e594ac26bf33bf376e1ec65aa13020cafd9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ go:
services:
- postgresql
+ - docker
env:
- DEX_POSTGRES_DATABASE=postgres DEX_POSTGRES_USER=postgres DEX_POSTGRES_HOST="localhost" DEX_LDAP_TESTS=1 DEBIAN_FRONTEND=noninteractive
@@ -21,6 +22,7 @@ install:
script:
- make testall
+ - ./scripts/test-k8s.sh
notifications:
email: false
diff --git a/Documentation/dev-integration-tests.md b/Documentation/dev-integration-tests.md
index 366b4ae3a06becc13e9998573567f538eaee83e5..a9ce1ad3557dda42479aec190c0a431dbcbbc346 100644
--- a/Documentation/dev-integration-tests.md
+++ b/Documentation/dev-integration-tests.md
@@ -2,7 +2,7 @@
## Kubernetes
-Kubernetes tests will only run if the `DEX_KUBECONFIG` environment variable is set.
+Kubernetes tests run against a Kubernetes API server, and are enabled by the `DEX_KUBECONFIG` environment variable:
```
$ export DEX_KUBECONFIG=~/.kube/config
@@ -10,7 +10,11 @@ $ go test -v -i ./storage/kubernetes
$ go test -v ./storage/kubernetes
```
-Because third party resources creation isn't synchronized it's expected that the tests fail the first time. Fear not, and just run them again.
+These tests can be executed locally using docker by running the following script:
+
+```
+$ ./scripts/test-k8s.sh
+```
## Postgres
diff --git a/scripts/test-k8s.sh b/scripts/test-k8s.sh
new file mode 100755
index 0000000000000000000000000000000000000000..86127be13e9e1a2da82180936a27cb3a873d1524
--- /dev/null
+++ b/scripts/test-k8s.sh
@@ -0,0 +1,56 @@
+#!/bin/bash -e
+
+TEMPDIR=$( mktemp -d )
+
+cat << EOF > $TEMPDIR/kubeconfig
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+ cluster:
+ server: http://localhost:8080
+users:
+- name: local
+ user:
+contexts:
+- context:
+ cluster: local
+ user: local
+EOF
+
+cleanup () {
+ docker rm -f $( cat $TEMPDIR/etcd )
+ docker rm -f $( cat $TEMPDIR/kube-apiserver )
+ rm -rf $TEMPDIR
+}
+
+trap "{ CODE=$?; cleanup ; exit $CODE; }" EXIT
+
+docker run \
+ --cidfile=$TEMPDIR/etcd \
+ -d \
+ --net=host \
+ gcr.io/google_containers/etcd:3.1.10 \
+ etcd
+
+docker run \
+ --cidfile=$TEMPDIR/kube-apiserver \
+ -d \
+ -v $TEMPDIR:/var/run/kube-test:ro \
+ --net=host \
+ gcr.io/google_containers/kube-apiserver-amd64:v1.7.4 \
+ kube-apiserver \
+ --etcd-servers=http://localhost:2379 \
+ --service-cluster-ip-range=10.0.0.1/16 \
+ --insecure-bind-address=0.0.0.0 \
+ --insecure-port=8080
+
+until $(curl --output /dev/null --silent --head --fail http://localhost:8080/healthz); do
+ printf '.'
+ sleep 1
+done
+echo "API server ready"
+
+export DEX_KUBECONFIG=$TEMPDIR/kubeconfig
+go test -v -i ./storage/kubernetes
+go test -v ./storage/kubernetes
diff --git a/storage/kubernetes/client.go b/storage/kubernetes/client.go
index 57c975cfa7bf2400f84b6c3afbc05119df16ae98..6eb4e25843efbd15412f7b57e17897d3d3e86c4b 100644
--- a/storage/kubernetes/client.go
+++ b/storage/kubernetes/client.go
@@ -157,7 +157,11 @@ func closeResp(r *http.Response) {
}
func (c *client) get(resource, name string, v interface{}) error {
- url := c.urlFor(c.apiVersion, c.namespace, resource, name)
+ return c.getResource(c.apiVersion, c.namespace, resource, name, v)
+}
+
+func (c *client) getResource(apiVersion, namespace, resource, name string, v interface{}) error {
+ url := c.urlFor(apiVersion, namespace, resource, name)
resp, err := c.client.Get(url)
if err != nil {
return err
diff --git a/storage/kubernetes/storage.go b/storage/kubernetes/storage.go
index 56a7dd8990acab342034cec571ab4b1c4a92db6b..6acf89a04c28a31c4cce524ca7941590677f3e8c 100644
--- a/storage/kubernetes/storage.go
+++ b/storage/kubernetes/storage.go
@@ -53,9 +53,9 @@ func (c *Config) Open(logger logrus.FieldLogger) (storage.Storage, error) {
// open returns a kubernetes client, initializing the third party resources used
// by dex.
//
-// errOnResources controls if errors creating the resources cause this method to return
+// waitForResources controls if errors creating the resources cause this method to return
// immediately (used during testing), or if the client will asynchronously retry.
-func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client, error) {
+func (c *Config) open(logger logrus.FieldLogger, waitForResources bool) (*client, error) {
if c.InCluster && (c.KubeConfigFile != "") {
return nil, errors.New("cannot specify both 'inCluster' and 'kubeConfigFile'")
}
@@ -86,7 +86,7 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
ctx, cancel := context.WithCancel(context.Background())
if !cli.registerCustomResources(c.UseTPR) {
- if errOnResources {
+ if waitForResources {
cancel()
return nil, fmt.Errorf("failed creating custom resources")
}
@@ -110,6 +110,13 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
}()
}
+ if waitForResources {
+ if err := cli.waitForCRDs(ctx); err != nil {
+ cancel()
+ return nil, err
+ }
+ }
+
// If the client is closed, stop trying to create resources.
cli.cancel = cancel
return cli, nil
@@ -122,9 +129,6 @@ func (c *Config) open(logger logrus.FieldLogger, errOnResources bool) (*client,
// It logs all errors, returning true if the resources were created successfully.
//
// Creating a custom resource does not mean that they'll be immediately available.
-//
-// TODO(ericchiang): Provide an option to wait for the resources to actually
-// be available.
func (cli *client) registerCustomResources(useTPR bool) (ok bool) {
ok = true
length := len(customResourceDefinitions)
@@ -164,6 +168,49 @@ func (cli *client) registerCustomResources(useTPR bool) (ok bool) {
return ok
}
+// waitForCRDs waits for all CRDs to be in a ready state, and is used
+// by the tests to synchronize before running conformance.
+func (cli *client) waitForCRDs(ctx context.Context) error {
+ ctx, cancel := context.WithTimeout(ctx, time.Second*30)
+ defer cancel()
+
+ for _, crd := range customResourceDefinitions {
+ for {
+ err := cli.isCRDReady(crd.Name)
+ if err == nil {
+ break
+ }
+
+ cli.logger.Errorf("checking CRD: %v", err)
+
+ select {
+ case <-ctx.Done():
+ return errors.New("timed out waiting for CRDs to be available")
+ case <-time.After(time.Millisecond * 100):
+ }
+ }
+ }
+ return nil
+}
+
+// isCRDReady determines if a CRD is ready by inspecting its conditions.
+func (cli *client) isCRDReady(name string) error {
+ var r k8sapi.CustomResourceDefinition
+ err := cli.getResource("apiextensions.k8s.io/v1beta1", "", "customresourcedefinitions", name, &r)
+ if err != nil {
+ return fmt.Errorf("get crd %s: %v", name, err)
+ }
+
+ conds := make(map[string]string) // For debugging, keep the conditions around.
+ for _, c := range r.Status.Conditions {
+ if c.Type == k8sapi.Established && c.Status == k8sapi.ConditionTrue {
+ return nil
+ }
+ conds[string(c.Type)] = string(c.Status)
+ }
+ return fmt.Errorf("crd %s not ready %#v", name, conds)
+}
+
func (cli *client) Close() error {
if cli.cancel != nil {
cli.cancel()