Merge commit 'b587a908' as 'repo-infra'

repo-infra/.gitignore

+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+/bazel-*

repo-infra/.kazelcfg.json

+{
+	"GoPrefix": "k8s.io/repo-infra"
+}

repo-infra/.travis.yml

+dist: trusty
+sudo: required
+
+# Install Bazel and set up GOPATH.
+before_install:
+- go get -u github.com/alecthomas/gometalinter
+- wget https://github.com/bazelbuild/bazel/releases/download/0.4.4/bazel_0.4.4-linux-x86_64.deb
+- sudo dpkg -i bazel_0.4.4-linux-x86_64.deb
+- mkdir -p $GOPATH/src/k8s.io
+- mv $TRAVIS_BUILD_DIR $GOPATH/src/k8s.io
+- cd $GOPATH/src/k8s.io/repo-infra
+
+install:
+  - gometalinter --install
+  - go get -u github.com/bazelbuild/buildifier/buildifier
+  - go install ./...
+
+script:
+  - verify/verify-boilerplate.sh --rootdir="$GOPATH/src/k8s.io/repo-infra" -v
+  - verify/verify-go-src.sh --rootdir "$GOPATH/src/k8s.io/repo-infra" -v
+  - buildifier -mode=check $(find . -name BUILD -o -name '*.bzl' -type f)
+  - kazel --print-diff --validate
+  - bazel build //...

repo-infra/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load("@io_bazel_rules_go//go:def.bzl", "go_prefix")
+
+go_prefix("k8s.io/repo-infra")

repo-infra/LICENSE

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

repo-infra/README.md

+# Kubernetes repository infrastructure
+
+This repository contains repository infrastructure tools for use in
+`kubernetes` and `kubernetes-incubator` repositories.  Examples:
+
+- Boilerplate verification
+- Go source code quality verification
+- Golang build infrastructure
+
+---
+
+## Using this repository
+
+This repository can be used via some golang "vendoring" mechanism 
+(such as glide), or it can be used via
+[git subtree](http://git.kernel.org/cgit/git/git.git/plain/contrib/subtree/git-subtree.txt).
+
+### Using "vendoring"
+
+The exact mechanism to pull in this repository will vary depending on
+the tool you use. However, unless you end up having this repository
+at the root of your project's repository you wll probably need to 
+make sure you use the `--rootdir` command line parameter to let the
+`verify-boilerplate.sh` know its location, eg:
+
+    verify-boilerplate.sh --rootdir=/home/myrepo
+
+### Using `git subtree`
+
+When using the git subtree mechanism, this repository should be placed in the 
+top level of your project.
+
+To add `repo-infra` to your repository, use the following commands from the 
+root directory of **your** repository.
+
+First, add a git remote for the `repo-infra` repository:
+
+```
+$ git remote add repo-infra git://github.com/kubernetes/repo-infra
+```
+
+This is not strictly necessary, but reduces the typing required for subsequent
+commands.
+
+Next, use `git subtree add` to create a new subtree in the `repo-infra`
+directory within your project:
+
+```
+$ git subtree add -P repo-infra repo-infra master --squash
+```
+
+After this command, you will have:
+
+1.  A `repo-infra` directory in your project containing the content of **this**
+    project
+2.  2 new commits in the active branch:
+  1.  A commit that squashes the git history of the `repo-infra` project
+  2.  A merge commit whose ancestors are:
+    1.  The `HEAD` of the branch prior to when you ran `git subtree add`
+    2.  The commit containing the squashed `repo-infra` commits
+

repo-infra/WORKSPACE

+workspace(name = "io_kubernetes_build")
+
+git_repository(
+    name = "io_bazel_rules_go",
+    commit = "adfad77dabd529ed9d90a4e7b823323628e908d9",
+    remote = "https://github.com/bazelbuild/rules_go.git",
+)
+
+load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
+
+go_repositories()

repo-infra/defs/BUILD

+package(default_visibility = ["//visibility:public"])
+
+py_binary(
+    name = "gcs_uploader",
+    srcs = [
+        "gcs_uploader.py",
+    ],
+)

repo-infra/defs/build.bzl

+def _gcs_upload_impl(ctx):
+  output_lines = []
+  for t in ctx.attr.data:
+      label = str(t.label)
+      upload_path=ctx.attr.upload_paths.get(label, "")
+      for f in t.files:
+          output_lines.append("%s\t%s" % (f.short_path, upload_path))
+
+  ctx.file_action(
+      output = ctx.outputs.targets,
+      content  = "\n".join(output_lines),
+  )
+
+  ctx.file_action(
+      content = "%s --manifest %s --root $PWD -- $@" % (
+          ctx.attr.uploader.files_to_run.executable.short_path,
+          ctx.outputs.targets.short_path,
+      ),
+      output = ctx.outputs.executable,
+      executable = True,
+  )
+
+  return struct(
+      runfiles = ctx.runfiles(
+          files = ctx.files.data + ctx.files.uploader +
+            [ctx.info_file, ctx.version_file, ctx.outputs.targets]
+          )
+      )
+
+# Adds an executable rule to upload the specified artifacts to GCS.
+#
+# The keys in upload_paths must match the elaborated targets exactly; i.e.,
+# one must specify "//foo/bar:bar" and not just "//foo/bar".
+#
+# Both the upload_paths and the path supplied on the commandline can include
+# Python format strings which will be replaced by values from the workspace status,
+# e.g. gs://my-bucket-{BUILD_USER}/stash/{STABLE_BUILD_SCM_REVISION}
+gcs_upload = rule(
+    attrs = {
+        "data": attr.label_list(
+            mandatory = True,
+            allow_files = True,
+        ),
+        "uploader": attr.label(
+            default = Label("//defs:gcs_uploader"),
+            allow_files = True,
+        ),
+        # TODO: combine with 'data' when label_keyed_string_dict is supported in Bazel
+        "upload_paths": attr.string_dict(
+            allow_empty = True,
+        ),
+    },
+    executable = True,
+    outputs = {
+        "targets": "%{name}-targets.txt",
+    },
+    implementation = _gcs_upload_impl,
+)
+
+# Computes the md5sum of the provided src file, saving it in a file named 'name'.
+def md5sum(name, src, visibility=None):
+    native.genrule(
+        name = name + "_genmd5sum",
+        srcs = [src],
+        outs = [name],
+        # Currently each go_binary target has two outputs (the binary and the library),
+        # so we hash both but only save the hash for the binary.
+        cmd = "for f in $(SRCS); do if command -v md5 >/dev/null; then md5 -q $$f>$@; else md5sum $$f | awk '{print $$1}' > $@; fi; done",
+        message = "Computing md5sum",
+        visibility = visibility,
+)
+
+# Computes the sha1sum of the provided src file, saving it in a file named 'name'.
+def sha1sum(name, src, visibility=None):
+    native.genrule(
+        name = name + "_gensha1sum",
+        srcs = [src],
+        outs = [name],
+        # Currently each go_binary target has two outputs (the binary and the library),
+        # so we hash both but only save the hash for the binary.
+        cmd = "command -v sha1sum >/dev/null && cmd=sha1sum || cmd='shasum -a1'; for f in $(SRCS); do $$cmd $$f | awk '{print $$1}' > $@; done",
+        message = "Computing sha1sum",
+        visibility = visibility,
+)
+
+# Creates 3+N rules based on the provided targets:
+# * A filegroup with just the provided targets (named 'name')
+# * A filegroup containing all of the md5 and sha1 hash files ('name-hashes')
+# * A filegroup containing both of the above ('name-and-hashes')
+# * All of the necessary md5sum and sha1sum rules
+def release_filegroup(name, srcs, visibility=None):
+    hashes = []
+    for src in srcs:
+        parts = src.split(":")
+        if len(parts) > 1:
+            basename = parts[1]
+        else:
+            basename = src.split("/")[-1]
+
+        md5sum(name=basename + ".md5", src=src, visibility=visibility)
+        hashes.append(basename + ".md5")
+        sha1sum(name=basename + ".sha1", src=src, visibility=visibility)
+        hashes.append(basename + ".sha1")
+
+    native.filegroup(
+        name = name,
+        srcs = srcs,
+        visibility = visibility,
+    )
+    native.filegroup(
+        name = name + "-hashes",
+        srcs = hashes,
+        visibility = visibility,
+    )
+    native.filegroup(
+        name = name + "-and-hashes",
+        srcs = [name, name + "-hashes"],
+        visibility = visibility,
+    )

repo-infra/defs/deb.bzl

+load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar", "pkg_deb")
+
+KUBERNETES_AUTHORS = "Kubernetes Authors <kubernetes-dev+release@googlegroups.com>"
+
+KUBERNETES_HOMEPAGE = "http://kubernetes.io"
+
+def k8s_deb(name, depends = [], description = ""):
+  pkg_deb(
+      name = name,
+      architecture = "amd64",
+      data = name + "-data",
+      depends = depends,
+      description = description,
+      homepage = KUBERNETES_HOMEPAGE,
+      maintainer = KUBERNETES_AUTHORS,
+      package =  name,
+      version = "1.6.0-alpha",
+  )
+
+def deb_data(name, data = []):
+  deps = []
+  for i, info in enumerate(data):
+    dname = "%s-deb-data-%s" % (name, i)
+    deps += [dname]
+    pkg_tar(
+        name = dname,
+        files = info["files"],
+        mode = info["mode"],
+        package_dir = info["dir"],
+    )
+  pkg_tar(
+      name = name + "-data",
+      deps = deps,
+  )

repo-infra/defs/gcs_uploader.py

+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import atexit
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+def _workspace_status_dict(root):
+    d = {}
+    for f in ("stable-status.txt", "volatile-status.txt"):
+        with open(os.path.join(root, f)) as info_file:
+            for info_line in info_file:
+                info_line = info_line.strip("\n")
+                key, value = info_line.split(" ")
+                d[key] = value
+    return d
+
+def main(argv):
+    scratch = tempfile.mkdtemp(prefix="bazel-gcs.")
+    atexit.register(lambda: shutil.rmtree(scratch))
+
+    workspace_status = _workspace_status_dict(argv.root)
+    with open(argv.manifest) as manifest:
+        for artifact in manifest:
+            artifact = artifact.strip("\n")
+            src_file, dest_dir = artifact.split("\t")
+            dest_dir = dest_dir.format(**workspace_status)
+            scratch_dest_dir = os.path.join(scratch, dest_dir)
+            try:
+                os.makedirs(scratch_dest_dir)
+            except (OSError):
+                # skip directory already exists errors
+                pass
+
+            src = os.path.join(argv.root, src_file)
+            dest = os.path.join(scratch_dest_dir, os.path.basename(src_file))
+            os.symlink(src, dest)
+
+    ret = 0
+    uploaded_paths = []
+    for gcs_path in argv.gcs_paths:
+        gcs_path = gcs_path.format(**workspace_status)
+        local_path = None
+        if gcs_path.startswith("file://"):
+            local_path = gcs_path[len("file://"):]
+        elif "://" not in gcs_path:
+            local_path = gcs_path
+        if local_path and not os.path.exists(local_path):
+            os.makedirs(local_path)
+        ret |= subprocess.call(["gsutil", "-m", "rsync", "-C", "-r", scratch, gcs_path])
+        uploaded_paths.append(gcs_path)
+    print "Uploaded to %s" % " ".join(uploaded_paths)
+    sys.exit(ret)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Upload build targets to GCS.')
+
+    parser.add_argument("--manifest", required=True, help="path to manifest of targets")
+    parser.add_argument("--root", required=True, help="path to root of workspace")
+    parser.add_argument("gcs_paths", nargs="+", help="path in gcs to push targets")
+
+    main(parser.parse_args())

repo-infra/defs/go.bzl

+go_filetype = ["*.go"]
+
+def _compute_genrule_variables(resolved_srcs, resolved_outs):
+  variables = {"SRCS": cmd_helper.join_paths(" ", resolved_srcs),
+               "OUTS": cmd_helper.join_paths(" ", resolved_outs)}
+  if len(resolved_srcs) == 1:
+    variables["<"] = list(resolved_srcs)[0].path
+  if len(resolved_outs) == 1:
+    variables["@"] = list(resolved_outs)[0].path
+  return variables
+
+def _go_sources_aspect_impl(target, ctx):
+  transitive_sources = set(target.go_sources)
+  for dep in ctx.rule.attr.deps:
+    transitive_sources = transitive_sources | dep.transitive_sources
+  return struct(transitive_sources = transitive_sources)
+
+go_sources_aspect = aspect(
+    attr_aspects = ["deps"],
+    implementation = _go_sources_aspect_impl,
+)
+
+def _compute_genrule_command(ctx):
+  cmd = [
+      'set -e',
+      # setup main GOPATH
+      'export GOPATH=/tmp/gopath',
+      'export GO_WORKSPACE=$${GOPATH}/src/' + ctx.attr.go_prefix.go_prefix,
+      'mkdir -p $${GO_WORKSPACE%/*}',
+      'ln -s $$(pwd) $${GO_WORKSPACE}',
+      # setup genfile GOPATH
+      'export GENGOPATH=/tmp/gengopath',
+      'export GENGO_WORKSPACE=$${GENGOPATH}/src/' + ctx.attr.go_prefix.go_prefix,
+      'mkdir -p $${GENGO_WORKSPACE%/*}',
+      'ln -s $$(pwd)/$(GENDIR) $${GENGO_WORKSPACE}',
+      # drop into WORKSPACE
+      'export GOPATH=$${GOPATH}:$${GENGOPATH}',
+      'cd $${GO_WORKSPACE}',
+      # execute user command
+      ctx.attr.cmd.strip(' \t\n\r'),
+  ]
+  return '\n'.join(cmd)
+
+def _go_genrule_impl(ctx):
+  all_srcs = set(ctx.attr._go_toolchain.stdlib)
+  label_dict = {}
+
+  for dep in ctx.attr.go_deps:
+    all_srcs = all_srcs | dep.transitive_sources
+
+  for dep in ctx.attr.srcs:
+    all_srcs = all_srcs | dep.files
+    label_dict[dep.label] = dep.files
+
+  cmd = _compute_genrule_command(ctx)
+
+  resolved_inputs, argv, runfiles_manifests = ctx.resolve_command(
+      command=cmd,
+      attribute="cmd",
+      expand_locations=True,
+      make_variables=_compute_genrule_variables(all_srcs, set(ctx.outputs.outs)),
+      tools=ctx.attr.tools,
+      label_dict=label_dict
+  )
+
+  ctx.action(
+      inputs = list(all_srcs) + resolved_inputs,
+      outputs = ctx.outputs.outs,
+      env = ctx.configuration.default_shell_env + ctx.attr._go_toolchain.env,
+      command = argv,
+      progress_message = "%s %s" % (ctx.attr.message, ctx),
+      mnemonic = "GoGenrule",
+  )
+
+# We have codegen procedures that depend on the "go/*" stdlib packages
+# and thus depend on executing with a valid GOROOT and GOPATH containing
+# some amount transitive go src of dependencies. This go_genrule enables
+# the creation of these sandboxes.
+go_genrule = rule(
+    attrs = {
+        "srcs": attr.label_list(allow_files = True),
+        "tools": attr.label_list(
+            cfg = "host",
+            allow_files = True,
+        ),
+        "outs": attr.output_list(mandatory = True),
+        "cmd": attr.string(mandatory = True),
+        "go_deps": attr.label_list(
+            aspects = [go_sources_aspect],
+        ),
+        "message": attr.string(),
+        "executable": attr.bool(default = False),
+        # Next two rules copied from bazelbuild/rules_go@a9df110cf04e167b33f10473c7e904d780d921e6
+        # and then modified a bit.
+        # These will likely break at some point in the future, pending Bazel toolchain changes.
+        "_go_toolchain": attr.label(
+            default = Label("@io_bazel_rules_go_toolchain//:go_toolchain"),
+        ),
+        "go_prefix": attr.label(
+            providers = ["go_prefix"],
+            default = Label(
+                "//:go_prefix",
+                relative_to_caller_repository = True,
+            ),
+            allow_files = False,
+            cfg = "host",
+        ),
+    },
+    output_to_genfiles = True,
+    implementation = _go_genrule_impl,
+)

repo-infra/kazel/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "kazel",
+    library = ":go_default_library",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "diff.go",
+        "generator.go",
+        "kazel.go",
+        "sourcerer.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//vendor:github.com/bazelbuild/buildifier/core",
+        "//vendor:github.com/golang/glog",
+        "//vendor:go/path/filepath",
+    ],
+)

repo-infra/kazel/README.rst

+kazel - a BUILD file generator for go and bazel
+===============================================
+
+Requirements:
+#############
+
+* Your project must be somewhat compatible with go tool because
+  kazel uses go tool to parse your import tree.
+* You must have a **GOPATH** and **GOROOT** setup and your project must
+  be in the correct location in your **GOPATH**.
+* Your ``./vendor`` directory may not contain ``BUILD`` files.
+
+Usage:
+######
+
+1. Get kazel by running ``go get k8s.io/repo-infra/kazel``.
+
+2. Create a ``.kazelcfg.json`` in the root of the repository. For the
+   kazel repository, the ``.kazelcfg.json`` would look like:
+
+  .. code-block:: json
+
+   {
+     "GoPrefix": "k8s.io/repo-infra",
+     "SrcDirs": [
+       "./kazel"
+     ],
+     "SkippedPaths": [
+       ".*foobar(baz)?.*$"
+     ]
+   }
+
+3. Run kazel:
+
+  .. code-block:: bash
+
+    $ kazel -root=$GOPATH/src/k8s.io/repo-infra
+
+Defaults:
+#########
+
+* **SrcDirs** in ``.kazelcfg.json`` defaults to ``["./"]``
+* ``-root`` option defaults to the current working directory
+
+Automanagement:
+###############
+
+kazel reconciles rules that have the "**automanaged**" tag. If
+you no longer want kazel to manage a rule, you can remove the
+**automanaged** tag and kazel will no longer manage that rule.
+
+kazel only manages srcs, deps, and library attributes of a
+rule after initial creation so you can add and managed other
+attributes like data and copts and kazel will respect your
+changes.
+
+kazel automatically formats all ``BUILD`` files in your repository
+except for those matching **SkippedPaths**.
+
+Adding "sources" rules:
+#######################
+
+If you set "**AddSourcesRules**": ``true`` in your ``.kazelcfg.json``,
+kazel will create "**package-srcs**" and "**all-srcs**" rules in every
+package.
+
+The "**package-srcs**" rule is a glob matching all files in the
+package recursively, but not any files owned by packages in
+subdirectories.
+
+The "**all-srcs**" rule includes both the "**package-srcs**" rule and
+the "**all-srcs**" rules of all subpackages; i.e. **//:all-srcs** will
+include all files in your repository.
+
+The "**package-srcs**" rule defaults to private visibility,
+since it is safer to depend on the "**all-srcs**" rule: if a
+subpackage is added, the "**package-srcs**" rule will no longer
+include those files.
+
+You can remove the "**automanaged**" tag from the "**package-srcs**"
+rule if you need to modify the glob (such as adding excludes).
+It's recommended that you leave the "**all-srcs**" rule
+automanaged.
+
+Validating BUILD files in CI:
+#############################
+
+If you run kazel with ``--validate``, it will not update any ``BUILD`` files, but it
+will exit nonzero if any ``BUILD`` files are out-of-date. You can add ``--print-diff``
+to print out the changes needed.

repo-infra/kazel/config.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+// Cfg defines the configuration options for kazel.
+type Cfg struct {
+	GoPrefix string
+	// evaluated recursively, defaults to ["."]
+	SrcDirs []string
+	// regexps that match packages to skip
+	SkippedPaths []string
+	// whether to add "pkg-srcs" and "all-srcs" filegroups
+	// note that this operates on the entire tree (not just SrcsDirs) but skips anything matching SkippedPaths
+	AddSourcesRules bool
+	// whether to have multiple build files in vendor/ or just one.
+	VendorMultipleBuildFiles bool
+	// whether to manage kubernetes' pkg/generated/openapi.
+	K8sOpenAPIGen bool
+}
+
+// ReadCfg reads and unmarshals the specified json file into a Cfg struct.
+func ReadCfg(cfgPath string) (*Cfg, error) {
+	b, err := ioutil.ReadFile(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+	var cfg Cfg
+	if err := json.Unmarshal(b, &cfg); err != nil {
+		return nil, err
+	}
+	defaultCfg(&cfg)
+	return &cfg, nil
+}
+
+func defaultCfg(c *Cfg) {
+	if len(c.SrcDirs) == 0 {
+		c.SrcDirs = []string{"."}
+	}
+}

repo-infra/kazel/diff.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+// Diff prints the unified diff of the two provided byte slices
+// using the unix diff command.
+func Diff(left, right []byte) error {
+	lf, err := ioutil.TempFile("/tmp", "actual-file-")
+	if err != nil {
+		return err
+	}
+	defer lf.Close()
+	defer os.Remove(lf.Name())
+
+	rf, err := ioutil.TempFile("/tmp", "expected-file-")
+	if err != nil {
+		return err
+	}
+	defer rf.Close()
+	defer os.Remove(rf.Name())
+
+	_, err = lf.Write(left)
+	if err != nil {
+		return err
+	}
+	lf.Close()
+
+	_, err = rf.Write(right)
+	if err != nil {
+		return err
+	}
+	rf.Close()
+
+	cmd := exec.Command("/usr/bin/diff", "-u", lf.Name(), rf.Name())
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Run()
+
+	return nil
+}

repo-infra/kazel/generator.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+const (
+	openAPIGenTag = "// +k8s:openapi-gen"
+
+	baseImport = "k8s.io/kubernetes/"
+	staging    = "staging/src/"
+)
+
+// walkGenerated updates the rule for kubernetes' OpenAPI generated file.
+// This involves reading all go files in the source tree and looking for the
+// "+k8s:openapi-gen" tag. If present, then that package must be supplied to
+// the genrule.
+func (v *Vendorer) walkGenerated() error {
+	if !v.cfg.K8sOpenAPIGen {
+		return nil
+	}
+	v.managedAttrs = append(v.managedAttrs, "openapi_targets", "vendor_targets")
+	paths, err := v.findOpenAPI(v.root)
+	if err != nil {
+		return err
+	}
+	return v.addGeneratedOpenAPIRule(paths)
+}
+
+// findOpenAPI searches for all packages under root that request OpenAPI. It
+// returns the go import paths. It does not follow symlinks.
+func (v *Vendorer) findOpenAPI(root string) ([]string, error) {
+	finfos, err := ioutil.ReadDir(root)
+	if err != nil {
+		return nil, err
+	}
+	var res []string
+	var includeMe bool
+	for _, finfo := range finfos {
+		path := filepath.Join(root, finfo.Name())
+		if finfo.IsDir() && (finfo.Mode()&os.ModeSymlink == 0) {
+			children, err := v.findOpenAPI(path)
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, children...)
+		} else if strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") {
+			b, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, err
+			}
+			if bytes.Contains(b, []byte(openAPIGenTag)) {
+				includeMe = true
+			}
+		}
+	}
+	if includeMe {
+		pkg, err := v.ctx.ImportDir(root, 0)
+		if err != nil {
+			return nil, err
+		}
+		res = append(res, pkg.ImportPath)
+	}
+	return res, nil
+}
+
+// addGeneratedOpenAPIRule updates the pkg/generated/openapi go_default_library
+// rule with the automanaged openapi_targets and vendor_targets.
+func (v *Vendorer) addGeneratedOpenAPIRule(paths []string) error {
+	var openAPITargets []string
+	var vendorTargets []string
+	for _, p := range paths {
+		if !strings.HasPrefix(p, baseImport) {
+			return fmt.Errorf("openapi-gen path outside of kubernetes: %s", p)
+		}
+		np := p[len(baseImport):]
+		if strings.HasPrefix(np, staging) {
+			vendorTargets = append(vendorTargets, np[len(staging):])
+		} else {
+			openAPITargets = append(openAPITargets, np)
+		}
+	}
+	sort.Strings(openAPITargets)
+	sort.Strings(vendorTargets)
+
+	pkgPath := filepath.Join("pkg", "generated", "openapi")
+	for _, r := range v.newRules[pkgPath] {
+		if r.Name() == "go_default_library" {
+			r.SetAttr("openapi_targets", asExpr(openAPITargets))
+			r.SetAttr("vendor_targets", asExpr(vendorTargets))
+			break
+		}
+	}
+	return nil
+}

repo-infra/kazel/kazel.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/build"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"runtime"
+	"sort"
+	"strings"
+
+	"go/path/filepath"
+
+	bzl "github.com/bazelbuild/buildifier/core"
+	"github.com/golang/glog"
+)
+
+const (
+	vendorPath     = "vendor/"
+	automanagedTag = "automanaged"
+)
+
+var (
+	root      = flag.String("root", ".", "root of go source")
+	dryRun    = flag.Bool("dry-run", false, "run in dry mode")
+	printDiff = flag.Bool("print-diff", false, "print diff to stdout")
+	validate  = flag.Bool("validate", false, "run in dry mode and exit nonzero if any BUILD files need to be updated")
+	cfgPath   = flag.String("cfg-path", ".kazelcfg.json", "path to kazel config (relative paths interpreted relative to -repo.")
+)
+
+func main() {
+	flag.Parse()
+	flag.Set("alsologtostderr", "true")
+	if *root == "" {
+		glog.Fatalf("-root argument is required")
+	}
+	if *validate {
+		*dryRun = true
+	}
+	v, err := newVendorer(*root, *cfgPath, *dryRun)
+	if err != nil {
+		glog.Fatalf("unable to build vendorer: %v", err)
+	}
+	if err = os.Chdir(v.root); err != nil {
+		glog.Fatalf("cannot chdir into root %q: %v", v.root, err)
+	}
+
+	if err = v.walkVendor(); err != nil {
+		glog.Fatalf("err walking vendor: %v", err)
+	}
+	if err = v.walkRepo(); err != nil {
+		glog.Fatalf("err walking repo: %v", err)
+	}
+	if err = v.walkGenerated(); err != nil {
+		glog.Fatalf("err walking generated: %v", err)
+	}
+	if _, err = v.walkSource("."); err != nil {
+		glog.Fatalf("err walking source: %v", err)
+	}
+	written := 0
+	if written, err = v.reconcileAllRules(); err != nil {
+		glog.Fatalf("err reconciling rules: %v", err)
+	}
+	if *validate && written > 0 {
+		fmt.Fprintf(os.Stderr, "\n%d BUILD files not up-to-date.\n", written)
+		os.Exit(1)
+	}
+}
+
+// Vendorer collects context, configuration, and cache while walking the tree.
+type Vendorer struct {
+	ctx          *build.Context
+	icache       map[icacheKey]icacheVal
+	skippedPaths []*regexp.Regexp
+	dryRun       bool
+	root         string
+	cfg          *Cfg
+	newRules     map[string][]*bzl.Rule // package path -> list of rules to add or update
+	managedAttrs []string
+}
+
+func newVendorer(root, cfgPath string, dryRun bool) (*Vendorer, error) {
+	absRoot, err := filepath.Abs(root)
+	if err != nil {
+		return nil, fmt.Errorf("could not get absolute path: %v", err)
+	}
+	if !filepath.IsAbs(cfgPath) {
+		cfgPath = filepath.Join(absRoot, cfgPath)
+	}
+	cfg, err := ReadCfg(cfgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	v := Vendorer{
+		ctx:          context(),
+		dryRun:       dryRun,
+		root:         absRoot,
+		icache:       map[icacheKey]icacheVal{},
+		cfg:          cfg,
+		newRules:     make(map[string][]*bzl.Rule),
+		managedAttrs: []string{"srcs", "deps", "library"},
+	}
+
+	for _, sp := range cfg.SkippedPaths {
+		r, err := regexp.Compile(sp)
+		if err != nil {
+			return nil, err
+		}
+		v.skippedPaths = append(v.skippedPaths, r)
+	}
+	for _, builtinSkip := range []string{
+		"^\\.git",
+		"^bazel-*",
+	} {
+		v.skippedPaths = append(v.skippedPaths, regexp.MustCompile(builtinSkip))
+	}
+
+	return &v, nil
+
+}
+
+type icacheKey struct {
+	path, srcDir string
+}
+
+type icacheVal struct {
+	pkg *build.Package
+	err error
+}
+
+func (v *Vendorer) importPkg(path string, srcDir string) (*build.Package, error) {
+	k := icacheKey{path: path, srcDir: srcDir}
+	if val, ok := v.icache[k]; ok {
+		return val.pkg, val.err
+	}
+
+	// cache miss
+	pkg, err := v.ctx.Import(path, srcDir, build.ImportComment)
+	v.icache[k] = icacheVal{pkg: pkg, err: err}
+	return pkg, err
+}
+
+func writeHeaders(file *bzl.File) {
+	pkgRule := bzl.Rule{
+		Call: &bzl.CallExpr{
+			X: &bzl.LiteralExpr{Token: "package"},
+		},
+	}
+	pkgRule.SetAttr("default_visibility", asExpr([]string{"//visibility:public"}))
+
+	file.Stmt = append(file.Stmt,
+		[]bzl.Expr{
+			pkgRule.Call,
+			&bzl.CallExpr{
+				X:    &bzl.LiteralExpr{Token: "licenses"},
+				List: []bzl.Expr{asExpr([]string{"notice"})},
+			},
+			&bzl.CallExpr{
+				X: &bzl.LiteralExpr{Token: "load"},
+				List: asExpr([]string{
+					"@io_bazel_rules_go//go:def.bzl",
+				}).(*bzl.ListExpr).List,
+			},
+		}...,
+	)
+}
+
+func writeRules(file *bzl.File, rules []*bzl.Rule) {
+	for _, rule := range rules {
+		file.Stmt = append(file.Stmt, rule.Call)
+	}
+}
+
+func (v *Vendorer) resolve(ipath string) Label {
+	if ipath == v.cfg.GoPrefix {
+		return Label{
+			tag: "go_default_library",
+		}
+	} else if strings.HasPrefix(ipath, v.cfg.GoPrefix) {
+		return Label{
+			pkg: strings.TrimPrefix(ipath, v.cfg.GoPrefix+"/"),
+			tag: "go_default_library",
+		}
+	}
+	if v.cfg.VendorMultipleBuildFiles {
+		return Label{
+			pkg: "vendor/" + ipath,
+			tag: "go_default_library",
+		}
+	}
+	return Label{
+		pkg: "vendor",
+		tag: ipath,
+	}
+}
+
+func (v *Vendorer) walk(root string, f func(path, ipath string, pkg *build.Package) error) error {
+	skipVendor := true
+	if root == vendorPath {
+		skipVendor = false
+	}
+	return sfilepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			return nil
+		}
+		if skipVendor && strings.HasPrefix(path, vendorPath) {
+			return filepath.SkipDir
+		}
+		for _, r := range v.skippedPaths {
+			if r.Match([]byte(path)) {
+				return filepath.SkipDir
+			}
+		}
+		ipath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		pkg, err := v.importPkg(".", filepath.Join(v.root, path))
+		if err != nil {
+			if _, ok := err.(*build.NoGoError); err != nil && ok {
+				return nil
+			}
+			return err
+		}
+
+		return f(path, ipath, pkg)
+	})
+}
+
+func (v *Vendorer) walkRepo() error {
+	for _, root := range v.cfg.SrcDirs {
+		if err := v.walk(root, v.updatePkg); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (v *Vendorer) updateSinglePkg(path string) error {
+	pkg, err := v.importPkg(".", "./"+path)
+	if err != nil {
+		if _, ok := err.(*build.NoGoError); err != nil && ok {
+			return nil
+		}
+		return err
+	}
+	return v.updatePkg(path, "", pkg)
+}
+
+type ruleType int
+
+// The RuleType* constants enumerate the bazel rules supported by this tool.
+const (
+	RuleTypeGoBinary ruleType = iota
+	RuleTypeGoLibrary
+	RuleTypeGoTest
+	RuleTypeGoXTest
+	RuleTypeCGoGenrule
+	RuleTypeFileGroup
+	RuleTypeOpenAPILibrary
+)
+
+// RuleKind converts a value of the RuleType* enum into the BUILD string.
+func (rt ruleType) RuleKind() string {
+	switch rt {
+	case RuleTypeGoBinary:
+		return "go_binary"
+	case RuleTypeGoLibrary:
+		return "go_library"
+	case RuleTypeGoTest:
+		return "go_test"
+	case RuleTypeGoXTest:
+		return "go_test"
+	case RuleTypeCGoGenrule:
+		return "cgo_genrule"
+	case RuleTypeFileGroup:
+		return "filegroup"
+	case RuleTypeOpenAPILibrary:
+		return "openapi_library"
+	}
+	panic("unreachable")
+}
+
+// NamerFunc is a function that returns the appropriate name for the rule for the provided RuleType.
+type NamerFunc func(ruleType) string
+
+func (v *Vendorer) updatePkg(path, _ string, pkg *build.Package) error {
+
+	srcNameMap := func(srcs ...[]string) *bzl.ListExpr {
+		return asExpr(merge(srcs...)).(*bzl.ListExpr)
+	}
+
+	srcs := srcNameMap(pkg.GoFiles, pkg.SFiles)
+	cgoSrcs := srcNameMap(pkg.CgoFiles, pkg.CFiles, pkg.CXXFiles, pkg.HFiles)
+	testSrcs := srcNameMap(pkg.TestGoFiles)
+	xtestSrcs := srcNameMap(pkg.XTestGoFiles)
+
+	v.addRules(path, v.emit(srcs, cgoSrcs, testSrcs, xtestSrcs, pkg, func(rt ruleType) string {
+		switch rt {
+		case RuleTypeGoBinary:
+			return filepath.Base(pkg.Dir)
+		case RuleTypeGoLibrary:
+			return "go_default_library"
+		case RuleTypeGoTest:
+			return "go_default_test"
+		case RuleTypeGoXTest:
+			return "go_default_xtest"
+		case RuleTypeCGoGenrule:
+			return "cgo_codegen"
+		}
+		panic("unreachable")
+	}))
+
+	return nil
+}
+
+func (v *Vendorer) emit(srcs, cgoSrcs, testSrcs, xtestSrcs *bzl.ListExpr, pkg *build.Package, namer NamerFunc) []*bzl.Rule {
+	var goLibAttrs = make(Attrs)
+	var rules []*bzl.Rule
+
+	deps := v.extractDeps(pkg.Imports)
+
+	if len(srcs.List) >= 0 {
+		goLibAttrs.Set("srcs", srcs)
+	} else if len(cgoSrcs.List) == 0 {
+		return nil
+	}
+
+	if len(deps.List) > 0 {
+		goLibAttrs.SetList("deps", deps)
+	}
+
+	if pkg.IsCommand() {
+		rules = append(rules, newRule(RuleTypeGoBinary, namer, map[string]bzl.Expr{
+			"library": asExpr(":" + namer(RuleTypeGoLibrary)),
+		}))
+	}
+
+	addGoDefaultLibrary := len(cgoSrcs.List) > 0 || len(srcs.List) > 0
+	if len(cgoSrcs.List) != 0 {
+		cgoRuleAttrs := make(Attrs)
+
+		cgoRuleAttrs.SetList("srcs", cgoSrcs)
+		cgoRuleAttrs.SetList("clinkopts", asExpr([]string{"-lz", "-lm", "-lpthread", "-ldl"}).(*bzl.ListExpr))
+
+		rules = append(rules, newRule(RuleTypeCGoGenrule, namer, cgoRuleAttrs))
+
+		goLibAttrs.Set("library", asExpr(":"+namer(RuleTypeCGoGenrule)))
+	}
+
+	if len(testSrcs.List) != 0 {
+		testRuleAttrs := make(Attrs)
+
+		testRuleAttrs.SetList("srcs", testSrcs)
+		testRuleAttrs.SetList("deps", v.extractDeps(pkg.TestImports))
+
+		if addGoDefaultLibrary {
+			testRuleAttrs.Set("library", asExpr(":"+namer(RuleTypeGoLibrary)))
+		}
+		rules = append(rules, newRule(RuleTypeGoTest, namer, testRuleAttrs))
+	}
+
+	if addGoDefaultLibrary {
+		rules = append(rules, newRule(RuleTypeGoLibrary, namer, goLibAttrs))
+	}
+
+	if len(xtestSrcs.List) != 0 {
+		xtestRuleAttrs := make(Attrs)
+
+		xtestRuleAttrs.SetList("srcs", xtestSrcs)
+		xtestRuleAttrs.SetList("deps", v.extractDeps(pkg.XTestImports))
+
+		rules = append(rules, newRule(RuleTypeGoXTest, namer, xtestRuleAttrs))
+	}
+
+	return rules
+}
+
+func (v *Vendorer) addRules(pkgPath string, rules []*bzl.Rule) {
+	cleanPath := filepath.Clean(pkgPath)
+	v.newRules[cleanPath] = append(v.newRules[cleanPath], rules...)
+}
+
+func (v *Vendorer) walkVendor() error {
+	var rules []*bzl.Rule
+	updateFunc := func(path, ipath string, pkg *build.Package) error {
+		srcNameMap := func(srcs ...[]string) *bzl.ListExpr {
+			return asExpr(
+				apply(
+					merge(srcs...),
+					mapper(func(s string) string {
+						return strings.TrimPrefix(filepath.Join(path, s), "vendor/")
+					}),
+				),
+			).(*bzl.ListExpr)
+		}
+
+		srcs := srcNameMap(pkg.GoFiles, pkg.SFiles)
+		cgoSrcs := srcNameMap(pkg.CgoFiles, pkg.CFiles, pkg.CXXFiles, pkg.HFiles)
+		testSrcs := srcNameMap(pkg.TestGoFiles)
+		xtestSrcs := srcNameMap(pkg.XTestGoFiles)
+
+		tagBase := v.resolve(ipath).tag
+
+		rules = append(rules, v.emit(srcs, cgoSrcs, testSrcs, xtestSrcs, pkg, func(rt ruleType) string {
+			switch rt {
+			case RuleTypeGoBinary:
+				return tagBase + "_bin"
+			case RuleTypeGoLibrary:
+				return tagBase
+			case RuleTypeGoTest:
+				return tagBase + "_test"
+			case RuleTypeGoXTest:
+				return tagBase + "_xtest"
+			case RuleTypeCGoGenrule:
+				return tagBase + "_cgo"
+			}
+			panic("unreachable")
+		})...)
+
+		return nil
+	}
+	if v.cfg.VendorMultipleBuildFiles {
+		updateFunc = v.updatePkg
+	}
+	if err := v.walk(vendorPath, updateFunc); err != nil {
+		return err
+	}
+	v.addRules(vendorPath, rules)
+
+	return nil
+}
+
+func (v *Vendorer) extractDeps(deps []string) *bzl.ListExpr {
+	return asExpr(
+		apply(
+			merge(deps),
+			filterer(func(s string) bool {
+				pkg, err := v.importPkg(s, v.root)
+				if err != nil {
+					if strings.Contains(err.Error(), `cannot find package "C"`) ||
+						// added in go1.7
+						strings.Contains(err.Error(), `cannot find package "context"`) ||
+						strings.Contains(err.Error(), `cannot find package "net/http/httptrace"`) {
+						return false
+					}
+					fmt.Fprintf(os.Stderr, "extract err: %v\n", err)
+					return false
+				}
+				if pkg.Goroot {
+					return false
+				}
+				return true
+			}),
+			mapper(func(s string) string {
+				return v.resolve(s).String()
+			}),
+		),
+	).(*bzl.ListExpr)
+}
+
+func (v *Vendorer) reconcileAllRules() (int, error) {
+	var paths []string
+	for path := range v.newRules {
+		paths = append(paths, path)
+	}
+	sort.Strings(paths)
+	written := 0
+	for _, path := range paths {
+		w, err := ReconcileRules(path, v.newRules[path], v.managedAttrs, v.dryRun)
+		if w {
+			written++
+		}
+		if err != nil {
+			return written, err
+		}
+	}
+	return written, nil
+}
+
+// Attrs collects the attributes for a rule.
+type Attrs map[string]bzl.Expr
+
+// Set sets the named attribute to the provided bazel expression.
+func (a Attrs) Set(name string, expr bzl.Expr) {
+	a[name] = expr
+}
+
+// SetList sets the named attribute to the provided bazel expression list.
+func (a Attrs) SetList(name string, expr *bzl.ListExpr) {
+	if len(expr.List) == 0 {
+		return
+	}
+	a[name] = expr
+}
+
+// Label defines a bazel label.
+type Label struct {
+	pkg, tag string
+}
+
+func (l Label) String() string {
+	return fmt.Sprintf("//%v:%v", l.pkg, l.tag)
+}
+
+func asExpr(e interface{}) bzl.Expr {
+	rv := reflect.ValueOf(e)
+	switch rv.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return &bzl.LiteralExpr{Token: fmt.Sprintf("%d", e)}
+	case reflect.Float32, reflect.Float64:
+		return &bzl.LiteralExpr{Token: fmt.Sprintf("%f", e)}
+	case reflect.String:
+		return &bzl.StringExpr{Value: e.(string)}
+	case reflect.Slice, reflect.Array:
+		var list []bzl.Expr
+		for i := 0; i < rv.Len(); i++ {
+			list = append(list, asExpr(rv.Index(i).Interface()))
+		}
+		return &bzl.ListExpr{List: list}
+	default:
+		glog.Fatalf("Uh oh")
+		return nil
+	}
+}
+
+type sed func(s []string) []string
+
+func mapString(in []string, f func(string) string) []string {
+	var out []string
+	for _, s := range in {
+		out = append(out, f(s))
+	}
+	return out
+}
+
+func mapper(f func(string) string) sed {
+	return func(in []string) []string {
+		return mapString(in, f)
+	}
+}
+
+func filterString(in []string, f func(string) bool) []string {
+	var out []string
+	for _, s := range in {
+		if f(s) {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+func filterer(f func(string) bool) sed {
+	return func(in []string) []string {
+		return filterString(in, f)
+	}
+}
+
+func apply(stream []string, seds ...sed) []string {
+	for _, sed := range seds {
+		stream = sed(stream)
+	}
+	return stream
+}
+
+func merge(streams ...[]string) []string {
+	var out []string
+	for _, stream := range streams {
+		out = append(out, stream...)
+	}
+	return out
+}
+
+func newRule(rt ruleType, namer NamerFunc, attrs map[string]bzl.Expr) *bzl.Rule {
+	rule := &bzl.Rule{
+		Call: &bzl.CallExpr{
+			X: &bzl.LiteralExpr{Token: rt.RuleKind()},
+		},
+	}
+	rule.SetAttr("name", asExpr(namer(rt)))
+	for k, v := range attrs {
+		rule.SetAttr(k, v)
+	}
+	rule.SetAttr("tags", asExpr([]string{automanagedTag}))
+	return rule
+}
+
+// findBuildFile determines the name of a preexisting BUILD file, returning
+// a default if no such file exists.
+func findBuildFile(pkgPath string) (bool, string) {
+	options := []string{"BUILD.bazel", "BUILD"}
+	for _, b := range options {
+		path := filepath.Join(pkgPath, b)
+		info, err := os.Stat(path)
+		if err == nil && !info.IsDir() {
+			return true, path
+		}
+	}
+	return false, filepath.Join(pkgPath, "BUILD")
+}
+
+// ReconcileRules reconciles, simplifies, and writes the rules for the specified package, adding
+// additional dependency rules as needed.
+func ReconcileRules(pkgPath string, rules []*bzl.Rule, managedAttrs []string, dryRun bool) (bool, error) {
+	_, path := findBuildFile(pkgPath)
+	info, err := os.Stat(path)
+	if err != nil && os.IsNotExist(err) {
+		f := &bzl.File{}
+		writeHeaders(f)
+		reconcileLoad(f, rules)
+		writeRules(f, rules)
+		return writeFile(path, f, false, dryRun)
+	} else if err != nil {
+		return false, err
+	}
+	if info.IsDir() {
+		return false, fmt.Errorf("%q cannot be a directory", path)
+	}
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return false, err
+	}
+	f, err := bzl.Parse(path, b)
+	if err != nil {
+		return false, err
+	}
+	oldRules := make(map[string]*bzl.Rule)
+	for _, r := range f.Rules("") {
+		oldRules[r.Name()] = r
+	}
+	for _, r := range rules {
+		o, ok := oldRules[r.Name()]
+		if !ok {
+			f.Stmt = append(f.Stmt, r.Call)
+			continue
+		}
+		if !RuleIsManaged(o) {
+			continue
+		}
+		reconcileAttr := func(o, n *bzl.Rule, name string) {
+			if e := n.Attr(name); e != nil {
+				o.SetAttr(name, e)
+			} else {
+				o.DelAttr(name)
+			}
+		}
+		for _, attr := range managedAttrs {
+			reconcileAttr(o, r, attr)
+		}
+		delete(oldRules, r.Name())
+	}
+
+	for _, r := range oldRules {
+		if !RuleIsManaged(r) {
+			continue
+		}
+		f.DelRules(r.Kind(), r.Name())
+	}
+	reconcileLoad(f, f.Rules(""))
+
+	return writeFile(path, f, true, dryRun)
+}
+
+func reconcileLoad(f *bzl.File, rules []*bzl.Rule) {
+	usedRuleKindsMap := map[string]bool{}
+	for _, r := range rules {
+		// Select only the Go rules we need to import, excluding builtins like filegroup.
+		// TODO: make less fragile
+		switch r.Kind() {
+		case "go_prefix", "go_library", "go_binary", "go_test", "go_proto_library", "cgo_genrule", "cgo_library":
+			usedRuleKindsMap[r.Kind()] = true
+		}
+	}
+
+	usedRuleKindsList := []string{}
+	for k := range usedRuleKindsMap {
+		usedRuleKindsList = append(usedRuleKindsList, k)
+	}
+	sort.Strings(usedRuleKindsList)
+
+	for _, r := range f.Rules("load") {
+		const goRulesLabel = "@io_bazel_rules_go//go:def.bzl"
+		args := bzl.Strings(&bzl.ListExpr{List: r.Call.List})
+		if len(args) == 0 {
+			continue
+		}
+		if args[0] != goRulesLabel {
+			continue
+		}
+		if len(usedRuleKindsList) == 0 {
+			f.DelRules(r.Kind(), r.Name())
+			continue
+		}
+		r.Call.List = asExpr(append(
+			[]string{goRulesLabel}, usedRuleKindsList...,
+		)).(*bzl.ListExpr).List
+		break
+	}
+}
+
+// RuleIsManaged returns whether the provided rule is managed by this tool,
+// based on the tags set on the rule.
+func RuleIsManaged(r *bzl.Rule) bool {
+	var automanaged bool
+	for _, tag := range r.AttrStrings("tags") {
+		if tag == automanagedTag {
+			automanaged = true
+			break
+		}
+	}
+	return automanaged
+}
+
+func writeFile(path string, f *bzl.File, exists, dryRun bool) (bool, error) {
+	var info bzl.RewriteInfo
+	bzl.Rewrite(f, &info)
+	out := bzl.Format(f)
+	if exists {
+		orig, err := ioutil.ReadFile(path)
+		if err != nil {
+			return false, err
+		}
+		if bytes.Compare(orig, out) == 0 {
+			return false, nil
+		}
+		if *printDiff {
+			Diff(orig, out)
+		}
+	}
+	if dryRun {
+		fmt.Fprintf(os.Stderr, "DRY-RUN: wrote %q\n", path)
+		return true, nil
+	}
+	werr := ioutil.WriteFile(path, out, 0644)
+	if werr == nil {
+		fmt.Fprintf(os.Stderr, "wrote %q\n", path)
+	}
+	return werr == nil, werr
+}
+
+func context() *build.Context {
+	return &build.Context{
+		GOARCH:      "amd64",
+		GOOS:        "linux",
+		GOROOT:      build.Default.GOROOT,
+		GOPATH:      build.Default.GOPATH,
+		ReleaseTags: []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7", "go1.8"},
+		Compiler:    runtime.Compiler,
+		CgoEnabled:  true,
+	}
+}
+
+func walk(root string, walkFn filepath.WalkFunc) error {
+	return nil
+}

repo-infra/kazel/sourcerer.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	bzl "github.com/bazelbuild/buildifier/core"
+)
+
+const (
+	pkgSrcsTarget = "package-srcs"
+	allSrcsTarget = "all-srcs"
+)
+
+// walkSource walks the source tree recursively from pkgPath, adding
+// any BUILD files to v.newRules to be formatted.
+//
+// If AddSourcesRules is enabled in the kazel config, then we additionally add
+// package-sources and recursive all-srcs filegroups rules to every BUILD file.
+//
+// Returns the list of children all-srcs targets that should be added to the
+// all-srcs rule of the enclosing package.
+func (v *Vendorer) walkSource(pkgPath string) ([]string, error) {
+	// clean pkgPath since we access v.newRules directly
+	pkgPath = filepath.Clean(pkgPath)
+	for _, r := range v.skippedPaths {
+		if r.Match([]byte(pkgPath)) {
+			return nil, nil
+		}
+	}
+	files, err := ioutil.ReadDir(pkgPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Find any children packages we need to include in an all-srcs rule.
+	var children []string
+	for _, f := range files {
+		if f.IsDir() {
+			c, err := v.walkSource(filepath.Join(pkgPath, f.Name()))
+			if err != nil {
+				return nil, err
+			}
+			children = append(children, c...)
+		}
+	}
+
+	// This path is a package either if we've added rules or if a BUILD file already exists.
+	_, hasRules := v.newRules[pkgPath]
+	isPkg := hasRules
+	if !isPkg {
+		isPkg, _ = findBuildFile(pkgPath)
+	}
+
+	if !isPkg {
+		// This directory isn't a package (doesn't contain a BUILD file),
+		// but there might be subdirectories that are packages,
+		// so pass that up to our parent.
+		return children, nil
+	}
+
+	// Enforce formatting the BUILD file, even if we're not adding srcs rules
+	if !hasRules {
+		v.addRules(pkgPath, nil)
+	}
+
+	if !v.cfg.AddSourcesRules {
+		return nil, nil
+	}
+
+	pkgSrcsExpr := &bzl.LiteralExpr{Token: `glob(["**"])`}
+	if pkgPath == "." {
+		pkgSrcsExpr = &bzl.LiteralExpr{Token: `glob(["**"], exclude=["bazel-*/**", ".git/**"])`}
+	}
+
+	v.addRules(pkgPath, []*bzl.Rule{
+		newRule(RuleTypeFileGroup,
+			func(_ ruleType) string { return pkgSrcsTarget },
+			map[string]bzl.Expr{
+				"srcs":       pkgSrcsExpr,
+				"visibility": asExpr([]string{"//visibility:private"}),
+			}),
+		newRule(RuleTypeFileGroup,
+			func(_ ruleType) string { return allSrcsTarget },
+			map[string]bzl.Expr{
+				"srcs": asExpr(append(children, fmt.Sprintf(":%s", pkgSrcsTarget))),
+			}),
+	})
+	return []string{fmt.Sprintf("//%s:%s", pkgPath, allSrcsTarget)}, nil
+}

repo-infra/vendor/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "github.com/bazelbuild/buildifier/core",
+    srcs = [
+        "github.com/bazelbuild/buildifier/core/lex.go",
+        "github.com/bazelbuild/buildifier/core/parse.y.go",
+        "github.com/bazelbuild/buildifier/core/print.go",
+        "github.com/bazelbuild/buildifier/core/quote.go",
+        "github.com/bazelbuild/buildifier/core/rewrite.go",
+        "github.com/bazelbuild/buildifier/core/rule.go",
+        "github.com/bazelbuild/buildifier/core/syntax.go",
+        "github.com/bazelbuild/buildifier/core/tables.go",
+        "github.com/bazelbuild/buildifier/core/walk.go",
+    ],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "github.com/golang/glog",
+    srcs = [
+        "github.com/golang/glog/glog.go",
+        "github.com/golang/glog/glog_file.go",
+    ],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go/path/filepath",
+    srcs = ["go/path/filepath/path.go"],
+    tags = ["automanaged"],
+)