From 5443f0152f77eb983e6475474b02c2e11d975f1c Mon Sep 17 00:00:00 2001 From: French Ben Date: Fri, 27 Jan 2017 16:47:41 -0800 Subject: [PATCH] docs: added support for CLI yaml file generation Signed-off-by: French Ben Signed-off-by: Tibor Vass --- Makefile | 3 + docs/reference/commandline/load.md | 2 +- docs/reference/commandline/stack_ls.md | 2 +- docs/yaml/Dockerfile | 4 + docs/yaml/generate.go | 86 ++++++++++ docs/yaml/yaml.go | 212 +++++++++++++++++++++++++ hack/make/yaml-docs-generator | 12 ++ hooks/post_build | 19 +++ 8 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 docs/yaml/Dockerfile create mode 100644 docs/yaml/generate.go create mode 100644 docs/yaml/yaml.go create mode 100644 hack/make/yaml-docs-generator create mode 100755 hooks/post_build diff --git a/Makefile b/Makefile index 8f2154c93..b3f839f1a 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,9 @@ run: build ## run the docker daemon in a container shell: build ## start a shell inside the build env $(DOCKER_RUN_DOCKER) bash +yaml-docs-gen: build ## generate documentation YAML files consumed by docs repo + $(DOCKER_RUN_DOCKER) sh -c 'hack/make.sh yaml-docs-generator && ( cd bundles/latest/yaml-docs-generator; mkdir docs; ./yaml-docs-generator --target $$(pwd)/docs )' + test: build ## run the unit, integration and docker-py tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py diff --git a/docs/reference/commandline/load.md b/docs/reference/commandline/load.md index f19362b43..3ce6c19e2 100644 --- a/docs/reference/commandline/load.md +++ b/docs/reference/commandline/load.md @@ -26,7 +26,7 @@ Options: The tarball may be compressed with gzip, bzip, or xz -q, --quiet Suppress the load output but still outputs the imported images ``` -## Descriptino +## Description `docker load` loads a tarred repository from a file or the standard input stream. It restores both images and tags. diff --git a/docs/reference/commandline/stack_ls.md b/docs/reference/commandline/stack_ls.md index 58cbde004..567d947ba 100644 --- a/docs/reference/commandline/stack_ls.md +++ b/docs/reference/commandline/stack_ls.md @@ -27,7 +27,7 @@ Options: --help Print usage ``` -## Descriptino +## Description Lists the stacks. diff --git a/docs/yaml/Dockerfile b/docs/yaml/Dockerfile new file mode 100644 index 000000000..059b97a91 --- /dev/null +++ b/docs/yaml/Dockerfile @@ -0,0 +1,4 @@ +FROM scratch +COPY docs /docs +# CMD cannot be nil so we set it to empty string +CMD [""] diff --git a/docs/yaml/generate.go b/docs/yaml/generate.go new file mode 100644 index 000000000..ea5c00ea1 --- /dev/null +++ b/docs/yaml/generate.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "github.com/docker/docker/cli/command" + "github.com/docker/docker/cli/command/commands" + "github.com/docker/docker/pkg/term" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const descriptionSourcePath = "docs/reference/commandline/" + +func generateCliYaml(opts *options) error { + stdin, stdout, stderr := term.StdStreams() + dockerCli := command.NewDockerCli(stdin, stdout, stderr) + cmd := &cobra.Command{Use: "docker"} + commands.AddCommands(cmd, dockerCli) + source := filepath.Join(opts.source, descriptionSourcePath) + if err := loadLongDescription(cmd, source); err != nil { + return err + } + + cmd.DisableAutoGenTag = true + return GenYamlTree(cmd, opts.target) +} + +func loadLongDescription(cmd *cobra.Command, path ...string) error { + for _, cmd := range cmd.Commands() { + if cmd.Name() == "" { + continue + } + fullpath := filepath.Join(path[0], strings.Join(append(path[1:], cmd.Name()), "_")+".md") + + if cmd.HasSubCommands() { + loadLongDescription(cmd, path[0], cmd.Name()) + } + + if _, err := os.Stat(fullpath); err != nil { + log.Printf("WARN: %s does not exist, skipping\n", fullpath) + continue + } + + content, err := ioutil.ReadFile(fullpath) + if err != nil { + return err + } + description, examples := parseMDContent(string(content)) + cmd.Long = description + cmd.Example = examples + } + return nil +} + +type options struct { + source string + target string +} + +func parseArgs() (*options, error) { + opts := &options{} + cwd, _ := os.Getwd() + flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + flags.StringVar(&opts.source, "root", cwd, "Path to project root") + flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files") + err := flags.Parse(os.Args[1:]) + return opts, err +} + +func main() { + opts, err := parseArgs() + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + } + fmt.Printf("Project root: %s\n", opts.source) + fmt.Printf("Generating yaml files into %s\n", opts.target) + if err := generateCliYaml(opts); err != nil { + fmt.Fprintf(os.Stderr, "Failed to generate yaml files: %s\n", err.Error()) + } +} diff --git a/docs/yaml/yaml.go b/docs/yaml/yaml.go new file mode 100644 index 000000000..575f9bec5 --- /dev/null +++ b/docs/yaml/yaml.go @@ -0,0 +1,212 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "gopkg.in/yaml.v2" +) + +type cmdOption struct { + Option string + Shorthand string `yaml:",omitempty"` + DefaultValue string `yaml:"default_value,omitempty"` + Description string `yaml:",omitempty"` +} + +type cmdDoc struct { + Name string `yaml:"command"` + SeeAlso []string `yaml:"parent,omitempty"` + Version string `yaml:"engine_version,omitempty"` + Aliases string `yaml:",omitempty"` + Short string `yaml:",omitempty"` + Long string `yaml:",omitempty"` + Usage string `yaml:",omitempty"` + Pname string `yaml:",omitempty"` + Plink string `yaml:",omitempty"` + Cname []string `yaml:",omitempty"` + Clink []string `yaml:",omitempty"` + Options []cmdOption `yaml:",omitempty"` + InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"` + Example string `yaml:"examples,omitempty"` +} + +// GenYamlTree creates yaml structured ref files +func GenYamlTree(cmd *cobra.Command, dir string) error { + identity := func(s string) string { return s } + emptyStr := func(s string) string { return "" } + return GenYamlTreeCustom(cmd, dir, emptyStr, identity) +} + +// GenYamlTreeCustom creates yaml structured ref files +func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender, linkHandler func(string) string) error { + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsHelpCommand() { + continue + } + if err := GenYamlTreeCustom(c, dir, filePrepender, linkHandler); err != nil { + return err + } + } + + basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml" + filename := filepath.Join(dir, basename) + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + return err + } + if err := GenYamlCustom(cmd, f, linkHandler); err != nil { + return err + } + return nil +} + +// GenYamlCustom creates custom yaml output +func GenYamlCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error { + cliDoc := cmdDoc{} + cliDoc.Name = cmd.CommandPath() + + // Check experimental: ok := cmd.Tags["experimental"] + + cliDoc.Aliases = strings.Join(cmd.Aliases, ", ") + cliDoc.Short = cmd.Short + cliDoc.Long = cmd.Long + if len(cliDoc.Long) == 0 { + cliDoc.Long = cliDoc.Short + } + + if cmd.Runnable() { + cliDoc.Usage = cmd.UseLine() + } + + if len(cmd.Example) > 0 { + cliDoc.Example = cmd.Example + } + + flags := cmd.NonInheritedFlags() + if flags.HasFlags() { + cliDoc.Options = genFlagResult(flags) + } + flags = cmd.InheritedFlags() + if flags.HasFlags() { + cliDoc.InheritedOptions = genFlagResult(flags) + } + + if hasSeeAlso(cmd) { + if cmd.HasParent() { + parent := cmd.Parent() + cliDoc.Pname = parent.CommandPath() + link := cliDoc.Pname + ".yaml" + cliDoc.Plink = strings.Replace(link, " ", "_", -1) + cmd.VisitParents(func(c *cobra.Command) { + if c.DisableAutoGenTag { + cmd.DisableAutoGenTag = c.DisableAutoGenTag + } + }) + } + + children := cmd.Commands() + sort.Sort(byName(children)) + + for _, child := range children { + if !child.IsAvailableCommand() || child.IsHelpCommand() { + continue + } + currentChild := cliDoc.Name + " " + child.Name() + cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name()) + link := currentChild + ".yaml" + cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1)) + } + } + + final, err := yaml.Marshal(&cliDoc) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if _, err := fmt.Fprintln(w, string(final)); err != nil { + return err + } + return nil +} + +func genFlagResult(flags *pflag.FlagSet) []cmdOption { + var result []cmdOption + + flags.VisitAll(func(flag *pflag.Flag) { + // Todo, when we mark a shorthand is deprecated, but specify an empty message. + // The flag.ShorthandDeprecated is empty as the shorthand is deprecated. + // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok. + if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 { + opt := cmdOption{ + Option: flag.Name, + Shorthand: flag.Shorthand, + DefaultValue: flag.DefValue, + Description: forceMultiLine(flag.Usage), + } + result = append(result, opt) + } else { + opt := cmdOption{ + Option: flag.Name, + DefaultValue: forceMultiLine(flag.DefValue), + Description: forceMultiLine(flag.Usage), + } + result = append(result, opt) + } + }) + + return result +} + +// Temporary workaround for yaml lib generating incorrect yaml with long strings +// that do not contain \n. +func forceMultiLine(s string) string { + if len(s) > 60 && !strings.Contains(s, "\n") { + s = s + "\n" + } + return s +} + +// Small duplication for cobra utils +func hasSeeAlso(cmd *cobra.Command) bool { + if cmd.HasParent() { + return true + } + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsHelpCommand() { + continue + } + return true + } + return false +} + +func parseMDContent(mdString string) (description string, examples string) { + parsedContent := strings.Split(mdString, "\n## ") + for _, s := range parsedContent { + if strings.Index(s, "Description") == 0 { + description = strings.Trim(s, "Description\n") + } + if strings.Index(s, "Examples") == 0 { + examples = strings.Trim(s, "Examples\n") + } + } + return +} + +type byName []*cobra.Command + +func (s byName) Len() int { return len(s) } +func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } diff --git a/hack/make/yaml-docs-generator b/hack/make/yaml-docs-generator new file mode 100644 index 000000000..8548deebb --- /dev/null +++ b/hack/make/yaml-docs-generator @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +[ -z "$KEEPDEST" ] && \ + rm -rf "$DEST" + +( + source "${MAKEDIR}/.binary-setup" + export BINARY_SHORT_NAME="yaml-docs-generator" + export GO_PACKAGE='github.com/docker/docker/docs/yaml' + source "${MAKEDIR}/.binary" +) diff --git a/hooks/post_build b/hooks/post_build new file mode 100755 index 000000000..528170712 --- /dev/null +++ b/hooks/post_build @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ -n "${BUILD_DOCS}" ]; then + set -e + DOCS_IMAGE=${DOCS_IMAGE:-${IMAGE_NAME}-docs} + docker run \ + --entrypoint '' \ + --privileged \ + -e DOCKER_GITCOMMIT=$(git rev-parse --short HEAD) \ + -v $(pwd)/docs/yaml/docs:/docs \ + "${IMAGE_NAME}" \ + sh -c 'hack/make.sh yaml-docs-generator && bundles/latest/yaml-docs-generator/yaml-docs-generator --target /docs' + + ( + cd docs/yaml + docker build -t ${DOCS_IMAGE} . + docker push ${DOCS_IMAGE} + ) +fi