diff --git a/docs/yaml/generate.go b/docs/yaml/generate.go index ff451c1920..da8bd08c96 100644 --- a/docs/yaml/generate.go +++ b/docs/yaml/generate.go @@ -17,7 +17,7 @@ import ( const descriptionSourcePath = "docs/reference/commandline/" func generateCliYaml(opts *options) error { - dockerCli, err := command.NewDockerCli() + dockerCLI, err := command.NewDockerCli() if err != nil { return err } @@ -25,7 +25,7 @@ func generateCliYaml(opts *options) error { Use: "docker [OPTIONS] COMMAND [ARG...]", Short: "The base command for the Docker CLI.", } - commands.AddCommands(cmd, dockerCli) + commands.AddCommands(cmd, dockerCLI) disableFlagsInUseLine(cmd) source := filepath.Join(opts.source, descriptionSourcePath) fmt.Println("Markdown source:", source) @@ -33,6 +33,10 @@ func generateCliYaml(opts *options) error { return err } + if err := os.MkdirAll(opts.target, 0755); err != nil { + return err + } + cmd.DisableAutoGenTag = true return GenYamlTree(cmd, opts.target) } @@ -80,9 +84,7 @@ func loadLongDescription(parentCmd *cobra.Command, path string) error { if err != nil { return err } - description, examples := parseMDContent(string(content)) - cmd.Long = description - cmd.Example = examples + applyDescriptionAndExamples(cmd, string(content)) } return nil } diff --git a/docs/yaml/markdown.go b/docs/yaml/markdown.go new file mode 100644 index 0000000000..0ad78613a8 --- /dev/null +++ b/docs/yaml/markdown.go @@ -0,0 +1,73 @@ +package main + +import ( + "regexp" + "strings" + "unicode" +) + +var ( + // mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce + // false positives for (e.g.) comments in code-blocks (# this is a comment), + // so should not be used as a generic regex for other purposes. + mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`) + // htmlAnchor matches inline HTML anchors. This is intended to only match anchors + // for our use-case; DO NOT consider using this as a generic regex, or at least + // not before reading https://stackoverflow.com/a/1732454/1811501. + htmlAnchor = regexp.MustCompile(`\s*`) +) + +// getSections returns all H2 sections by title (lowercase) +func getSections(mdString string) map[string]string { + parsedContent := strings.Split("\n"+mdString, "\n## ") + sections := make(map[string]string, len(parsedContent)) + for _, s := range parsedContent { + if strings.HasPrefix(s, "#") { + // not a H2 Section + continue + } + parts := strings.SplitN(s, "\n", 2) + if len(parts) == 2 { + sections[strings.ToLower(parts[0])] = parts[1] + } + } + return sections +} + +// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in +// YAML. It removes trailing whitespace and substitutes tabs for four spaces +// to prevent YAML switching to use "compact" form; ("line1 \nline\t2\n") +// which, although equivalent, is hard to read. +func cleanupMarkDown(mdString string) (md string, anchors []string) { + // remove leading/trailing whitespace, and replace tabs in the whole content + mdString = strings.TrimSpace(mdString) + mdString = strings.ReplaceAll(mdString, "\t", " ") + mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "") + + var id string + // replace trailing whitespace per line, and handle custom anchors + lines := strings.Split(mdString, "\n") + for i := 0; i < len(lines); i++ { + lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace) + lines[i], id = convertHTMLAnchor(lines[i]) + if id != "" { + anchors = append(anchors, id) + } + } + return strings.Join(lines, "\n"), anchors +} + +// convertHTMLAnchor converts inline anchor-tags in headings () +// to an extended-markdown property ({#myanchor}). Extended Markdown properties +// are not supported in GitHub Flavored Markdown, but are supported by Jekyll, +// and lead to cleaner HTML in our docs, and prevents duplicate anchors. +// It returns the converted MarkDown heading and the custom ID (if present) +func convertHTMLAnchor(mdLine string) (md string, customID string) { + if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 { + if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 { + customID = a[1] + mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}" + } + } + return mdLine, customID +} diff --git a/docs/yaml/markdown_test.go b/docs/yaml/markdown_test.go new file mode 100644 index 0000000000..1d244c9662 --- /dev/null +++ b/docs/yaml/markdown_test.go @@ -0,0 +1,132 @@ +package main + +import "testing" + +func TestCleanupMarkDown(t *testing.T) { + tests := []struct { + doc, in, expected string + }{ + { + doc: "whitespace around sections", + in: ` + + ## Section start + +Some lines. +And more lines. + +`, + expected: `## Section start + +Some lines. +And more lines.`, + }, + { + doc: "lines with inline tabs", + in: `## Some Heading + +A line with tabs in it. +Tabs should be replaced by spaces`, + expected: `## Some Heading + +A line with tabs in it. +Tabs should be replaced by spaces`, + }, + { + doc: "lines with trailing spaces", + in: `## Some Heading with spaces + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + expected: `## Some Heading with spaces + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + }, + { + doc: "lines with trailing tabs", + in: `## Some Heading with tabs + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + expected: `## Some Heading with tabs + +This is a line. + This is an indented line + +### Some other heading + +Last line.`, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + out, _ := cleanupMarkDown(tc.in) + if out != tc.expected { + t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out) + } + }) + } +} + +func TestConvertHTMLAnchor(t *testing.T) { + tests := []struct { + in, id, expected string + }{ + { + in: `# Heading 1`, + id: "heading1", + expected: `# Heading 1 {#heading1}`, + }, + { + in: `## Heading 2 `, + id: "heading2", + expected: `## Heading 2 {#heading2}`, + }, + { + in: `### Heading 3`, + id: "heading3", + expected: `### Heading 3 {#heading3}`, + }, + { + in: `#### Heading 4`, + id: "heading4", + expected: `#### Heading 4 {#heading4}`, + }, + { + in: `##### Heading 5`, + id: "heading5", + expected: `##### Heading 5 {#heading5}`, + }, + { + in: `###### hello!Heading 6`, + id: "", + expected: `###### hello!Heading 6`, + }, + } + for _, tc := range tests { + tc := tc + t.Run(tc.in, func(t *testing.T) { + out, id := convertHTMLAnchor(tc.in) + if id != tc.id { + t.Fatalf("expected: %s, actual: %s\n", tc.id, id) + } + if out != tc.expected { + t.Fatalf("\nexpected: %s\nactual: %s\n", tc.expected, out) + } + }) + } +} diff --git a/docs/yaml/yaml.go b/docs/yaml/yaml.go index 9ffd8bba3a..e5e875940f 100644 --- a/docs/yaml/yaml.go +++ b/docs/yaml/yaml.go @@ -19,6 +19,7 @@ type cmdOption struct { ValueType string `yaml:"value_type,omitempty"` DefaultValue string `yaml:"default_value,omitempty"` Description string `yaml:",omitempty"` + DetailsURL string `yaml:"details_url,omitempty"` // DetailsURL contains an anchor-id or link for more information on this flag Deprecated bool MinAPIVersion string `yaml:"min_api_version,omitempty"` Experimental bool @@ -61,14 +62,20 @@ func GenYamlTree(cmd *cobra.Command, dir string) error { // GenYamlTreeCustom creates yaml structured ref files func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error { for _, c := range cmd.Commands() { - if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + if !c.Runnable() && !c.HasAvailableSubCommands() { + // skip non-runnable commands without subcommands + // but *do* generate YAML for hidden and deprecated commands + // the YAML will have those included as metadata, so that the + // documentation repository can decide whether or not to present them continue } if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil { return err } } - + if !cmd.HasParent() { + return nil + } basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml" filename := filepath.Join(dir, basename) f, err := os.Create(filename) @@ -86,12 +93,29 @@ func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string // GenYamlCustom creates custom yaml output // nolint: gocyclo func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { - cliDoc := cmdDoc{} - cliDoc.Name = cmd.CommandPath() + const ( + // shortMaxWidth is the maximum width for the "Short" description before + // we force YAML to use multi-line syntax. The goal is to make the total + // width fit within 80 characters. This value is based on 80 characters + // minus the with of the field, colon, and whitespace ('short: '). + shortMaxWidth = 73 + + // longMaxWidth is the maximum width for the "Short" description before + // we force YAML to use multi-line syntax. The goal is to make the total + // width fit within 80 characters. This value is based on 80 characters + // minus the with of the field, colon, and whitespace ('long: '). + longMaxWidth = 74 + ) + + cliDoc := cmdDoc{ + Name: cmd.CommandPath(), + Aliases: strings.Join(cmd.Aliases, ", "), + Short: forceMultiLine(cmd.Short, shortMaxWidth), + Long: forceMultiLine(cmd.Long, longMaxWidth), + Example: cmd.Example, + Deprecated: len(cmd.Deprecated) > 0, + } - cliDoc.Aliases = strings.Join(cmd.Aliases, ", ") - cliDoc.Short = cmd.Short - cliDoc.Long = cmd.Long if len(cliDoc.Long) == 0 { cliDoc.Long = cliDoc.Short } @@ -100,12 +124,6 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { cliDoc.Usage = cmd.UseLine() } - if len(cmd.Example) > 0 { - cliDoc.Example = cmd.Example - } - if len(cmd.Deprecated) > 0 { - cliDoc.Deprecated = true - } // Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack` for curr := cmd; curr != nil; curr = curr.Parent() { if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" { @@ -123,26 +141,32 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm { cliDoc.Swarm = true } - if os, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" { - cliDoc.OSType = os + if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" { + cliDoc.OSType = o + } + } + + anchors := make(map[string]struct{}) + if a, ok := cmd.Annotations["anchors"]; ok && a != "" { + for _, anchor := range strings.Split(a, ",") { + anchors[anchor] = struct{}{} } } flags := cmd.NonInheritedFlags() if flags.HasFlags() { - cliDoc.Options = genFlagResult(flags) + cliDoc.Options = genFlagResult(flags, anchors) } flags = cmd.InheritedFlags() if flags.HasFlags() { - cliDoc.InheritedOptions = genFlagResult(flags) + cliDoc.InheritedOptions = genFlagResult(flags, anchors) } if hasSeeAlso(cmd) { if cmd.HasParent() { parent := cmd.Parent() cliDoc.Pname = parent.CommandPath() - link := cliDoc.Pname + ".yaml" - cliDoc.Plink = strings.Replace(link, " ", "_", -1) + cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml" cmd.VisitParents(func(c *cobra.Command) { if c.DisableAutoGenTag { cmd.DisableAutoGenTag = c.DisableAutoGenTag @@ -157,10 +181,8 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { continue } - currentChild := cliDoc.Name + " " + child.Name() cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name()) - link := currentChild + ".yaml" - cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1)) + cliDoc.Clink = append(cliDoc.Clink, strings.Replace(cliDoc.Name+"_"+child.Name(), " ", "_", -1)+".yaml") } } @@ -175,21 +197,41 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error { return nil } -func genFlagResult(flags *pflag.FlagSet) []cmdOption { +func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption { var ( result []cmdOption opt cmdOption ) + const ( + // shortMaxWidth is the maximum width for the "Short" description before + // we force YAML to use multi-line syntax. The goal is to make the total + // width fit within 80 characters. This value is based on 80 characters + // minus the with of the field, colon, and whitespace (' default_value: '). + defaultValueMaxWidth = 64 + + // longMaxWidth is the maximum width for the "Short" description before + // we force YAML to use multi-line syntax. The goal is to make the total + // width fit within 80 characters. This value is based on 80 characters + // minus the with of the field, colon, and whitespace (' description: '). + descriptionMaxWidth = 66 + ) + flags.VisitAll(func(flag *pflag.Flag) { opt = cmdOption{ Option: flag.Name, ValueType: flag.Value.Type(), - DefaultValue: forceMultiLine(flag.DefValue), - Description: forceMultiLine(flag.Usage), + DefaultValue: forceMultiLine(flag.DefValue, defaultValueMaxWidth), + Description: forceMultiLine(flag.Usage, descriptionMaxWidth), Deprecated: len(flag.Deprecated) > 0, } + if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 { + opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com") + } else if _, ok = anchors[flag.Name]; ok { + opt.DetailsURL = "#" + flag.Name + } + // Todo, when we mark a shorthand is deprecated, but specify an empty message. // The flag.ShorthandDeprecated is empty as the shorthand is deprecated. // Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok. @@ -230,10 +272,15 @@ func genFlagResult(flags *pflag.FlagSet) []cmdOption { return result } -// Temporary workaround for yaml lib generating incorrect yaml with long strings -// that do not contain \n. -func forceMultiLine(s string) string { - if len(s) > 60 && !strings.Contains(s, "\n") { +// forceMultiLine appends a newline (\n) to strings that are longer than max +// to force the yaml lib to use block notation (https://yaml.org/spec/1.2/spec.html#Block) +// instead of a single-line string with newlines and tabs encoded("string\nline1\nline2"). +// +// This makes the generated YAML more readable, and easier to review changes. +// max can be used to customize the width to keep the whole line < 80 chars. +func forceMultiLine(s string, max int) string { + s = strings.TrimSpace(s) + if len(s) > max && !strings.Contains(s, "\n") { s = s + "\n" } return s @@ -253,17 +300,30 @@ func hasSeeAlso(cmd *cobra.Command) bool { return false } -func parseMDContent(mdString string) (description string, examples string) { - parsedContent := strings.Split(mdString, "\n## ") - for _, s := range parsedContent { - if strings.Index(s, "Description") == 0 { - description = strings.TrimSpace(strings.TrimPrefix(s, "Description")) - } - if strings.Index(s, "Examples") == 0 { - examples = strings.TrimSpace(strings.TrimPrefix(s, "Examples")) - } +// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the +// "Description" and "Examples" H2 sections in mdString (if present). +func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) { + sections := getSections(mdString) + var ( + anchors []string + md string + ) + if sections["description"] != "" { + md, anchors = cleanupMarkDown(sections["description"]) + cmd.Long = md + anchors = append(anchors, md) + } + if sections["examples"] != "" { + md, anchors = cleanupMarkDown(sections["examples"]) + cmd.Example = md + anchors = append(anchors, md) + } + if len(anchors) > 0 { + if cmd.Annotations == nil { + cmd.Annotations = make(map[string]string) + } + cmd.Annotations["anchors"] = strings.Join(anchors, ",") } - return description, examples } type byName []*cobra.Command diff --git a/scripts/docs/generate-yaml.sh b/scripts/docs/generate-yaml.sh index 2a7b0a8980..634876aa75 100755 --- a/scripts/docs/generate-yaml.sh +++ b/scripts/docs/generate-yaml.sh @@ -4,5 +4,5 @@ set -eu -o pipefail mkdir -p docs/yaml/gen -go build -o build/yaml-docs-generator github.com/docker/cli/docs/yaml +GO111MODULE=off go build -o build/yaml-docs-generator github.com/docker/cli/docs/yaml build/yaml-docs-generator --root "$(pwd)" --target "$(pwd)/docs/yaml/gen"