yamldocs: various improvements

- make sure the target directory is created if missing
- add support for custom ID's in headings through `<a>` tags (e.g.
  `<a name=heading2></a>`). This allows use of custom anchors that
  work both on GitHub (GFM doesn't support extended MarkDown), and
  in Jekyll (which does).
- add code to cleanup markdown for use in our docs:
    - remove absolute URLs to https://docs.docker.com
    - remove tabs in MarkDown, and convert them to 4 spaces. This
      prevents the YAML conversion from switching between "short"
      and "long" syntax. Tabs in code examples also don't always
      work well, so using spaces doesn't hurt for that.
- refactor some code for readability, and to be less "hacky" (still
  lots to be improved though)

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

docs/yaml/generate.go

@@ -17,7 +17,7 @@ import (
 const descriptionSourcePath = "docs/reference/commandline/"
 
 func generateCliYaml(opts *options) error {
-	dockerCli, err := command.NewDockerCli()
+	dockerCLI, err := command.NewDockerCli()
 	if err != nil {
 		return err
 	}
@@ -25,7 +25,7 @@ func generateCliYaml(opts *options) error {
 		Use:   "docker [OPTIONS] COMMAND [ARG...]",
 		Short: "The base command for the Docker CLI.",
 	}
-	commands.AddCommands(cmd, dockerCli)
+	commands.AddCommands(cmd, dockerCLI)
 	disableFlagsInUseLine(cmd)
 	source := filepath.Join(opts.source, descriptionSourcePath)
 	fmt.Println("Markdown source:", source)
@@ -33,6 +33,10 @@ func generateCliYaml(opts *options) error {
 		return err
 	}
 
+	if err := os.MkdirAll(opts.target, 0755); err != nil {
+		return err
+	}
+
 	cmd.DisableAutoGenTag = true
 	return GenYamlTree(cmd, opts.target)
 }
@@ -80,9 +84,7 @@ func loadLongDescription(parentCmd *cobra.Command, path string) error {
 		if err != nil {
 			return err
 		}
-		description, examples := parseMDContent(string(content))
-		cmd.Long = description
-		cmd.Example = examples
+		applyDescriptionAndExamples(cmd, string(content))
 	}
 	return nil
 }

docs/yaml/markdown.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+var (
+	// mdHeading matches MarkDown H1..h6 headings. Note that this regex may produce
+	// false positives for (e.g.) comments in code-blocks (# this is a comment),
+	// so should not be used as a generic regex for other purposes.
+	mdHeading = regexp.MustCompile(`^([#]{1,6})\s(.*)$`)
+	// htmlAnchor matches inline HTML anchors. This is intended to only match anchors
+	// for our use-case; DO NOT consider using this as a generic regex, or at least
+	// not before reading https://stackoverflow.com/a/1732454/1811501.
+	htmlAnchor = regexp.MustCompile(`<a\s+(?:name|id)="?([^"]+)"?\s*></a>\s*`)
+)
+
+// getSections returns all H2 sections by title (lowercase)
+func getSections(mdString string) map[string]string {
+	parsedContent := strings.Split("\n"+mdString, "\n## ")
+	sections := make(map[string]string, len(parsedContent))
+	for _, s := range parsedContent {
+		if strings.HasPrefix(s, "#") {
+			// not a H2 Section
+			continue
+		}
+		parts := strings.SplitN(s, "\n", 2)
+		if len(parts) == 2 {
+			sections[strings.ToLower(parts[0])] = parts[1]
+		}
+	}
+	return sections
+}
+
+// cleanupMarkDown cleans up the MarkDown passed in mdString for inclusion in
+// YAML. It removes trailing whitespace and substitutes tabs for four spaces
+// to prevent YAML switching to use "compact" form; ("line1  \nline\t2\n")
+// which, although equivalent, is hard to read.
+func cleanupMarkDown(mdString string) (md string, anchors []string) {
+	// remove leading/trailing whitespace, and replace tabs in the whole content
+	mdString = strings.TrimSpace(mdString)
+	mdString = strings.ReplaceAll(mdString, "\t", "    ")
+	mdString = strings.ReplaceAll(mdString, "https://docs.docker.com", "")
+
+	var id string
+	// replace trailing whitespace per line, and handle custom anchors
+	lines := strings.Split(mdString, "\n")
+	for i := 0; i < len(lines); i++ {
+		lines[i] = strings.TrimRightFunc(lines[i], unicode.IsSpace)
+		lines[i], id = convertHTMLAnchor(lines[i])
+		if id != "" {
+			anchors = append(anchors, id)
+		}
+	}
+	return strings.Join(lines, "\n"), anchors
+}
+
+// convertHTMLAnchor converts inline anchor-tags in headings (<a name=myanchor></a>)
+// to an extended-markdown property ({#myanchor}). Extended Markdown properties
+// are not supported in GitHub Flavored Markdown, but are supported by Jekyll,
+// and lead to cleaner HTML in our docs, and prevents duplicate anchors.
+// It returns the converted MarkDown heading and the custom ID (if present)
+func convertHTMLAnchor(mdLine string) (md string, customID string) {
+	if m := mdHeading.FindStringSubmatch(mdLine); len(m) > 0 {
+		if a := htmlAnchor.FindStringSubmatch(m[2]); len(a) > 0 {
+			customID = a[1]
+			mdLine = m[1] + " " + htmlAnchor.ReplaceAllString(m[2], "") + " {#" + customID + "}"
+		}
+	}
+	return mdLine, customID
+}

docs/yaml/markdown_test.go

@@ -0,0 +1,132 @@
+package main
+
+import "testing"
+
+func TestCleanupMarkDown(t *testing.T) {
+	tests := []struct {
+		doc, in, expected string
+	}{
+		{
+			doc: "whitespace around sections",
+			in: `
+
+	## Section start
+
+Some lines.
+And more lines.
+
+`,
+			expected: `## Section start
+
+Some lines.
+And more lines.`,
+		},
+		{
+			doc: "lines with inline tabs",
+			in: `## Some	Heading
+
+A line with tabs		in it.
+Tabs	should be replaced by spaces`,
+			expected: `## Some    Heading
+
+A line with tabs        in it.
+Tabs    should be replaced by spaces`,
+		},
+		{
+			doc: "lines with trailing spaces",
+			in: `## Some Heading with spaces                  
+       
+This is a line.              
+    This is an indented line        
+
+### Some other heading         
+
+Last line.`,
+			expected: `## Some Heading with spaces
+
+This is a line.
+    This is an indented line
+
+### Some other heading
+
+Last line.`,
+		},
+		{
+			doc: "lines with trailing tabs",
+			in: `## Some Heading with tabs				
+		
+This is a line.		
+	This is an indented line		
+
+### Some other heading 	
+
+Last line.`,
+			expected: `## Some Heading with tabs
+
+This is a line.
+    This is an indented line
+
+### Some other heading
+
+Last line.`,
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.doc, func(t *testing.T) {
+			out, _ := cleanupMarkDown(tc.in)
+			if out != tc.expected {
+				t.Fatalf("\nexpected:\n%q\nactual:\n%q\n", tc.expected, out)
+			}
+		})
+	}
+}
+
+func TestConvertHTMLAnchor(t *testing.T) {
+	tests := []struct {
+		in, id, expected string
+	}{
+		{
+			in:       `# <a name=heading1></a> Heading 1`,
+			id:       "heading1",
+			expected: `# Heading 1 {#heading1}`,
+		},
+		{
+			in:       `## Heading 2<a name=heading2></a> `,
+			id:       "heading2",
+			expected: `## Heading 2 {#heading2}`,
+		},
+		{
+			in:       `### <a id=heading3></a>Heading 3`,
+			id:       "heading3",
+			expected: `### Heading 3 {#heading3}`,
+		},
+		{
+			in:       `#### <a id="heading4"></a> Heading 4`,
+			id:       "heading4",
+			expected: `#### Heading 4 {#heading4}`,
+		},
+		{
+			in:       `##### <a   id="heading5"  ></a>  Heading 5`,
+			id:       "heading5",
+			expected: `##### Heading 5 {#heading5}`,
+		},
+		{
+			in:       `###### <a id=hello href=foo>hello!</a>Heading 6`,
+			id:       "",
+			expected: `###### <a id=hello href=foo>hello!</a>Heading 6`,
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.in, func(t *testing.T) {
+			out, id := convertHTMLAnchor(tc.in)
+			if id != tc.id {
+				t.Fatalf("expected: %s, actual:   %s\n", tc.id, id)
+			}
+			if out != tc.expected {
+				t.Fatalf("\nexpected: %s\nactual:   %s\n", tc.expected, out)
+			}
+		})
+	}
+}

docs/yaml/yaml.go

@@ -19,6 +19,7 @@ type cmdOption struct {
 	ValueType       string `yaml:"value_type,omitempty"`
 	DefaultValue    string `yaml:"default_value,omitempty"`
 	Description     string `yaml:",omitempty"`
+	DetailsURL      string `yaml:"details_url,omitempty"` // DetailsURL contains an anchor-id or link for more information on this flag
 	Deprecated      bool
 	MinAPIVersion   string `yaml:"min_api_version,omitempty"`
 	Experimental    bool
@@ -61,14 +62,20 @@ func GenYamlTree(cmd *cobra.Command, dir string) error {
 // GenYamlTreeCustom creates yaml structured ref files
 func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string) string) error {
 	for _, c := range cmd.Commands() {
-		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
+		if !c.Runnable() && !c.HasAvailableSubCommands() {
+			// skip non-runnable commands without subcommands
+			// but *do* generate YAML for hidden and deprecated commands
+			// the YAML will have those included as metadata, so that the
+			// documentation repository can decide whether or not to present them
 			continue
 		}
 		if err := GenYamlTreeCustom(c, dir, filePrepender); err != nil {
 			return err
 		}
 	}
-
+	if !cmd.HasParent() {
+		return nil
+	}
 	basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
 	filename := filepath.Join(dir, basename)
 	f, err := os.Create(filename)
@@ -86,12 +93,29 @@ func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender func(string
 // GenYamlCustom creates custom yaml output
 // nolint: gocyclo
 func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
-	cliDoc := cmdDoc{}
-	cliDoc.Name = cmd.CommandPath()
+	const (
+		// shortMaxWidth is the maximum width for the "Short" description before
+		// we force YAML to use multi-line syntax. The goal is to make the total
+		// width fit within 80 characters. This value is based on 80 characters
+		// minus the with of the field, colon, and whitespace ('short: ').
+		shortMaxWidth = 73
+
+		// longMaxWidth is the maximum width for the "Short" description before
+		// we force YAML to use multi-line syntax. The goal is to make the total
+		// width fit within 80 characters. This value is based on 80 characters
+		// minus the with of the field, colon, and whitespace ('long: ').
+		longMaxWidth = 74
+	)
+
+	cliDoc := cmdDoc{
+		Name:       cmd.CommandPath(),
+		Aliases:    strings.Join(cmd.Aliases, ", "),
+		Short:      forceMultiLine(cmd.Short, shortMaxWidth),
+		Long:       forceMultiLine(cmd.Long, longMaxWidth),
+		Example:    cmd.Example,
+		Deprecated: len(cmd.Deprecated) > 0,
+	}
 
-	cliDoc.Aliases = strings.Join(cmd.Aliases, ", ")
-	cliDoc.Short = cmd.Short
-	cliDoc.Long = cmd.Long
 	if len(cliDoc.Long) == 0 {
 		cliDoc.Long = cliDoc.Short
 	}
@@ -100,12 +124,6 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
 		cliDoc.Usage = cmd.UseLine()
 	}
 
-	if len(cmd.Example) > 0 {
-		cliDoc.Example = cmd.Example
-	}
-	if len(cmd.Deprecated) > 0 {
-		cliDoc.Deprecated = true
-	}
 	// Check recursively so that, e.g., `docker stack ls` returns the same output as `docker stack`
 	for curr := cmd; curr != nil; curr = curr.Parent() {
 		if v, ok := curr.Annotations["version"]; ok && cliDoc.MinAPIVersion == "" {
@@ -123,26 +141,32 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
 		if _, ok := curr.Annotations["swarm"]; ok && !cliDoc.Swarm {
 			cliDoc.Swarm = true
 		}
-		if os, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
-			cliDoc.OSType = os
+		if o, ok := curr.Annotations["ostype"]; ok && cliDoc.OSType == "" {
+			cliDoc.OSType = o
+		}
+	}
+
+	anchors := make(map[string]struct{})
+	if a, ok := cmd.Annotations["anchors"]; ok && a != "" {
+		for _, anchor := range strings.Split(a, ",") {
+			anchors[anchor] = struct{}{}
 		}
 	}
 
 	flags := cmd.NonInheritedFlags()
 	if flags.HasFlags() {
-		cliDoc.Options = genFlagResult(flags)
+		cliDoc.Options = genFlagResult(flags, anchors)
 	}
 	flags = cmd.InheritedFlags()
 	if flags.HasFlags() {
-		cliDoc.InheritedOptions = genFlagResult(flags)
+		cliDoc.InheritedOptions = genFlagResult(flags, anchors)
 	}
 
 	if hasSeeAlso(cmd) {
 		if cmd.HasParent() {
 			parent := cmd.Parent()
 			cliDoc.Pname = parent.CommandPath()
-			link := cliDoc.Pname + ".yaml"
-			cliDoc.Plink = strings.Replace(link, " ", "_", -1)
+			cliDoc.Plink = strings.Replace(cliDoc.Pname, " ", "_", -1) + ".yaml"
 			cmd.VisitParents(func(c *cobra.Command) {
 				if c.DisableAutoGenTag {
 					cmd.DisableAutoGenTag = c.DisableAutoGenTag
@@ -157,10 +181,8 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
 			if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() {
 				continue
 			}
-			currentChild := cliDoc.Name + " " + child.Name()
 			cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
-			link := currentChild + ".yaml"
-			cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1))
+			cliDoc.Clink = append(cliDoc.Clink, strings.Replace(cliDoc.Name+"_"+child.Name(), " ", "_", -1)+".yaml")
 		}
 	}
 
@@ -175,21 +197,41 @@ func GenYamlCustom(cmd *cobra.Command, w io.Writer) error {
 	return nil
 }
 
-func genFlagResult(flags *pflag.FlagSet) []cmdOption {
+func genFlagResult(flags *pflag.FlagSet, anchors map[string]struct{}) []cmdOption {
 	var (
 		result []cmdOption
 		opt    cmdOption
 	)
 
+	const (
+		// shortMaxWidth is the maximum width for the "Short" description before
+		// we force YAML to use multi-line syntax. The goal is to make the total
+		// width fit within 80 characters. This value is based on 80 characters
+		// minus the with of the field, colon, and whitespace ('  default_value: ').
+		defaultValueMaxWidth = 64
+
+		// longMaxWidth is the maximum width for the "Short" description before
+		// we force YAML to use multi-line syntax. The goal is to make the total
+		// width fit within 80 characters. This value is based on 80 characters
+		// minus the with of the field, colon, and whitespace ('  description: ').
+		descriptionMaxWidth = 66
+	)
+
 	flags.VisitAll(func(flag *pflag.Flag) {
 		opt = cmdOption{
 			Option:       flag.Name,
 			ValueType:    flag.Value.Type(),
-			DefaultValue: forceMultiLine(flag.DefValue),
-			Description:  forceMultiLine(flag.Usage),
+			DefaultValue: forceMultiLine(flag.DefValue, defaultValueMaxWidth),
+			Description:  forceMultiLine(flag.Usage, descriptionMaxWidth),
 			Deprecated:   len(flag.Deprecated) > 0,
 		}
 
+		if v, ok := flag.Annotations["docs.external.url"]; ok && len(v) > 0 {
+			opt.DetailsURL = strings.TrimPrefix(v[0], "https://docs.docker.com")
+		} else if _, ok = anchors[flag.Name]; ok {
+			opt.DetailsURL = "#" + flag.Name
+		}
+
 		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
 		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
 		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
@@ -230,10 +272,15 @@ func genFlagResult(flags *pflag.FlagSet) []cmdOption {
 	return result
 }
 
-// Temporary workaround for yaml lib generating incorrect yaml with long strings
-// that do not contain \n.
-func forceMultiLine(s string) string {
-	if len(s) > 60 && !strings.Contains(s, "\n") {
+// forceMultiLine appends a newline (\n) to strings that are longer than max
+// to force the yaml lib to use block notation (https://yaml.org/spec/1.2/spec.html#Block)
+// instead of a single-line string with newlines and tabs encoded("string\nline1\nline2").
+//
+// This makes the generated YAML more readable, and easier to review changes.
+// max can be used to customize the width to keep the whole line < 80 chars.
+func forceMultiLine(s string, max int) string {
+	s = strings.TrimSpace(s)
+	if len(s) > max && !strings.Contains(s, "\n") {
 		s = s + "\n"
 	}
 	return s
@@ -253,17 +300,30 @@ func hasSeeAlso(cmd *cobra.Command) bool {
 	return false
 }
 
-func parseMDContent(mdString string) (description string, examples string) {
-	parsedContent := strings.Split(mdString, "\n## ")
-	for _, s := range parsedContent {
-		if strings.Index(s, "Description") == 0 {
-			description = strings.TrimSpace(strings.TrimPrefix(s, "Description"))
+// applyDescriptionAndExamples fills in cmd.Long and cmd.Example with the
+// "Description" and "Examples" H2 sections in  mdString (if present).
+func applyDescriptionAndExamples(cmd *cobra.Command, mdString string) {
+	sections := getSections(mdString)
+	var (
+		anchors []string
+		md      string
+	)
+	if sections["description"] != "" {
+		md, anchors = cleanupMarkDown(sections["description"])
+		cmd.Long = md
+		anchors = append(anchors, md)
 	}
-		if strings.Index(s, "Examples") == 0 {
-			examples = strings.TrimSpace(strings.TrimPrefix(s, "Examples"))
+	if sections["examples"] != "" {
+		md, anchors = cleanupMarkDown(sections["examples"])
+		cmd.Example = md
+		anchors = append(anchors, md)
 	}
+	if len(anchors) > 0 {
+		if cmd.Annotations == nil {
+			cmd.Annotations = make(map[string]string)
+		}
+		cmd.Annotations["anchors"] = strings.Join(anchors, ",")
 	}
-	return description, examples
 }
 
 type byName []*cobra.Command

scripts/docs/generate-yaml.sh

@@ -4,5 +4,5 @@ set -eu -o pipefail
 
 mkdir -p docs/yaml/gen
 
-go build -o build/yaml-docs-generator github.com/docker/cli/docs/yaml
+GO111MODULE=off go build -o build/yaml-docs-generator github.com/docker/cli/docs/yaml
 build/yaml-docs-generator --root "$(pwd)" --target "$(pwd)/docs/yaml/gen"