// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. package main import ( "bytes" "flag" "fmt" "go/ast" "go/parser" "go/token" "log" "maps" "os" "path/filepath" "reflect" "slices" "strings" "text/template" "github.com/siderolabs/gen/xslices" "gopkg.in/yaml.v3" "mvdan.cc/gofumpt/format" ) var tpl = `// This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. // Code generated by hack/docgen tool. DO NOT EDIT. package {{ .Package }} import ( "github.com/siderolabs/talos/pkg/machinery/config/encoder" ) {{ $tick := "` + "`" + `" -}} {{ range $struct := .Structs -}} func ({{ $struct.Name }}) Doc() *encoder.Doc { doc := &encoder.Doc{ Type : "{{ if $struct.Text.Alias }}{{ $struct.Text.Alias}}{{ else }}{{ $struct.Name }}{{ end }}", Comments: [3]string{ "" /* encoder.HeadComment */, "{{ $struct.Text.Comment }}" /* encoder.LineComment */, "" /* encoder.FootComment */}, Description: "{{ $struct.Text.Description }}", {{ if $struct.AppearsIn -}} AppearsIn: []encoder.Appearance{ {{ range $value := $struct.AppearsIn -}} { TypeName: "{{ $value.Struct.Name }}", FieldName: "{{ $value.FieldName }}", }, {{ end -}} }, {{ end -}} Fields: []encoder.Doc{ {{ range $index, $field := $struct.Fields -}} {{ if $field.Tag -}} { Name: "{{ $field.Tag }}", Type: "{{ $field.Type }}", Note: "{{ $field.Note }}", Description: "{{ $field.Text.Description }}", Comments: [3]string{ "" /* encoder.HeadComment */, "{{ $field.Text.Comment }}" /* encoder.LineComment */, "" /* encoder.FootComment */}, {{ if $field.Text.Values -}} Values : []string{ {{ range $value := $field.Text.Values -}} "{{ $value }}", {{ end -}} }, {{ end -}} }, {{ else -}} {}, {{- end }} {{- end }} }, } {{ range $example := $struct.Text.Examples }} {{ if $example.Value }} doc.AddExample("{{ $example.Name }}", {{ $example.Value }}) {{ end -}} {{ end }} {{ range $index, $field := $struct.Fields -}} {{ if $field.Tag -}} {{ if $field.Text.Examples -}} {{ range $example := $field.Text.Examples -}} {{- if $example.Value }} doc.Fields[{{ $index }}].AddExample("{{ $example.Name }}", {{ $example.Value }}) {{- end }} {{- end }} {{- end }} {{- end }} {{- end }} return doc } {{ end -}} // GetFileDoc returns documentation for the file {{ .File }}. func GetFileDoc() *encoder.FileDoc { return &encoder.FileDoc{ Name: "{{ .Name }}", Description: "{{ .Header }}", Structs: []*encoder.Doc{ {{ range $struct := .Structs -}} {{ $struct.Name }}{}.Doc(), {{ end -}} }, } } ` type Doc struct { Name string Package string Title string Header string File string Structs []*Struct } type Struct struct { Name string Text *Text Fields []*Field AppearsIn []Appearance } type Appearance struct { Struct *Struct FieldName string } type Example struct { Name string `yaml:"name"` Value string `yaml:"value"` } type Field struct { Name string Type string TypeRef string Text *Text Tag string Note string } type Text struct { Comment string `json:"-"` Description string `json:"description"` Examples []*Example `json:"examples"` Alias string `json:"alias"` Values []string `json:"values"` Schema *SchemaWrapper `json:"schema"` SchemaRoot bool `json:"schemaRoot" yaml:"schemaRoot"` SchemaRequired bool `json:"schemaRequired" yaml:"schemaRequired"` SchemaMeta string `json:"schemaMeta" yaml:"schemaMeta"` } func in(p string) (string, error) { return filepath.Abs(p) } func out(p string) (*os.File, error) { abs, err := filepath.Abs(p) if err != nil { return nil, err } return os.Create(abs) } type packageType struct { name string doc string file string structs []*structType } type structType struct { name string text *Text pos token.Pos node *ast.StructType } type aliasType struct { fieldType string fieldTypeRef string } func collectStructs(node ast.Node) ([]*structType, map[string]aliasType) { var structs []*structType aliases := map[string]aliasType{} collectStructs := func(n ast.Node) bool { g, ok := n.(*ast.GenDecl) if !ok { return true } isAlias := false if g.Doc != nil { for _, comment := range g.Doc.List { if strings.Contains(comment.Text, "docgen:nodoc") { return true } if strings.Contains(comment.Text, "docgen:alias") { isAlias = true } } } for _, spec := range g.Specs { t, ok := spec.(*ast.TypeSpec) if !ok { return true } if t.Type == nil { return true } x, ok := t.Type.(*ast.StructType) if !ok { if isAlias { aliases[t.Name.Name] = aliasType{ fieldType: formatFieldType(t.Type), fieldTypeRef: getFieldType(t.Type), } } return true } structName := t.Name.Name text := &Text{} if t.Doc != nil { text = parseComment([]byte(t.Doc.Text())) } else if g.Doc != nil { text = parseComment([]byte(g.Doc.Text())) } s := &structType{ name: structName, text: text, node: x, pos: x.Pos(), } structs = append(structs, s) } return true } ast.Inspect(node, collectStructs) return structs, aliases } func parseComment(comment []byte) *Text { text := &Text{} if err := yaml.Unmarshal(comment, text); err != nil { lines := strings.Split(string(comment), "\n") for i := range lines { lines[i] = strings.TrimLeft(lines[i], "\t") } // not yaml, fallback text.Description = strings.Join(lines, "\n") // take only the first line from the Description for the comment text.Comment = lines[0] // try to parse everything except for the first line as yaml if err = yaml.Unmarshal([]byte(strings.Join(lines[1:], "\n")), text); err == nil { // if parsed, remove it from the description text.Description = text.Comment } } else { text.Description = strings.TrimSpace(text.Description) // take only the first line from the Description for the comment text.Comment = strings.Split(text.Description, "\n")[0] } text.Comment = escape(text.Comment) text.Description = escape(text.Description) for _, example := range text.Examples { example.Name = escape(example.Name) example.Value = strings.TrimSpace(example.Value) } return text } func getFieldType(p any) string { if m, ok := p.(*ast.MapType); ok { return getFieldType(m.Value) } switch t := p.(type) { case *ast.Ident: return t.Name case *ast.ArrayType: return getFieldType(p.(*ast.ArrayType).Elt) case *ast.StarExpr: return getFieldType(t.X) case *ast.SelectorExpr: return getFieldType(t.Sel) default: return "" } } func formatFieldType(p any) string { if m, ok := p.(*ast.MapType); ok { return fmt.Sprintf("map[%s]%s", formatFieldType(m.Key), formatFieldType(m.Value)) } switch t := p.(type) { case *ast.Ident: return t.Name case *ast.ArrayType: return "[]" + formatFieldType(p.(*ast.ArrayType).Elt) case *ast.StructType: return "struct" case *ast.StarExpr: return formatFieldType(t.X) case *ast.SelectorExpr: return formatFieldType(t.Sel) default: log.Printf("unknown: %#v", t) return "" } } func escape(value string) string { value = strings.ReplaceAll(value, `"`, `\"`) value = strings.ReplaceAll(value, "\n", `\n`) return strings.TrimSpace(value) } func collectFields(s *structType, aliases map[string]aliasType) (fields []*Field) { fields = []*Field{} for _, f := range s.node.Fields.List { if f.Names == nil { // This is an embedded struct. fields = append(fields, &Field{Type: "unknown"}) continue } name := f.Names[0].Name if f.Doc == nil { log.Fatalf("field %q is missing a documentation", name) } if strings.Contains(f.Doc.Text(), "docgen:nodoc") { fields = append(fields, &Field{Type: "unknown"}) continue } if f.Tag == nil { log.Fatalf("field %q is missing a tag", name) } fieldType := formatFieldType(f.Type) if alias, ok := aliases[fieldType]; ok { fieldType = alias.fieldType } fieldTypeRef := getFieldType(f.Type) if alias, ok := aliases[fieldTypeRef]; ok { fieldTypeRef = alias.fieldTypeRef } tag := reflect.StructTag(strings.Trim(f.Tag.Value, "`")) yamlTag := tag.Get("yaml") yamlTag = strings.Split(yamlTag, ",")[0] if yamlTag == "" { log.Fatalf("field %q is missing a `yaml` tag or name in it", name) } text := parseComment([]byte(f.Doc.Text())) field := &Field{ Name: name, Tag: yamlTag, Type: fieldType, TypeRef: fieldTypeRef, Text: text, } if f.Comment != nil { field.Note = escape(f.Comment.Text()) } fields = append(fields, field) } return fields } func renderDoc(doc *Doc, dest string) { t := template.Must(template.New("docfile.tpl").Parse(tpl)) buf := bytes.Buffer{} err := t.Execute(&buf, doc) if err != nil { log.Fatalf("failed to render template: %v", err) } formatted, err := format.Source(buf.Bytes(), format.Options{}) if err != nil { log.Printf("data: %s", buf.Bytes()) log.Fatalf("failed to format source: %v", err) } out, err := out(dest) if err != nil { log.Fatalf("failed to create output file: %v", err) } defer out.Close() _, err = out.Write(formatted) if err != nil { log.Fatalf("failed to write output file: %v", err) } } func processFiles(inputFiles []string, outputFile, schemaOutputFile, versionTagFile string) { var packageNames []string packageNameToType := map[string]*packageType{} aliases := map[string]aliasType{} for _, inputFile := range inputFiles { abs, err := in(inputFile) if err != nil { log.Fatal(err) } log.Printf("creating package file set: %q", abs) fset := token.NewFileSet() node, err := parser.ParseFile(fset, abs, nil, parser.ParseComments) if err != nil { log.Fatal(err) } packageName := node.Name.Name if _, ok := packageNameToType[packageName]; !ok { packageNameToType[packageName] = &packageType{ name: packageName, file: outputFile, } packageNames = append(packageNames, packageName) } pkg := packageNameToType[packageName] if node.Doc != nil && node.Doc.Text() != "" { pkg.doc = node.Doc.Text() } tokenFile := fset.File(node.Pos()) if tokenFile == nil { log.Fatalf("No token") } log.Printf("parsing file in package %q: %s", packageName, tokenFile.Name()) fileStructs, fileAliases := collectStructs(node) pkg.structs = append(pkg.structs, fileStructs...) maps.Copy(aliases, fileAliases) } slices.Sort(packageNames) docs := xslices.Map(packageNames, func(name string) *Doc { return packageToDoc(packageNameToType[name], aliases) }) if schemaOutputFile != "" { renderSchema(docs, schemaOutputFile, versionTagFile) } if outputFile == "" { return } if len(docs) != 1 { log.Fatalf("expected exactly one package to generate docs, got %d", len(docs)) } renderDoc(docs[0], outputFile) } func packageToDoc(pkg *packageType, aliases map[string]aliasType) *Doc { if len(pkg.structs) == 0 { log.Fatalf("failed to find types that could be documented in %v", pkg.file) } doc := &Doc{ Package: pkg.name, Structs: []*Struct{}, } extraExamples := map[string][]*Example{} backReferences := map[string][]Appearance{} for _, s := range pkg.structs { log.Printf("generating docs for type: %q", s.name) fields := collectFields(s, aliases) s := &Struct{ Name: s.name, Text: s.text, Fields: fields, } for _, field := range fields { if field.TypeRef == "" { continue } if len(field.Text.Examples) > 0 { extraExamples[field.TypeRef] = append(extraExamples[field.TypeRef], field.Text.Examples...) } backReferences[field.TypeRef] = append(backReferences[field.TypeRef], Appearance{ Struct: s, FieldName: field.Tag, }) } doc.Structs = append(doc.Structs, s) } for _, s := range doc.Structs { if s.Text.Alias != "" { s.Text.Description = strings.ReplaceAll(s.Text.Description, s.Name, s.Text.Alias) s.Text.Comment = strings.ReplaceAll(s.Text.Comment, s.Name, s.Text.Alias) } if extra, ok := extraExamples[s.Name]; ok { s.Text.Examples = append(s.Text.Examples, extra...) } if ref, ok := backReferences[s.Name]; ok { s.AppearsIn = append(s.AppearsIn, ref...) } } doc.Package = pkg.name doc.Name = doc.Package doc.Header = escape(pkg.doc) doc.File = pkg.file return doc } func sourcesWithJSONSchema(dir string) []string { var sources []string if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if info.IsDir() || !strings.HasSuffix(info.Name(), ".go") { return nil } fileBytes, err := os.ReadFile(path) if err != nil { return err } if strings.Contains(string(fileBytes), "//docgen:jsonschema") { sources = append(sources, path) } return nil }); err != nil { log.Fatalf("failed to walk directory %q: %v", dir, err) } return sources } func determineFiles(generateSchemaFromDir string, args []string) []string { if generateSchemaFromDir != "" { if len(args) > 0 { log.Fatalf("cannot specify both -generate-schema-from-dir and input files as args") } files := sourcesWithJSONSchema(generateSchemaFromDir) if len(files) == 0 { log.Fatalf("no Go files annotated with //docgen:jsonschema found in %q", generateSchemaFromDir) } return files } if len(args) == 0 { log.Fatalf("no input files") } return args } func main() { outputFile := flag.String("output", "", "output file name") jsonSchemaOutputFile := flag.String("json-schema-output", "", "output file name for json schema") versionTagFile := flag.String("version-tag-file", "", "file name for version tag") generateSchemaFromDir := flag.String("generate-schema-from-dir", "", "generate a JSON schema by recursively parsing the sources in the specified directory") flag.Parse() files := determineFiles(*generateSchemaFromDir, flag.Args()) processFiles(files, *outputFile, *jsonSchemaOutputFile, *versionTagFile) }