qapi: add qapi2texi script

As the name suggests, the qapi2texi script converts JSON QAPI
description into a texi file suitable for different target
formats (info/man/txt/pdf/html...).

It parses the following kind of blocks:

Free-form:

  ##
  # = Section
  # == Subsection
  #
  # Some text foo with *emphasis*
  # 1. with a list
  # 2. like that
  #
  # And some code:
  # | $ echo foo
  # | -> do this
  # | <- get that
  #
  ##

Symbol description:

  ##
  # @symbol:
  #
  # Symbol body ditto ergo sum. Foo bar
  # baz ding.
  #
  # @param1: the frob to frobnicate
  # @param2: #optional how hard to frobnicate
  #
  # Returns: the frobnicated frob.
  #          If frob isn't frobnicatable, GenericError.
  #
  # Since: version
  # Notes: notes, comments can have
  #        - itemized list
  #        - like this
  #
  # Example:
  #
  # -> { "execute": "quit" }
  # <- { "return": {} }
  #
  ##

That's roughly following the following EBNF grammar:

api_comment = "##\n" comment "##\n"
comment = freeform_comment | symbol_comment
freeform_comment = { "# " text "\n" | "#\n" }
symbol_comment = "# @" name ":\n" { member | tag_section | freeform_comment }
member = "# @" name ':' [ text ] "\n" freeform_comment
tag_section = "# " ( "Returns:", "Since:", "Note:", "Notes:", "Example:", "Examples:" ) [ text ]  "\n" freeform_comment
text = free text with markup

Note that the grammar is ambiguous: a line "# @foo:\n" can be parsed
both as freeform_comment and as symbol_comment.  The actual parser
recognizes symbol_comment.

See docs/qapi-code-gen.txt for more details.

Deficiencies and limitations:
- the generated QMP documentation includes internal types
- union type support is lacking
- type information is lacking in generated documentation
- doc comment error message positions are imprecise, they point
  to the beginning of the comment.
- a few minor issues, all marked TODO/FIXME in the code

Signed-off-by: Marc-André Lureau <marcandre.lureau@redhat.com>
Message-Id: <20170113144135.5150-16-marcandre.lureau@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
[test-qapi.py tweaked to avoid trailing empty lines in .out]
Signed-off-by: Markus Armbruster <armbru@redhat.com>

scripts/qapi.py

@@ -125,6 +125,122 @@ class QAPISemError(QAPIError):
                            info['parent'], msg)
 
 
+class QAPIDoc(object):
+    class Section(object):
+        def __init__(self, name=None):
+            # optional section name (argument/member or section name)
+            self.name = name
+            # the list of lines for this section
+            self.content = []
+
+        def append(self, line):
+            self.content.append(line)
+
+        def __repr__(self):
+            return "\n".join(self.content).strip()
+
+    class ArgSection(Section):
+        pass
+
+    def __init__(self, parser, info):
+        # self.parser is used to report errors with QAPIParseError.  The
+        # resulting error position depends on the state of the parser.
+        # It happens to be the beginning of the comment.  More or less
+        # servicable, but action at a distance.
+        self.parser = parser
+        self.info = info
+        self.symbol = None
+        self.body = QAPIDoc.Section()
+        # dict mapping parameter name to ArgSection
+        self.args = OrderedDict()
+        # a list of Section
+        self.sections = []
+        # the current section
+        self.section = self.body
+        # associated expression (to be set by expression parser)
+        self.expr = None
+
+    def has_section(self, name):
+        """Return True if we have a section with this name."""
+        for i in self.sections:
+            if i.name == name:
+                return True
+        return False
+
+    def append(self, line):
+        """Parse a comment line and add it to the documentation."""
+        line = line[1:]
+        if not line:
+            self._append_freeform(line)
+            return
+
+        if line[0] != ' ':
+            raise QAPIParseError(self.parser, "Missing space after #")
+        line = line[1:]
+
+        # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
+        # recognized, and get silently treated as ordinary text
+        if self.symbol:
+            self._append_symbol_line(line)
+        elif not self.body.content and line.startswith("@"):
+            if not line.endswith(":"):
+                raise QAPIParseError(self.parser, "Line should end with :")
+            self.symbol = line[1:-1]
+            # FIXME invalid names other than the empty string aren't flagged
+            if not self.symbol:
+                raise QAPIParseError(self.parser, "Invalid name")
+        else:
+            self._append_freeform(line)
+
+    def _append_symbol_line(self, line):
+        name = line.split(' ', 1)[0]
+
+        if name.startswith("@") and name.endswith(":"):
+            line = line[len(name)+1:]
+            self._start_args_section(name[1:-1])
+        elif name in ("Returns:", "Since:",
+                      # those are often singular or plural
+                      "Note:", "Notes:",
+                      "Example:", "Examples:",
+                      "TODO:"):
+            line = line[len(name)+1:]
+            self._start_section(name[:-1])
+
+        self._append_freeform(line)
+
+    def _start_args_section(self, name):
+        # FIXME invalid names other than the empty string aren't flagged
+        if not name:
+            raise QAPIParseError(self.parser, "Invalid parameter name")
+        if name in self.args:
+            raise QAPIParseError(self.parser,
+                                 "'%s' parameter name duplicated" % name)
+        if self.sections:
+            raise QAPIParseError(self.parser,
+                                 "'@%s:' can't follow '%s' section"
+                                 % (name, self.sections[0].name))
+        self.section = QAPIDoc.ArgSection(name)
+        self.args[name] = self.section
+
+    def _start_section(self, name=""):
+        if name in ("Returns", "Since") and self.has_section(name):
+            raise QAPIParseError(self.parser,
+                                 "Duplicated '%s' section" % name)
+        self.section = QAPIDoc.Section(name)
+        self.sections.append(self.section)
+
+    def _append_freeform(self, line):
+        in_arg = isinstance(self.section, QAPIDoc.ArgSection)
+        if (in_arg and self.section.content
+                and not self.section.content[-1]
+                and line and not line[0].isspace()):
+            self._start_section()
+        if (in_arg or not self.section.name
+                or not self.section.name.startswith("Example")):
+            line = line.strip()
+        self.section.append(line)
+
+
 class QAPISchemaParser(object):
 
     def __init__(self, fp, previously_included=[], incl_info=None):
@@ -140,11 +256,17 @@ class QAPISchemaParser(object):
         self.line = 1
         self.line_pos = 0
         self.exprs = []
+        self.docs = []
         self.accept()
 
         while self.tok is not None:
             info = {'file': fname, 'line': self.line,
                     'parent': self.incl_info}
+            if self.tok == '#':
+                doc = self.get_doc(info)
+                self.docs.append(doc)
+                continue
+
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
@@ -162,6 +284,7 @@ class QAPISchemaParser(object):
                         raise QAPISemError(info, "Inclusion loop for %s"
                                            % include)
                     inf = inf['parent']
+
                 # skip multiple include of the same file
                 if incl_abs_fname in previously_included:
                     continue
@@ -172,12 +295,19 @@ class QAPISchemaParser(object):
                 exprs_include = QAPISchemaParser(fobj, previously_included,
                                                  info)
                 self.exprs.extend(exprs_include.exprs)
+                self.docs.extend(exprs_include.docs)
             else:
                 expr_elem = {'expr': expr,
                              'info': info}
+                if (self.docs
+                        and self.docs[-1].info['file'] == fname
+                        and not self.docs[-1].expr):
+                    self.docs[-1].expr = expr
+                    expr_elem['doc'] = self.docs[-1]
+
                 self.exprs.append(expr_elem)
 
-    def accept(self):
+    def accept(self, skip_comment=True):
         while True:
             self.tok = self.src[self.cursor]
             self.pos = self.cursor
@@ -185,7 +315,13 @@ class QAPISchemaParser(object):
             self.val = None
 
             if self.tok == '#':
+                if self.src[self.cursor] == '#':
+                    # Start of doc comment
+                    skip_comment = False
                 self.cursor = self.src.find('\n', self.cursor)
+                if not skip_comment:
+                    self.val = self.src[self.pos:self.cursor]
+                    return
             elif self.tok in "{}:,[]":
                 return
             elif self.tok == "'":
@@ -319,6 +455,28 @@ class QAPISchemaParser(object):
             raise QAPIParseError(self, 'Expected "{", "[" or string')
         return expr
 
+    def get_doc(self, info):
+        if self.val != '##':
+            raise QAPIParseError(self, "Junk after '##' at start of "
+                                 "documentation comment")
+
+        doc = QAPIDoc(self, info)
+        self.accept(False)
+        while self.tok == '#':
+            if self.val.startswith('##'):
+                # End of doc comment
+                if self.val != '##':
+                    raise QAPIParseError(self, "Junk after '##' at end of "
+                                         "documentation comment")
+                self.accept()
+                return doc
+            else:
+                doc.append(self.val)
+            self.accept(False)
+
+        raise QAPIParseError(self, "Documentation comment must end with '##'")
+
+
 #
 # Semantic analysis of schema expressions
 # TODO fold into QAPISchema
@@ -703,6 +861,11 @@ def check_exprs(exprs):
     for expr_elem in exprs:
         expr = expr_elem['expr']
         info = expr_elem['info']
+
+        if 'doc' not in expr_elem:
+            raise QAPISemError(info,
+                               "Expression missing documentation comment")
+
         if 'enum' in expr:
             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
             add_enum(expr['enum'], info, expr['data'])
@@ -761,6 +924,88 @@ def check_exprs(exprs):
     return exprs
 
 
+def check_freeform_doc(doc):
+    if doc.symbol:
+        raise QAPISemError(doc.info,
+                           "Documention for '%s' is not followed"
+                           " by the definition" % doc.symbol)
+
+    body = str(doc.body)
+    if re.search(r'@\S+:', body, re.MULTILINE):
+        raise QAPISemError(doc.info,
+                           "Free-form documentation block must not contain"
+                           " @NAME: sections")
+
+
+def check_definition_doc(doc, expr, info):
+    for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
+        if i in expr:
+            meta = i
+            break
+
+    name = expr[meta]
+    if doc.symbol != name:
+        raise QAPISemError(info, "Definition of '%s' follows documentation"
+                           " for '%s'" % (name, doc.symbol))
+    if doc.has_section('Returns') and 'command' not in expr:
+        raise QAPISemError(info, "'Returns:' is only valid for commands")
+
+    if meta == 'union':
+        args = expr.get('base', [])
+    else:
+        args = expr.get('data', [])
+    if isinstance(args, str):
+        return
+    if isinstance(args, dict):
+        args = args.keys()
+    assert isinstance(args, list)
+
+    if (meta == 'alternate'
+            or (meta == 'union' and not expr.get('discriminator'))):
+        args.append('type')
+
+    for arg in args:
+        if arg[0] == '*':
+            opt = True
+            desc = doc.args.get(arg[1:])
+        else:
+            opt = False
+            desc = doc.args.get(arg)
+        if not desc:
+            continue
+        desc_opt = "#optional" in str(desc)
+        if desc_opt and not opt:
+            raise QAPISemError(info, "Description has #optional, "
+                               "but the declaration doesn't")
+        if not desc_opt and opt:
+            # silently fix the doc
+            # TODO either fix the schema and make this an error,
+            # or drop #optional entirely
+            desc.append("#optional")
+
+    doc_args = set(doc.args.keys())
+    args = set([name.strip('*') for name in args])
+    if not doc_args.issubset(args):
+        raise QAPISemError(info, "The following documented members are not in "
+                           "the declaration: %s" % ", ".join(doc_args - args))
+
+
+def check_docs(docs):
+    for doc in docs:
+        for section in doc.args.values() + doc.sections:
+            content = str(section)
+            if not content or content.isspace():
+                raise QAPISemError(doc.info,
+                                   "Empty doc section '%s'" % section.name)
+
+        if not doc.expr:
+            check_freeform_doc(doc)
+        else:
+            check_definition_doc(doc, doc.expr, doc.info)
+
+    return docs
+
+
 #
 # Schema compiler frontend
 #
@@ -1229,7 +1474,9 @@ class QAPISchemaEvent(QAPISchemaEntity):
 class QAPISchema(object):
     def __init__(self, fname):
         try:
-            self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs)
+            parser = QAPISchemaParser(open(fname, "r"))
+            self.exprs = check_exprs(parser.exprs)
+            self.docs = check_docs(parser.docs)
             self._entity_dict = {}
             self._predefining = True
             self._def_predefineds()

scripts/qapi2texi.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python
+# QAPI texi generator
+#
+# This work is licensed under the terms of the GNU LGPL, version 2+.
+# See the COPYING file in the top-level directory.
+"""This script produces the documentation of a qapi schema in texinfo format"""
+import re
+import sys
+
+import qapi
+
+COMMAND_FMT = """
+@deftypefn {type} {{}} {name}
+
+{body}
+
+@end deftypefn
+
+""".format
+
+ENUM_FMT = """
+@deftp Enum {name}
+
+{body}
+
+@end deftp
+
+""".format
+
+STRUCT_FMT = """
+@deftp {{{type}}} {name}
+
+{body}
+
+@end deftp
+
+""".format
+
+EXAMPLE_FMT = """@example
+{code}
+@end example
+""".format
+
+
+def subst_strong(doc):
+    """Replaces *foo* by @strong{foo}"""
+    return re.sub(r'\*([^*\n]+)\*', r'@emph{\1}', doc)
+
+
+def subst_emph(doc):
+    """Replaces _foo_ by @emph{foo}"""
+    return re.sub(r'\b_([^_\n]+)_\b', r' @emph{\1} ', doc)
+
+
+def subst_vars(doc):
+    """Replaces @var by @code{var}"""
+    return re.sub(r'@([\w-]+)', r'@code{\1}', doc)
+
+
+def subst_braces(doc):
+    """Replaces {} with @{ @}"""
+    return doc.replace("{", "@{").replace("}", "@}")
+
+
+def texi_example(doc):
+    """Format @example"""
+    # TODO: Neglects to escape @ characters.
+    # We should probably escape them in subst_braces(), and rename the
+    # function to subst_special() or subs_texi_special().  If we do that, we
+    # need to delay it until after subst_vars() in texi_format().
+    doc = subst_braces(doc).strip('\n')
+    return EXAMPLE_FMT(code=doc)
+
+
+def texi_format(doc):
+    """
+    Format documentation
+
+    Lines starting with:
+    - |: generates an @example
+    - =: generates @section
+    - ==: generates @subsection
+    - 1. or 1): generates an @enumerate @item
+    - */-: generates an @itemize list
+    """
+    lines = []
+    doc = subst_braces(doc)
+    doc = subst_vars(doc)
+    doc = subst_emph(doc)
+    doc = subst_strong(doc)
+    inlist = ""
+    lastempty = False
+    for line in doc.split('\n'):
+        empty = line == ""
+
+        # FIXME: Doing this in a single if / elif chain is
+        # problematic.  For instance, a line without markup terminates
+        # a list if it follows a blank line (reaches the final elif),
+        # but a line with some *other* markup, such as a = title
+        # doesn't.
+        #
+        # Make sure to update section "Documentation markup" in
+        # docs/qapi-code-gen.txt when fixing this.
+        if line.startswith("| "):
+            line = EXAMPLE_FMT(code=line[2:])
+        elif line.startswith("= "):
+            line = "@section " + line[2:]
+        elif line.startswith("== "):
+            line = "@subsection " + line[3:]
+        elif re.match(r'^([0-9]*\.) ', line):
+            if not inlist:
+                lines.append("@enumerate")
+                inlist = "enumerate"
+            line = line[line.find(" ")+1:]
+            lines.append("@item")
+        elif re.match(r'^[*-] ', line):
+            if not inlist:
+                lines.append("@itemize %s" % {'*': "@bullet",
+                                              '-': "@minus"}[line[0]])
+                inlist = "itemize"
+            lines.append("@item")
+            line = line[2:]
+        elif lastempty and inlist:
+            lines.append("@end %s\n" % inlist)
+            inlist = ""
+
+        lastempty = empty
+        lines.append(line)
+
+    if inlist:
+        lines.append("@end %s\n" % inlist)
+    return "\n".join(lines)
+
+
+def texi_body(doc):
+    """
+    Format the body of a symbol documentation:
+    - main body
+    - table of arguments
+    - followed by "Returns/Notes/Since/Example" sections
+    """
+    body = texi_format(str(doc.body)) + "\n"
+    if doc.args:
+        body += "@table @asis\n"
+        for arg, section in doc.args.iteritems():
+            desc = str(section)
+            opt = ''
+            if "#optional" in desc:
+                desc = desc.replace("#optional", "")
+                opt = ' (optional)'
+            body += "@item @code{'%s'}%s\n%s\n" % (arg, opt,
+                                                   texi_format(desc))
+        body += "@end table\n"
+
+    for section in doc.sections:
+        name, doc = (section.name, str(section))
+        func = texi_format
+        if name.startswith("Example"):
+            func = texi_example
+
+        if name:
+            # FIXME the indentation produced by @quotation in .txt and
+            # .html output is confusing
+            body += "\n@quotation %s\n%s\n@end quotation" % \
+                    (name, func(doc))
+        else:
+            body += func(doc)
+
+    return body
+
+
+def texi_alternate(expr, doc):
+    """Format an alternate to texi"""
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Alternate",
+                      name=doc.symbol,
+                      body=body)
+
+
+def texi_union(expr, doc):
+    """Format a union to texi"""
+    discriminator = expr.get("discriminator")
+    if discriminator:
+        union = "Flat Union"
+    else:
+        union = "Simple Union"
+
+    body = texi_body(doc)
+    return STRUCT_FMT(type=union,
+                      name=doc.symbol,
+                      body=body)
+
+
+def texi_enum(expr, doc):
+    """Format an enum to texi"""
+    for i in expr['data']:
+        if i not in doc.args:
+            doc.args[i] = ''
+    body = texi_body(doc)
+    return ENUM_FMT(name=doc.symbol,
+                    body=body)
+
+
+def texi_struct(expr, doc):
+    """Format a struct to texi"""
+    body = texi_body(doc)
+    return STRUCT_FMT(type="Struct",
+                      name=doc.symbol,
+                      body=body)
+
+
+def texi_command(expr, doc):
+    """Format a command to texi"""
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Command",
+                       name=doc.symbol,
+                       body=body)
+
+
+def texi_event(expr, doc):
+    """Format an event to texi"""
+    body = texi_body(doc)
+    return COMMAND_FMT(type="Event",
+                       name=doc.symbol,
+                       body=body)
+
+
+def texi_expr(expr, doc):
+    """Format an expr to texi"""
+    (kind, _) = expr.items()[0]
+
+    fmt = {"command": texi_command,
+           "struct": texi_struct,
+           "enum": texi_enum,
+           "union": texi_union,
+           "alternate": texi_alternate,
+           "event": texi_event}[kind]
+
+    return fmt(expr, doc)
+
+
+def texi(docs):
+    """Convert QAPI schema expressions to texi documentation"""
+    res = []
+    for doc in docs:
+        expr = doc.expr
+        if not expr:
+            res.append(texi_body(doc))
+            continue
+        try:
+            doc = texi_expr(expr, doc)
+            res.append(doc)
+        except:
+            print >>sys.stderr, "error at @%s" % doc.info
+            raise
+
+    return '\n'.join(res)
+
+
+def main(argv):
+    """Takes schema argument, prints result to stdout"""
+    if len(argv) != 2:
+        print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0]
+        sys.exit(1)
+
+    schema = qapi.QAPISchema(argv[1])
+    print texi(schema.docs)
+
+
+if __name__ == "__main__":
+    main(sys.argv)