Implement generating overview page. (#30)

This change adds a new feature to generate an overview page for all of the rule
sets in the generated documentation. This is particularly useful for HTML
documentation and allows users to directly drop the generated HTML docs onto
a webserver and serve the docs as a standalone website.

This change also adds a link_ext attribute, which allows for customizing the
file extension used for links in the generated documentation since some
webservers serve Markdown pages with the .md file extension rather than .html.

Fixes #24

Author: davidzchen

skydoc/build.proto

@@ -94,6 +94,13 @@ message RuleDefinition {
 
   // The list of outputs for this rule.
   repeated OutputTarget output = 5;
+
+  enum Type {
+    RULE = 1;
+    MACRO = 2;
+    REPOSITORY_RULE = 3;
+  }
+  optional Type type = 6;
 }
 
 message BuildLanguage {

skydoc/macro_extractor.py

@@ -71,6 +71,7 @@ def _add_macro_doc(self, stmt):
 
     rule = self.__language.rule.add()
     rule.name = stmt.name
+    rule.type = build_pb2.RuleDefinition.MACRO
 
     doc = ast.get_docstring(stmt)
     if doc:

skydoc/macro_extractor_test.py

@@ -86,6 +86,7 @@ def multiline(name, foo=False, visibility=None):
             mandatory: false
             documentation: "The visibility of this rule.\\n\\nDocumentation for visibility continued here."
           }
+          type: MACRO
         }
         """)
 
@@ -115,6 +116,7 @@ def undocumented(name, visibility=None):
             type: UNKNOWN
             mandatory: false
           }
+          type: MACRO
         }
         """)
 
@@ -167,6 +169,7 @@ def public(name, visibility=None):
             mandatory: false
             documentation: "The visibility of this rule."
           }
+          type: MACRO
         }
         """)
 
@@ -230,6 +233,7 @@ def example_macro(name, foo, visibility=None):
             mandatory: false
             documentation: "The visibility of this rule."
           }
+          type: MACRO
         }
         """)
 
@@ -279,6 +283,7 @@ def macro_with_example(name, foo, visibility=None):
             mandatory: false
             documentation: "The visibility of this rule."
           }
+          type: MACRO
         }
         """)
 
@@ -338,6 +343,7 @@ def macro_with_outputs(name, foo, visibility=None):
             template: "%{name}.jar"
             documentation: "A Java archive."
           }
+          type: MACRO
         }
         """)
 

skydoc/main.py

@@ -48,6 +48,11 @@
     'generated in subdirectories that match the package structure of the '
     'input .bzl files. The prefix to strip must be common to all .bzl files; '
     'otherwise, skydoc will raise an error.')
+gflags.DEFINE_bool('overview', False, 'Whether to generate an overview page')
+gflags.DEFINE_string('overview_filename', 'index',
+    'The file name to use for the overview page.')
+gflags.DEFINE_string('link_ext', 'html',
+    'The file extension used for links in the generated documentation')
 
 FLAGS = gflags.FLAGS
 
@@ -60,12 +65,13 @@
 
 CSS_FILE = 'main.css'
 
-def _create_jinja_environment():
+def _create_jinja_environment(link_ext):
   env = jinja2.Environment(
       loader=jinja2.FileSystemLoader(_runfile_path(TEMPLATE_PATH)),
       keep_trailing_newline=True,
       line_statement_prefix='%')
   env.filters['markdown'] = lambda text: jinja2.Markup(mistune.markdown(text))
+  env.filters['link'] = lambda fname: '/' + fname + '.' + link_ext
   return env
 
 
@@ -112,14 +118,21 @@ def merge_languages(macro_language, rule_language):
     new_rule.CopyFrom(rule)
   return macro_language
 
+class WriterOptions(object):
+  def __init__(self, output_dir, output_file, output_zip, overview,
+               overview_filename, link_ext):
+    self.output_dir = output_dir
+    self.output_file = output_file
+    self.output_zip = output_zip
+    self.overview = overview
+    self.overview_filename = overview_filename
+    self.link_ext = link_ext
+
 class MarkdownWriter(object):
   """Writer for generating documentation in Markdown."""
 
-  def __init__(self, output_dir, output_file, output_zip, strip_prefix):
-    self.__output_dir = output_dir
-    self.__output_file = output_file
-    self.__output_zip = output_zip
-    self.__strip_prefix = strip_prefix
+  def __init__(self, writer_options):
+    self.__options = writer_options
 
   def write(self, rulesets):
     """Write the documentation for the rules contained in rulesets."""
@@ -129,19 +142,21 @@ def write(self, rulesets):
       for ruleset in rulesets:
         if len(ruleset.rules) > 0:
           output_files.append(self._write_ruleset(temp_dir, ruleset))
+      if self.__options.overview:
+        output_files.append(self._write_overview(temp_dir, rulesets))
 
-      if self.__output_zip:
+      if self.__options.output_zip:
         # We are generating a zip archive containing all the documentation.
         # Write each documentation file generated in the temp directory to the
         # zip file.
-        with zipfile.ZipFile(self.__output_file, 'w') as zf:
+        with zipfile.ZipFile(self.__options.output_file, 'w') as zf:
           for output_file, output_path in output_files:
             zf.write(output_file, output_path)
       else:
         # We are generating documentation in the output_dir directory. Copy each
         # documentation file to output_dir.
         for output_file, output_path in output_files:
-          dest_file = os.path.join(self.__output_dir, output_path)
+          dest_file = os.path.join(self.__options.output_dir, output_path)
           dest_dir = os.path.dirname(dest_file)
           if not os.path.exists(dest_dir):
             os.makedirs(dest_dir)
@@ -153,13 +168,13 @@ def write(self, rulesets):
 
   def _write_ruleset(self, output_dir, ruleset):
     # Load template and render Markdown.
-    env = _create_jinja_environment()
+    env = _create_jinja_environment(self.__options.link_ext)
     template = env.get_template('markdown.jinja')
     out = template.render(ruleset=ruleset)
 
     # Write output to file. Output files are created in a directory structure
     # that matches that of the input file.
-    output_path = ruleset.output_filename(self.__strip_prefix, 'md')
+    output_path = ruleset.output_file + '.md'
     output_file = "%s/%s" % (output_dir, output_path)
     file_dirname = os.path.dirname(output_file)
     if not os.path.exists(file_dirname):
@@ -168,57 +183,68 @@ def _write_ruleset(self, output_dir, ruleset):
       f.write(out)
     return (output_file, output_path)
 
+  def _write_overview(self, output_dir, rulesets):
+    template = self.__env.get_template('markdown_overview.jinja')
+    out = template.render(rulesets=rulesets)
+
+    output_file = "%s/%s.md" % (output_dir, self.options.overview_filename)
+    with open(output_file, "w") as f:
+      f.write(out)
+    return (output_file, "%s.md" % self.options.overview_filename)
+
 class HtmlWriter(object):
   """Writer for generating documentation in HTML."""
 
-  def __init__(self, output_dir, output_file, output_zip, strip_prefix):
-    self.__output_dir = output_dir
-    self.__output_file = output_file
-    self.__output_zip = output_zip
-    self.__strip_prefix = strip_prefix
-    self.__env = _create_jinja_environment()
+  def __init__(self, options):
+    self.__options = options
+    self.__env = _create_jinja_environment(self.__options.link_ext)
 
   def write(self, rulesets):
     # Generate navigation used for all rules.
     nav_template = self.__env.get_template('nav.jinja')
-    nav = nav_template.render(rulesets=rulesets)
+    nav = nav_template.render(
+        rulesets=rulesets,
+        overview=self.__options.overview,
+        overview_filename=self.__options.overview_filename)
 
     try:
       temp_dir = tempfile.mkdtemp()
       output_files = []
       for ruleset in rulesets:
         if len(ruleset.rules) > 0:
           output_files.append(self._write_ruleset(temp_dir, ruleset, nav))
+      if self.__options.overview:
+        output_files.append(self._write_overview(temp_dir, rulesets, nav))
 
-      if self.__output_zip:
-        with zipfile.ZipFile(self.__output_file, 'w') as zf:
+      if self.__options.output_zip:
+        with zipfile.ZipFile(self.__options.output_file, 'w') as zf:
           for output_file, output_path in output_files:
             zf.write(output_file, output_path)
           zf.write(os.path.join(_runfile_path(CSS_PATH), CSS_FILE),
                                 '%s' % CSS_FILE)
       else:
         for output_file, output_path in output_files:
-          dest_file = os.path.join(self.__output_dir, output_path)
+          dest_file = os.path.join(self.__options.output_dir, output_path)
           dest_dir = os.path.dirname(dest_file)
           if not os.path.exists(dest_dir):
             os.makedirs(dest_dir)
           shutil.copyfile(output_file, dest_file)
 
         # Copy CSS file.
         shutil.copyfile(os.path.join(_runfile_path(CSS_PATH), CSS_FILE),
-                        os.path.join(self.__output_dir, CSS_FILE))
+                        os.path.join(self.__options.output_dir, CSS_FILE))
     finally:
       # Delete temporary directory.
       shutil.rmtree(temp_dir)
 
   def _write_ruleset(self, output_dir, ruleset, nav):
     # Load template and render markdown.
     template = self.__env.get_template('html.jinja')
-    out = template.render(ruleset=ruleset, nav=nav)
+    out = template.render(title=ruleset.title, ruleset=ruleset, nav=nav)
 
     # Write output to file. Output files are created in a directory structure
     # that matches that of the input file.
-    output_path = ruleset.output_filename(self.__strip_prefix, 'html')
+    output_path = ruleset.output_file + '.html'
     output_file = "%s/%s" % (output_dir, output_path)
     file_dirname = os.path.dirname(output_file)
     if not os.path.exists(file_dirname):
@@ -227,6 +253,15 @@ def _write_ruleset(self, output_dir, ruleset, nav):
       f.write(out)
     return (output_file, output_path)
 
+  def _write_overview(self, output_dir, rulesets, nav):
+    template = self.__env.get_template('html_overview.jinja')
+    out = template.render(title='Overview', rulesets=rulesets, nav=nav)
+
+    output_file = "%s/%s.html" % (output_dir, self.__options.overview_filename)
+    with open(output_file, "w") as f:
+      f.write(out)
+    return (output_file, "%s.html" % self.__options.overview_filename)
+
 def main(argv):
   if FLAGS.output_dir and FLAGS.output_file:
     sys.stderr.write('Only one of --output_dir or --output_file can be set.')
@@ -252,17 +287,19 @@ def main(argv):
     rule_doc_extractor.parse_bzl(bzl_file)
     merged_language = merge_languages(macro_doc_extractor.proto(),
                                       rule_doc_extractor.proto())
-    rulesets.append(rule.RuleSet(bzl_file, merged_language,
-                                 macro_doc_extractor.title,
-                                 macro_doc_extractor.description))
-
+    rulesets.append(
+        rule.RuleSet(bzl_file, merged_language, macro_doc_extractor.title,
+                     macro_doc_extractor.description, strip_prefix,
+                     FLAGS.format))
+
+  writer_options = WriterOptions(
+      FLAGS.output_dir, FLAGS.output_file, FLAGS.zip, FLAGS.overview,
+      FLAGS.overview_filename, FLAGS.link_ext)
   if FLAGS.format == "markdown":
-    markdown_writer = MarkdownWriter(FLAGS.output_dir, FLAGS.output_file,
-                                     FLAGS.zip, strip_prefix)
+    markdown_writer = MarkdownWriter(writer_options)
     markdown_writer.write(rulesets)
   elif FLAGS.format == "html":
-    html_writer = HtmlWriter(FLAGS.output_dir, FLAGS.output_file, FLAGS.zip,
-                             strip_prefix)
+    html_writer = HtmlWriter(writer_options)
     html_writer.write(rulesets)
   else:
     sys.stderr.write(

skydoc/rule.py

@@ -105,6 +105,7 @@ class Rule(object):
   def __init__(self, proto):
     self.__proto = proto
     self.name = proto.name
+    self.type = proto.type
     self.documentation = proto.documentation
     self.example_documentation = proto.example_documentation
     self.signature = self._get_signature(proto)
@@ -115,6 +116,9 @@ def __init__(self, proto):
     for output in proto.output:
       self.outputs.append(Output(output))
 
+    parts = proto.documentation.split("\n\n")
+    self.short_documentation = parts[0]
+
   def _get_signature(self, proto):
     """Returns the rule signature for this rule."""
     signature = proto.name + '('
@@ -131,18 +135,33 @@ def _get_signature(self, proto):
 class RuleSet(object):
   """Representation of a rule set used to render documentation templates."""
 
-  def __init__(self, bzl_file, language, title, description):
+  def __init__(self, bzl_file, language, title, description, strip_prefix,
+               format):
     self.bzl_file = bzl_file
     file_basename = os.path.basename(bzl_file)
     self.name = file_basename.replace('.bzl', '')
     self.language = language
     self.title = title if title else "%s Rules" % self.name
     self.description = description
-    self.rules = []
-    for rule_proto in language.rule:
-      self.rules.append(Rule(rule_proto))
 
-  def output_filename(self, strip_prefix, file_ext):
+    # Generate output file name.
+    file_extension = 'html' if format == 'html' else 'md'
     assert self.bzl_file.startswith(strip_prefix)
-    output_path = self.bzl_file.replace('.bzl', '.%s' % file_ext)
-    return output_path[len(strip_prefix):]
+    output_path = self.bzl_file.replace('.bzl', '')
+    self.output_file = output_path[len(strip_prefix):]
+
+    # Populate all rules in this ruleset.
+    self.definitions = []
+    self.rules = []
+    self.repository_rules = []
+    self.macros = []
+    for rule_proto in language.rule:
+      definition = Rule(rule_proto)
+      self.definitions.append(definition)
+      if rule_proto.type == build_pb2.RuleDefinition.RULE:
+        self.rules.append(definition)
+      elif rule_proto.type == build_pb2.RuleDefinition.MACRO:
+        self.macros.append(definition)
+      else:
+        assert rule_proto.type == build_pb2.RuleDefinition.REPOSITORY_RULE
+        self.repository_rules.append(definition)

skydoc/rule_extractor.py

@@ -146,6 +146,10 @@ def _assemble_protos(self):
     for rule_desc in rules:
       rule = self.__language.rule.add()
       rule.name = rule_desc.name
+      if rule_desc.type == 'rule':
+        rule.type = build_pb2.RuleDefinition.RULE
+      else:
+        rule.type = build_pb2.RuleDefinition.REPOSITORY_RULE
       if rule_desc.doc:
         rule.documentation = rule_desc.doc
       if rule_desc.example_doc: