generate_json_docs.py

# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import cgi
import doctest
import inspect
import json
import os
import shutil
import types

import pdoc
from parinx.parser import parse_docstring
from parinx.errors import MethodParsingException
import six

from script_utils import PROJECT_ROOT
from verify_included_modules import get_public_modules


_DOCSTRING_TEST_PARSER = doctest.DocTestParser()


class Module(object):

    def __init__(self, module_id, name, description=None,
                 examples=None, methods=None, source=None):
        self.module_id = module_id
        self.name = name
        self.description = description
        self.examples = examples or []
        self.methods = methods
        self.source = source

    @classmethod
    def from_module_name(cls, name, base_path):
        module = pdoc.Module(pdoc.import_module(name), allsubmodules=True)
        methods = module.functions() + module.variables()
        examples = []

        if '__init__' in name:
            snippets = get_snippet_examples(name.split('.')[1],
                                            os.path.join(base_path, 'docs'))
            examples.extend(snippets)

        source_path = clean_source_path(module)

        return cls(module_id=name,
                   name=name.split('.')[-1].title(),
                   description=module.docstring,
                   examples=examples or [],
                   methods=[Method.from_pdoc(m) for m in methods],
                   source=source_path)

    def to_dict(self):
        return {'id': self.module_id,
                'name': self.name,
                'description': format_sphinx_doc(self.description),
                'examples': self.examples,
                'methods': [m.to_dict() for m in self.methods],
                'source': self.source}


class Klass(object):
    def __init__(self, module_id, name, refname=None, description=None,
                 examples=None, methods=None, source=None):
        self.module_id = module_id
        self.name = name
        self.refname = refname
        self.description = description
        self.examples = examples or []
        self.methods = methods
        self.source = source

    @classmethod
    def from_class_name(cls, module, kls):
        methods = kls.methods()

        examples = []
        source_path = clean_source_path(module)

        return cls(module_id=kls.name,
                   name=kls.name.split('.')[-1].title(),
                   refname=module.refname,
                   description=module.docstring,
                   examples=examples,
                   methods=[Method.from_pdoc(m) for m in methods],
                   source=source_path)

    def to_dict(self):
        return {'id': '%s.%s' % (self.refname, self.name.lower()),
                'name': self.name,
                'description': format_sphinx_doc(self.description),
                'examples': self.examples,
                'methods': [m.to_dict() for m in self.methods],
                'source': self.source}


class Method(object):

    def __init__(self, method_id, name, is_class, examples=None, params=None,
                 exceptions=None, returns=None, source=None):
        self.method_id = method_id
        self.name = name
        self.examples = examples or []
        self.params = params or []
        self.exceptions = exceptions or []
        self.returns = returns or []
        self.source = source or ''

        if is_class:
            self.type = 'constructor'
        else:
            self.type = 'instance'

    def add_param(self, param):
        self.params.append(param)

    def add_example(self, example):
        self.examples.append({'code': example})

    def add_source_line(self, source_line):
        self.source = source_line

    def set_returns(self, return_types):
        self.returns = [{'types': [return_types]}]

    def to_dict(self):
        return {'id': self.method_id,
                'name': self.name,
                'examples': self.examples,
                'source': self.source,
                'params': [p.to_dict() for p in self.params],
                'exceptions': self.exceptions,
                'returns': self.returns,
                'type': self.type}

    @classmethod
    def from_pdoc(cls, element):
        is_class = isinstance(element, pdoc.Class)
        method = cls(element.refname, element.name, is_class)
        components = element.refname.split('.')

        mod = __import__(components[0])
        for comp in components[1:]:
            mod = getattr(mod, comp)

        # Get method line number.
        method.add_source_line(get_source_line_number(mod))

        # Get method Examples.
        examples = get_examples_from_docstring(element.docstring)
        if examples:
            method.add_example(examples)

        if element.docstring:
            if not isinstance(element, pdoc.Class) and element.cls:
                klass = element.cls.cls
            elif element.cls:
                klass = element.cls
            else:
                klass = None

            # Hack for old-style classes
            if not str(klass).startswith('<'):
                klass = '<class \'%s\'>' % (klass,)

            try:
                method_info = parse_docstring(element.docstring, klass)
            except (MethodParsingException, IndexError):
                return method

            for name, data in method_info['arguments'].items():
                param = Param.from_docstring_section(name, data)
                method.add_param(param)

            if method_info.get('return'):
                if len(method_info['return']['type_name']) > 0:
                    type_name = method_info.get('return').get('type_name')

                    type_type = 'instance'
                    if any(x.isupper() for x in type_name):
                        type_type = 'constructor'

                    type_markup = build_link_from_type(type_name, type_type)

                    method.set_returns(type_markup)

        return method


class Param(object):

    def __init__(self, name, description=None, param_types=None, optional=None,
                 nullable=None):
        self.name = name
        self.description = description
        self.param_types = param_types or []
        self.optional = optional
        self.nullable = nullable

    def to_dict(self):
        return {'name': self.name,
                'description': process_words(self.description),
                'types': build_link_from_list_of_types(self.param_types),
                'optional': self.optional,
                'nullable': self.nullable}

    @classmethod
    def from_docstring_section(cls, name, data):
        param_types = build_link_from_type(data['type_name'])
        return cls(name=name, description=data['description'],
                   param_types=[param_types])


def clean_type_name(type_name):
    if type_name.lower().startswith('list of'):
        type_name = (type_name.replace('list of', '')
                     .replace('List of', ''))
    type_name = (type_name.replace('`', '').replace('class', '')
                 .replace(':', ''))
    return type_name


def build_link_from_list_of_types(type_names, object_type=None):
    processed_types = []
    for type_link in type_names:
        type_link = clean_type_name(type_link)
        processed_types.append(build_link_from_type(type_link, object_type))
    return processed_types


def build_link_from_type(type_name, object_type=None):
    type_name = clean_type_name(type_name)

    if not type_name.startswith('google.cloud'):
        return type_name
    doc_path = type_name

    doc_path = '/'.join(doc_path.split('.')).lower()

    type_markup = '<a data-custom-type="%s"' % doc_path
    if object_type == 'instance':
        type_markup += ' data-method="%s"' % type_name
    type_markup += '>%s</a>' % (type_name,)

    return type_markup


def build_source(module, method):
    if isinstance(module, (types.ModuleType, types.ClassType,
                           types.MethodType, types.FunctionType,
                           types.TracebackType, types.FrameType,
                           types.CodeType, types.TypeType)):

        _, line = inspect.getsourcelines(module)
        source_path = clean_source_path(module)

        if line:
            source_path = source_path + '#L' + str(line)

        method.add_source_line(source_path)

        # Sketchy get examples from method docstring.
        examples = []
        for example in examples:
            method.add_example(example)


def build_toc_entry(title, toc_type):
    return {
        'title': title,
        'type': toc_type
    }


def build_type(type_id, title, contents):
    return {
        'id': type_id,
        'title': title,
        'contents': contents
    }


def clean_source_path(module):
    if isinstance(module, six.string_types):
        source_id = module
    elif hasattr(module, 'refname'):
        source_id = module.refname
    else:
        source_id = inspect.getmodule(module).__name__
    return '%s.py' % (source_id.replace('.', '/'),)


def get_examples_from_docstring(doc_str):
    """Parse doctest style code examples from a docstring."""
    examples = _DOCSTRING_TEST_PARSER.get_examples(doc_str)
    example_str = ''
    for example in examples:
        example_str += '%s' % (example.source,)
        example_str += '%s' % (example.want,)

    return cgi.escape(example_str)


def get_source_line_number(module):
    if isinstance(module, (types.ModuleType, types.ClassType,
                           types.MethodType, types.FunctionType,
                           types.TracebackType, types.FrameType,
                           types.CodeType, types.TypeType)):

        _, line = inspect.getsourcelines(module)
        source_path = clean_source_path(module)

        if line:
            source_path = source_path + '#L' + str(line)
        return source_path


def process_code_blocks(doc):
    blocks = []
    index = 0

    for line in doc.splitlines(True):
        if len(blocks) - 1 < index:
            blocks.append('')

        if line.strip():
            blocks[index] += line
        else:
            index += 1

    formatted_blocks = []
    for block in blocks:
        is_code = False
        if block.splitlines()[0].startswith('  '):
            is_code = True

        if is_code:
            block = {'code': '<pre><code>%s</code></pre>' % (block,)}

        formatted_blocks.append(block)
    return formatted_blocks


def format_sphinx_doc(doc):
    doc = process_code_blocks(doc)
    example_lines = ""
    for line in doc:
        if isinstance(line, dict):
            line = line['code']
        else:
            line = process_words(line)
        example_lines = '%s\n%s' % (example_lines, line)
    return example_lines


def process_words(line):
    processed_line = ''
    for word in line.split():
        end_sentence = False
        if word.endswith('.'):
            end_sentence = True
            word = word[:-1]

        if word.startswith('``') and word.endswith('``'):
            word = word.replace('``', '')
            word = '<code>%s</code>' % (word,)

        if word.startswith('**') and word.endswith('**'):
            word = word.replace('**', '')
            word = '<b>%s</b>' % (word,)

        if word.startswith(':class:'):
            word = word.replace(':class:', '').replace('`', '')
            word = build_link_from_type(word)

        if word.startswith(':mod:'):
            word = word.replace(':mod:', '').replace('`', '')
            word = build_link_from_type(word)

        if word.startswith(':meth:'):
            word = word.replace(':meth:', '').replace('`', '')
            word = build_link_from_type(word)

        if word.startswith(':func:'):
            word = word.replace(':func:', '').replace('`', '')
            word = build_link_from_type(word)

        word = word.replace('`', '')

        if word.startswith('https://') or word.startswith('http://'):
            word = '<a href="%s">%s</a>' % (word, word)

        if end_sentence:
            word += '.'

        processed_line += ' %s' % (word,)

    processed_line = processed_line.replace('::', '')

    return processed_line


def write_docs_file(path, contents):
    if not os.path.exists(os.path.dirname(path)):
        try:
            os.makedirs(os.path.dirname(path))
        except OSError:
            raise
    with open(path, 'w') as output_file:
        output_file.write(contents)


def generate_doc_types_json(modules, types_file_path):
    doc_types_list = [{
        'id': 'google-cloud',
        'contents': 'index.json',
        'title': 'google-cloud'
    }]

    for module_name in modules:
        if module_name == 'google.cloud.__init__':
            continue

        module_title = module_name.replace('.__init__', '').split('.')
        module_contents = (module_name.replace('.', '/')
                           .replace('__init__', 'index'))

        if len(module_name.split('.')) > 2:
            module_id = module_name.replace('.', '/')
        else:
            module_id = (module_name.replace('.', '/'))

        module_contents += '.json'

        doc_type_object = build_type(module_id.replace('/__init__', ''),
                                     module_title, module_contents)

        doc_types_list.append(doc_type_object)

        pdoc_module = pdoc.Module(pdoc.import_module(module_name),
                                  allsubmodules=True)
        for c in pdoc_module.classes():
            generate_doc_types_classes_json(c, doc_types_list)

    write_docs_file(types_file_path,
                    json.dumps(doc_types_list))


def generate_doc_types_classes_json(klass, doc_types_list):

    module_contents = (klass.refname.replace('.', '/')
                       .replace('__init__', 'index'))
    module_contents += '.json'

    doc_type_object = build_type(klass.refname.lower().replace('.', '/'),
                                 klass.refname.split('.'),
                                 module_contents.lower())

    doc_types_list.append(doc_type_object)


def generate_module_docs(modules, docs_path, real_base_path, toc):
    for module_name in modules:
        module = Module.from_module_name(module_name, real_base_path)
        pdoc_module = pdoc.Module(pdoc.import_module(module_name),
                                  allsubmodules=True)
        for c in pdoc_module.classes():
            generate_class_docs(pdoc_module, c, docs_path, toc)

        module_path = (module_name.replace('.', '/')
                       .replace('__init__', 'index'))
        module_docs_path = os.path.join(docs_path, module_path) + '.json'

        if pdoc_module.functions():
            toc_key = module_name.replace('google.cloud.', '').split('.')[0]
            toc_entry = build_toc_entry(module.name, module_path)
            toc['services'][toc_key].append(toc_entry)

        write_docs_file(module_docs_path,
                        json.dumps(module.to_dict(),
                                   indent=2, sort_keys=True))


def generate_class_docs(module, klass, base_path, toc):
    kls = Klass.from_class_name(module, klass)

    module_path = (module.refname.replace('.', '/')
                   .replace('__init__', 'index'))
    module_docs_path = os.path.join(base_path, module_path,
                                    klass.name.lower()) + '.json'
    toc_key = module.name.replace('google.cloud.', '').split('.')[0]

    toc_entry = build_toc_entry(klass.name,
                                os.path.join(module_path,
                                             klass.name.lower()))

    toc['services'][toc_key].append(toc_entry)

    write_docs_file(module_docs_path,
                    json.dumps(kls.to_dict(),
                               indent=2, sort_keys=True))


def get_snippet_examples(module, json_docs_dir):
    snippets_file_path = os.path.join(json_docs_dir,
                                      module + '_snippets.py')
    usage_rst_path = os.path.join(json_docs_dir, module + '-usage.rst')

    snippet_labels = {}

    if os.path.isfile(usage_rst_path):
        with open(usage_rst_path, 'r') as snippet_labels_file:
            usage_rst = snippet_labels_file.read().splitlines()
            line_index = 0

            include_string = '.. literalinclude:: %s_snippets.py'
            for line in usage_rst:
                if line.startswith(include_string % module):
                    label_key = (usage_rst[line_index + 1]
                                 .replace('   :start-after: [START ', '')
                                 .replace(']', ''))
                    snippet_labels[label_key] = usage_rst[line_index - 2]
                line_index += 1

    snippets = []

    if os.path.isfile(snippets_file_path):
        with open(snippets_file_path, 'r') as snippets_file:
            snippets_string = snippets_file.read()
            label = None
            snippet = ''
            for line in snippets_string.splitlines(True):
                if line.strip().startswith('# [END'):
                    example = {
                        'caption': snippet_labels.get(label),
                        'code': snippet
                    }
                    snippets.append(example)

                    # Reset for next snippet
                    snippet = ''
                    label = None

                if label:
                    snippet += line

                if line.strip().startswith('# [START'):
                    label = (line.replace('# [START', '').replace(']', '')
                             .strip())
                    snippet = '# %s\n' % label
    return snippets


def package_files(generated_json_dir, docs_build_dir, static_json_dir,
                  tag='master'):
    """Copy app and JSON files into a convenient place to deploy from.

    Structure needs to be...
    root
        - src/
            - images/
            - app.js
            - app.css
            - vendor.js
            - vendor.css
        - json/
            - master/
                - toc.json
                - types.json
                - index.json
                - overview.html
            - home.html
        - index.html
        - manifest.json
    """
    package_path = os.path.join(docs_build_dir, 'json_build')
    shutil.rmtree(package_path, ignore_errors=True)

    shutil.copytree(static_json_dir, package_path)
    shutil.copytree(os.path.join(generated_json_dir, 'google', 'cloud'),
                    os.path.join(package_path, 'json', tag, 'google', 'cloud'))
    shutil.copyfile(os.path.join(generated_json_dir, 'types.json'),
                    os.path.join(package_path, 'json', tag, 'types.json'))


def main():
    parser = argparse.ArgumentParser(description='Document Python modules.')
    parser.add_argument('--tag', help='The version of the documentation.',
                        default='master')
    parser.add_argument('--basepath', help='Path to the library.',
                        default=PROJECT_ROOT)
    parser.add_argument('--show-toc', help='Prints partial table of contents',
                        default=False)
    args = parser.parse_args()

    toc = {
        'services': {
            '__init__': [],
            'google': [],
            'cloud': [],
            'bigquery': [],
            'bigtable': [],
            'client': [],
            'connection': [],
            'credentials': [],
            'datastore': [],
            'dns': [],
            'environment_vars': [],
            'error_reporting': [],
            'exceptions': [],
            'iterator': [],
            'language': [],
            'logging': [],
            'monitoring': [],
            'operation': [],
            'pubsub': [],
            'resource_manager': [],
            'storage': [],
            'streaming': [],
            'translate': [],
            'vision': [],
        }
    }

    BASE_JSON_DOCS_DIR = os.path.join(PROJECT_ROOT, 'docs', 'json')

    DOCS_BUILD_DIR = os.path.join(PROJECT_ROOT, 'docs', '_build')
    JSON_DOCS_DIR = os.path.join(DOCS_BUILD_DIR, 'json', args.tag)
    LIB_DIR = os.path.abspath(args.basepath)

    library_dir = os.path.join(LIB_DIR, 'google', 'cloud')
    public_mods = get_public_modules(library_dir,
                                     base_package='google.cloud')

    generate_module_docs(public_mods, JSON_DOCS_DIR, PROJECT_ROOT, toc)
    generate_doc_types_json(public_mods,
                            os.path.join(JSON_DOCS_DIR, 'types.json'))
    package_files(JSON_DOCS_DIR, DOCS_BUILD_DIR, BASE_JSON_DOCS_DIR)
    if args.show_toc:
        print(json.dumps(toc, indent=4))


if __name__ == '__main__':
    main()