flectra/doc/_extensions/autojsdoc/parser/parser.py
2018-02-11 23:55:56 -08:00

572 lines
20 KiB
Python

# -*- coding: utf-8 -*-
import collections
import pyjsdoc
from . import jsdoc
from . import utils
from .visitor import Visitor, SKIP
DECLARATOR_INIT_TO_REF = ('Literal', 'Identifier', 'MemberExpression')
class ModuleMatcher(Visitor):
"""Looks for structures of the form::
flectra.define($string, function ($name) {
These are *Flectra module definitions*, upon encountering one the
matcher:
* creates a module entry, optionally associated with the module comment
* spawns off a :class:`ModuleBodyMatcher` on the function's body with the
module name and the $name as "require" function
"""
def __init__(self, filename):
super(ModuleMatcher, self).__init__()
self.filename = filename
self.result = []
def enter_Program(self, node):
pass # allows visiting toplevel
def enter_ExpressionStatement(self, node):
# we're interested in expression statements (toplevel call)
if utils.match(node, {'expression': {
'callee': {
'object': {'name': 'flectra'},
'property': {'name': 'define'},
},
}}):
[module, func] = node['expression']['arguments']
mod = jsdoc.parse_comments(node.get('comments'), jsdoc.ModuleDoc)
# set module name
mod.set_name(module['value'])
mod.parsed['sourcefile'] = self.filename
self.result.append(mod)
# get name of require parameter
require = None # a module can have no dependencies
if func['params']:
require = func['params'][0]['name']
mod.parsed['dependency'], post = ModuleExtractor(mod, require).visit(func['body'])
mod._post_process.extend(post)
# don't recurse since we've fired off a sub-visitor for the
# bits we're interested in
return SKIP
def enter_generic(self, node):
# skip all other toplevel statements
return SKIP
ref = collections.namedtuple('Ref', 'object property')
def _name(r):
bits = []
while isinstance(r, ref):
bits.append(r.property)
r = r.object
return '.'.join(reversed(bits))
def deref(item, prop=None):
assert isinstance(item, ref)
while isinstance(item, ref):
obj = item.object
if isinstance(obj, ref):
obj = deref(obj)
if isinstance(obj, (jsdoc.NSDoc, jsdoc.Unknown)):
item = obj.get_property(item.property)
elif isinstance(obj, dict):
item = obj[item.property]
elif isinstance(obj, jsdoc.PropertyDoc):
# f'n dynamic crap
item = jsdoc.Unknown(obj.to_dict()).get_property(item.property)
else:
raise ValueError("%r (%s) should be a dict or namespace" % (obj, type(obj)))
return item
def m2r(me, scope):
# create a ref to the scope in case it's a hoisted function declaration,
# this may false-positive on other hoisting but w/e
if me['type'] == 'Literal':
return jsdoc.LiteralDoc({'value': me['value']})
if me['type'] == 'Identifier':
return ref(scope, me['name'])
if me['type'] != 'MemberExpression':
raise ValueError(me)
return ref(
m2r(me['object'], scope),
utils._value(me['property'], strict=True)
)
NOTHING = object()
class Declaration(object):
__slots__ = ['id', 'comments']
def __init__(self, id=None, comments=NOTHING):
self.id = id
self.comments = [] if comments is NOTHING else comments
class ModuleContent(object):
__slots__ = ['dependencies', 'post']
def __init__(self, dependencies=NOTHING, post=NOTHING):
self.dependencies = set() if dependencies is NOTHING else dependencies
self.post = [] if post is NOTHING else post
def __iter__(self):
yield self.dependencies
yield self.post
class Nothing(object):
def __init__(self, name):
self.name = name
def __bool__(self):
return False
__nonzero__ = __bool__
class RefProxy(object):
def __init__(self, r):
self._ref = r
def become(self, modules):
s = self
other = deref(self._ref) or Nothing(_name(self._ref))
# ??? shouldn't all previous refs have been resolved?
if isinstance(other, (RefProxy, ModuleProxy)):
other = other.become(modules)
self.__class__ = other.__class__
self.__dict__ = other.__dict__
return self
def set_name(self, name):
pass # ???
class ModuleProxy(object):
def __init__(self, name):
self._name = name
# replace the ModuleProxy by the module's exports
def become(self, modules):
s = self
m = modules[self._name].get_property('<exports>') or Nothing(self._name)
self.__class__ = m.__class__
self.__dict__ = m.__dict__
return self
def set_name(self, name):
pass # FIXME: ???
jq = jsdoc.UnknownNS({
'name': u'jQuery',
'doc': u'<jQuery>',
})
window = jsdoc.UnknownNS({
'doc': '<window>',
'name': 'window',
})
class BaseScope(collections.defaultdict):
""" The base scope assumes anything it's asked for is just an unknown
(global) namespace of some sort. Can hold a bunch of predefined params but
avoids the variables inference system blowing up when new (browser)
globals get used in module bodies.
"""
def __missing__(self, key):
it = jsdoc.UnknownNS({
'name': key,
'doc': u'<%s>' % key,
})
self[key] = it
return it
BASE_SCOPE = BaseScope(None, {
'_': jsdoc.UnknownNS({'doc': u'<underscore.js>', 'name': u'_'}),
'$': jq, 'jQuery': jq,
'window': window,
'document': window.get_property('document'),
'Date': jsdoc.ClassDoc({
'name': u'Date',
'doc': u'',
}),
'Backbone': jsdoc.UnknownNS({
'_members': [
('Model', jsdoc.ClassDoc({
'name': u'Model',
'doc': u'',
})),
('Collection', jsdoc.ClassDoc({
'name': u'Collection',
'doc': u'',
})),
]
}),
'flectra': jsdoc.UnknownNS({
'name': u'flectra',
'doc': u"Flectra",
'_members': [
('name', jsdoc.PropertyDoc({'name': u'csrf_token', 'type': u'{String}'})),
]
}),
'undefined': jsdoc.LiteralDoc({'name': u'undefined', 'value': None}),
})
class Scope(object):
"""
Add hoc scope versioning/SSA such that rebinding a symbol in a module
scope does not screw everything up e.g. "Foo = Foo.extend({})" should not
have the final Foo extending itself...
"""
def __init__(self, mapping):
self._namemap = self._empty(mapping)
self._targets = []
for k, v in mapping.items():
self[k] = v
@staticmethod
def _empty(mapping):
m = mapping.copy()
m.clear()
return m
def __setitem__(self, k, v):
self._namemap[k] = len(self._targets)
self._targets.append(v)
def freeze(self):
d = self._empty(self._namemap)
for k, v in self._namemap.items():
d[k] = self._targets[v]
return d
class ModuleExtractor(Visitor):
def __init__(self, module, requirefunc):
super(ModuleExtractor, self).__init__()
self.module = module
self.requirefunc = requirefunc
self.result = ModuleContent()
self.scope = Scope(BASE_SCOPE)
self.declaration = None
def enter_BlockStatement(self, node):
Hoistifier(self).visit(node)
def exit_BlockStatement(self, node):
for k, v in self.scope._namemap.items():
if k not in BASE_SCOPE:
self.module.add_member(k, self.scope._targets[v])
for t in TypedefMatcher(self).visit(node):
self.module.add_member(t.name, t)
def enter_VariableDeclaration(self, node):
self.declaration = Declaration(comments=node.get('comments'))
def enter_VariableDeclarator(self, node):
# otherwise we've already hoisted the declaration so the variable
# already exist initialised to undefined, and ValueExtractor does not
# handle a None input node so it returns None which makes no sense
if node['init']:
self.declaration.id = node['id']['name']
self.scope[self.declaration.id] = ValueExtractor(
self, self.declaration
).visit(node['init'] or [])
self.declaration.id = None
return SKIP
def exit_VariableDeclaration(self, node):
self.declaration = None
# as the name denotes, AssignmentExpression is an *expression*, which
# means at the module toplevel it is wrapped into an ExpressionStatement
# which is where the comments get attached
def enter_ExpressionStatement(self, node):
self.declaration = Declaration(comments=node.get('comments'))
def exit_ExpressionStatement(self, node):
self.declaration = None
def enter_AssignmentExpression(self, node):
target = node['left']
if target['type'] == 'Identifier':
self.declaration.id = target['name']
self.scope[self.declaration.id] = ValueExtractor(
self, self.declaration
).visit(node['right'])
self.declaration.id = None
return SKIP
if target['type'] != 'MemberExpression':
raise ValueError("Unhandled assign to %s" % target['type'])
# only assign to straight a.b.c patterns (OK and trivial literals)
if target['computed'] and target['property']['type'] != 'Literal':
return SKIP
name = utils._value(target['property'], strict=True)
if isinstance(name, type(u'')) and name.endswith('.extend'):
return SKIP # ignore overwrite of .extend (WTF)
self.declaration.id = name
it = ValueExtractor(
self, self.declaration
).visit(node['right'])
self.declaration.id = None
assert it, "assigned a non-value from %s to %s" % (node['right'], name)
@self.result.post.append
def _augment_module(modules):
try:
t = deref(m2r(target['object'], self.scope.freeze()))
except ValueError:
return # f'n extension of global libraries garbage
if not isinstance(t, jsdoc.NSDoc):
# function Foo(){}; Foo.prototype = bar
# fuck that yo
return
# TODO: note which module added this
m = it
if isinstance(m, jsdoc.LiteralDoc):
m = jsdoc.PropertyDoc(m.to_dict())
t.add_member(name, m)
return SKIP
def enter_FunctionDeclaration(self, node):
""" Already processed by hoistitifier
"""
return SKIP
def enter_ReturnStatement(self, node):
self.declaration = Declaration(comments=node.get('comments'))
if node['argument']:
export = ValueExtractor(self, self.declaration).visit(node['argument'])
if isinstance(export, RefProxy):
self.module.parsed['exports'] = _name(export._ref)
self.scope['<exports>'] = export
self.declaration = None
return SKIP
def enter_CallExpression(self, node):
if utils.match(node, {
'callee': {
'type': 'MemberExpression',
'object': lambda n: (
n['type'] in ('Identifier', 'MemberExpression')
# _.str.include
and not utils._name(n).startswith('_')
),
'property': {'name': 'include'},
}
}):
target = RefProxy(m2r(node['callee']['object'], self.scope.freeze()))
target_name = utils._name(node['callee']['object'])
items = ClassProcessor(self).visit(node['arguments'])
@self.result.post.append
def resolve_extension(modules):
t = target.become(modules)
if not isinstance(t, jsdoc.ClassDoc):
raise ValueError("include() subjects should be classes, %s is %s" % (target_name, type(t)))
# TODO: note which module added these
for it in items:
if isinstance(it, dict):
for n, member in it.items():
t.add_member(n, member)
else:
t.parsed.setdefault('mixes', []).append(it.become(modules))
return SKIP
def refify(self, node, also=None):
it = m2r(node, self.scope.freeze())
assert isinstance(it, ref), "Expected ref, got {}".format(it)
px = RefProxy(it)
@self.result.post.append
def resolve(modules):
p = px.become(modules)
if also: also(p)
return px
class ValueExtractor(Visitor):
def __init__(self, parent, declaration=None):
super(ValueExtractor, self).__init__()
self.parent = parent
self.declaration = declaration or Declaration()
def enter_generic(self, node):
self.result = jsdoc.parse_comments(
self.declaration.comments,
jsdoc.Unknown.from_(node['type'])
)
self._update_result_meta()
return SKIP
def _update_result_meta(self, name=None):
self.result.parsed['sourcemodule'] = self.parent.module
n = name or self.declaration.id
if n:
self.result.set_name(n)
def enter_Literal(self, node):
self.result = jsdoc.parse_comments(
self.declaration.comments, jsdoc.LiteralDoc)
self._update_result_meta()
self.result.parsed['value'] = node['value']
return SKIP
def enter_Identifier(self, node):
self.result = self.parent.refify(node)
return SKIP
def enter_MemberExpression(self, node):
self.result = RefProxy(ref(
ValueExtractor(self.parent).visit(node['object']),
utils._value(node['property'], strict=True)
))
self.parent.result.post.append(self.result.become)
return SKIP
def enter_FunctionExpression(self, node):
name, comments = (self.declaration.id, self.declaration.comments)
self.result = jsdoc.parse_comments(comments, jsdoc.FunctionDoc)
self.result.parsed['name'] = node['id'] and node['id']['name']
self._update_result_meta()
self.result.parsed['guessed_params'] = [p['name'] for p in node['params']]
return SKIP
def enter_NewExpression(self, node):
comments = self.declaration.comments if self.declaration else node.get('comments')
self.result = ns = jsdoc.parse_comments(comments, jsdoc.InstanceDoc)
self._update_result_meta()
def _update_contents(cls):
if not isinstance(cls, jsdoc.ClassDoc):
return
ns.parsed['cls'] = cls
self.parent.refify(node['callee'], also=_update_contents)
return SKIP
def enter_ObjectExpression(self, node):
self.result = obj = jsdoc.parse_comments(self.declaration.comments)
self._update_result_meta()
for n, p in MemberExtractor(parent=self.parent).visit(node['properties']).items():
obj.add_member(n, p)
return SKIP
def enter_CallExpression(self, node):
# require(a_module_name)
if utils.match(node, {'callee': {'type': 'Identifier', 'name': self.parent.requirefunc}}):
depname = node['arguments'][0]['value']
# TODO: clean this up
self.parent.result.dependencies.add(depname)
self.result = ModuleProxy(name=depname)
self.parent.result.post.append(self.result.become)
# Class.extend(..mixins, {})
elif utils.match(node, {
'callee': {
'type': 'MemberExpression',
'object': lambda n: (
n['type'] in ('Identifier', 'MemberExpression')
# $.extend, $.fn.extend, _.extend
and not utils._name(n).startswith(('_', '$'))
),
'property': {'name': 'extend'},
},
}): # creates a new class, but may not actually return it
obj = node['callee']['object']
comments = self.declaration.comments
self.result = cls = jsdoc.parse_comments(comments, jsdoc.ClassDoc)
cls.parsed['extends'] = self.parent.refify(obj)
self._update_result_meta()
items = ClassProcessor(self.parent).visit(node['arguments'])
@self.parent.result.post.append
def add_to_class(modules):
for item in items:
if isinstance(item, dict):
# methods/attributes
for n, method in item.items():
cls.add_member(n, method)
else:
cls.parsed.setdefault('mixes', []).append(item)
# other function calls
else:
self.result = jsdoc.parse_comments(self.declaration.comments, jsdoc.guess)
self._update_result_meta()
return SKIP
class ClassProcessor(Visitor):
def __init__(self, parent):
super(ClassProcessor, self).__init__()
self.result = []
self.parent = parent
def enter_generic(self, node):
self.result.append(self.parent.refify(node))
return SKIP
def enter_ObjectExpression(self, node):
self.result.append(MemberExtractor(parent=self.parent, for_class=True).visit(node['properties']))
return SKIP
String = type(u'')
class MemberExtractor(Visitor):
def __init__(self, parent, for_class=False):
super(MemberExtractor, self).__init__()
self.result = collections.OrderedDict()
self.parent = parent
self.for_class = for_class
def enter_Property(self, node):
name = utils._value(node['key'])
prop = ValueExtractor(
self.parent,
Declaration(id=name, comments=node.get('comments'))
).visit(node['value'])
if isinstance(prop, jsdoc.LiteralDoc):
prop = jsdoc.PropertyDoc(prop.to_dict())
# ValueExtractor can return a Ref, maybe this should be sent as part
# of the decl/comments?
if name.startswith('_') and hasattr(prop, 'parsed'):
prop.parsed['private'] = True
self.result[name] = prop
return SKIP
class Hoistifier(Visitor):
"""
Processor for variable and function declarations properly hoisting them
to the "top" of a module such that they are available with the relevant
value afterwards.
"""
def __init__(self, parent):
super(Hoistifier, self).__init__()
self.parent = parent
def enter_generic(self, node):
return SKIP
# nodes to straight recurse into, others are just skipped
enter_BlockStatement = enter_VariableDeclaration = lambda self, node: None
def enter_VariableDeclarator(self, node):
self.parent.scope[node['id']['name']] = BASE_SCOPE['undefined']
def enter_FunctionDeclaration(self, node):
funcname = node['id']['name']
self.parent.scope[funcname] = fn = jsdoc.parse_comments(
node.get('comments'),
jsdoc.FunctionDoc,
)
fn.parsed['sourcemodule'] = self.parent.module
fn.parsed['name'] = funcname
fn.parsed['guessed_params'] = [p['name'] for p in node['params']]
return SKIP
class TypedefMatcher(Visitor):
def __init__(self, parent):
super(TypedefMatcher, self).__init__()
self.parent = parent
self.result = []
enter_BlockStatement = lambda self, node: None
def enter_generic(self, node):
# just traverse all top-level statements, check their comments, and
# bail
for comment in node.get('comments') or []:
if '@typedef' in comment['value']:
extract = '\n' + jsdoc.strip_stars('/*' + comment['value'] + '\n*/')
parsed = pyjsdoc.parse_comment(extract, u'')
p = jsdoc.ParamDoc(parsed['typedef'])
parsed['name'] = p.name
parsed['sourcemodule'] = self.parent.module
# TODO: add p.type as superclass somehow? Builtin types not in scope :(
self.result.append(jsdoc.ClassDoc(parsed))
return SKIP