# -*- coding: utf-8 -*- import ast import collections import io import token from tokenize import generate_tokens LITERAL_NAME = ('null', 'undefined', 'true', 'false') __all__ = ['tokenize', 'parse', 'iterate', 'Alt', 'Type', 'Literal'] class _Marker(object): __slots__ = ['name'] def __init__(self, name): self.name = name def __repr__(self): return '<%s>' % self.name OP = _Marker('OP') NAME = _Marker('NAME') LITERAL = _Marker('LITERAL') def tokenize(typespec): typespec = typespec.replace('$', 'jQuery') for toktype, string, _, _, _ in generate_tokens(io.StringIO(typespec).readline): if toktype == token.NAME: if string == 'or': # special case "A or B" # TODO: deprecation warning yield (OP, '|') else: yield (NAME, string) elif toktype == token.OP: if string in '|<>[].,': yield (OP, string) elif string == '>>': yield (OP, '>') yield (OP, '>') elif string == '*': # maybe? yield (NAME, 'any') elif string in '()': # TODO: deprecation warning # seems useless unless we add support for "..." continue else: raise ValueError("Unknown typespec operator %r" % string) elif toktype in (token.STRING, token.NUMBER): # enum-ish yield (LITERAL, ast.literal_eval(string)) elif toktype == token.ENDMARKER: return else: raise ValueError("Unknown typespec token %s" % token.tok_name[toktype]) Alt = collections.namedtuple('Alt', ['params']) Literal = collections.namedtuple('Literal', ['value']) Type = collections.namedtuple('Type', ['name', 'params']) def iterate(t, fortype, forliteral): if isinstance(t, Alt): ps = Peekable(t.params) for param in ps: for it in iterate(param, fortype, forliteral): yield it if ps.peek(None): yield forliteral('|') elif isinstance(t, Type): yield fortype(t.name) if not t.params: return yield forliteral('<') ps = Peekable(t.params) for param in ps: for it in iterate(param, fortype, forliteral): yield it if ps.peek(None): yield forliteral(',') yield forliteral('>') elif isinstance(t, Literal): yield forliteral(t.value) else: raise ValueError('Unknown item %s' % t) def parse(typespec): tokens = Peekable(tokenize(typespec)) return parse_alt(tokens) # alt = param # | param '|' alt def parse_alt(tokens): alt = Alt([]) while True: alt.params.append(parse_param(tokens)) if not next_is(tokens, (OP, '|')): break return alt # param = 'null' | 'undefined' | 'true' | 'false' # | number # | string # | typename '[]' # X '[' alt ']' # | typename '<' params '>' # params = alt | alt ',' params def parse_param(tokens): t, v = tokens.peek() if t == LITERAL or (t == NAME and v in LITERAL_NAME): next(tokens) return Literal(str(v)) # # [typespec] # should this be Array or an n-uple allowing multiple items? # if tok == (OP, '['): # next(tokens) # t = Type('Array', [parse_alt(tokens)]) # rest = next(tokens) # expect(rest, (OP, ']')) # return t t = Type(parse_typename(tokens), []) # type[] peek = tokens.peek(None) if peek == (OP, '['): next(tokens) # skip peeked expect(next(tokens), (OP, ']')) return Type('Array', [t]) # type if peek == (OP, '<'): next(tokens) # skip peeked while True: t.params.append(parse_alt(tokens)) n = next(tokens) if n == (OP, ','): continue if n == (OP, '>'): break raise ValueError("Expected OP ',' or OP ',', got %s '%s'" % n) return t # typename = name | name '.' typename def parse_typename(tokens): typename = [] while True: (t, n) = next(tokens) if t != NAME: raise ValueError("Expected a name token, got %s '%s'" % (t, n)) if n in LITERAL_NAME: raise ValueError("Expected a type name, got literal %s" % n) typename.append(n) if not next_is(tokens, (OP, '.')): break return '.'.join(typename) def next_is(tokens, expected): """ Consumes the next token if it's `expected` otherwise does not touch the tokens. Returns whether it consumed a token """ if tokens.peek(None) == expected: next(tokens) return True return False def expect(actual, expected): """ Raises ValueError if `actual` and `expected` are different :type actual: (object, str) :type expected: (object, str) """ if actual != expected: raise ValueError("Expected %s '%s', got %s '%s'" % (expected + actual)) NONE = object() class Peekable(object): __slots__ = ['_head', '_it'] def __init__(self, iterable): self._head = NONE self._it = iter(iterable) def __iter__(self): return self def __next__(self): if self._head is not NONE: r, self._head = self._head, NONE return r return next(self._it) next = __next__ def peek(self, default=NONE): if self._head is NONE: try: self._head = next(self._it) except StopIteration: if default is not NONE: return default raise return self._head