forked from STI/Astroport.ONE
575 lines
19 KiB
Python
575 lines
19 KiB
Python
from typing import List, NamedTuple, Optional
|
|
|
|
from ..error import GraphQLSyntaxError
|
|
from .ast import Token
|
|
from .block_string import dedent_block_string_lines
|
|
from .character_classes import is_digit, is_name_start, is_name_continue
|
|
from .source import Source
|
|
from .token_kind import TokenKind
|
|
|
|
__all__ = ["Lexer", "is_punctuator_token_kind"]
|
|
|
|
|
|
class EscapeSequence(NamedTuple):
|
|
"""The string value and lexed size of an escape sequence."""
|
|
|
|
value: str
|
|
size: int
|
|
|
|
|
|
class Lexer:
|
|
"""GraphQL Lexer
|
|
|
|
A Lexer is a stateful stream generator in that every time it is advanced, it returns
|
|
the next token in the Source. Assuming the source lexes, the final Token emitted by
|
|
the lexer will be of kind EOF, after which the lexer will repeatedly return the same
|
|
EOF token whenever called.
|
|
"""
|
|
|
|
def __init__(self, source: Source):
|
|
"""Given a Source object, initialize a Lexer for that source."""
|
|
self.source = source
|
|
self.token = self.last_token = Token(TokenKind.SOF, 0, 0, 0, 0)
|
|
self.line, self.line_start = 1, 0
|
|
|
|
def advance(self) -> Token:
|
|
"""Advance the token stream to the next non-ignored token."""
|
|
self.last_token = self.token
|
|
token = self.token = self.lookahead()
|
|
return token
|
|
|
|
def lookahead(self) -> Token:
|
|
"""Look ahead and return the next non-ignored token, but do not change state."""
|
|
token = self.token
|
|
if token.kind != TokenKind.EOF:
|
|
while True:
|
|
if token.next:
|
|
token = token.next
|
|
else:
|
|
# Read the next token and form a link in the token linked-list.
|
|
next_token = self.read_next_token(token.end)
|
|
token.next = next_token
|
|
next_token.prev = token
|
|
token = next_token
|
|
if token.kind != TokenKind.COMMENT:
|
|
break
|
|
return token
|
|
|
|
def print_code_point_at(self, location: int) -> str:
|
|
"""Print the code point at the given location.
|
|
|
|
Prints the code point (or end of file reference) at a given location in a
|
|
source for use in error messages.
|
|
|
|
Printable ASCII is printed quoted, while other points are printed in Unicode
|
|
code point form (ie. U+1234).
|
|
"""
|
|
body = self.source.body
|
|
if location >= len(body):
|
|
return TokenKind.EOF.value
|
|
char = body[location]
|
|
# Printable ASCII
|
|
if "\x20" <= char <= "\x7E":
|
|
return "'\"'" if char == '"' else f"'{char}'"
|
|
# Unicode code point
|
|
point = ord(
|
|
body[location : location + 2]
|
|
.encode("utf-16", "surrogatepass")
|
|
.decode("utf-16")
|
|
if is_supplementary_code_point(body, location)
|
|
else char
|
|
)
|
|
return f"U+{point:04X}"
|
|
|
|
def create_token(
|
|
self, kind: TokenKind, start: int, end: int, value: Optional[str] = None
|
|
) -> Token:
|
|
"""Create a token with line and column location information."""
|
|
line = self.line
|
|
col = 1 + start - self.line_start
|
|
return Token(kind, start, end, line, col, value)
|
|
|
|
def read_next_token(self, start: int) -> Token:
|
|
"""Get the next token from the source starting at the given position.
|
|
|
|
This skips over whitespace until it finds the next lexable token, then lexes
|
|
punctuators immediately or calls the appropriate helper function for more
|
|
complicated tokens.
|
|
"""
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
position = start
|
|
|
|
while position < body_length:
|
|
char = body[position] # SourceCharacter
|
|
|
|
if char in " \t,\ufeff":
|
|
position += 1
|
|
continue
|
|
elif char == "\n":
|
|
position += 1
|
|
self.line += 1
|
|
self.line_start = position
|
|
continue
|
|
elif char == "\r":
|
|
if body[position + 1 : position + 2] == "\n":
|
|
position += 2
|
|
else:
|
|
position += 1
|
|
self.line += 1
|
|
self.line_start = position
|
|
continue
|
|
|
|
if char == "#":
|
|
return self.read_comment(position)
|
|
|
|
if char == '"':
|
|
if body[position + 1 : position + 3] == '""':
|
|
return self.read_block_string(position)
|
|
return self.read_string(position)
|
|
|
|
kind = _KIND_FOR_PUNCT.get(char)
|
|
if kind:
|
|
return self.create_token(kind, position, position + 1)
|
|
|
|
if is_digit(char) or char == "-":
|
|
return self.read_number(position, char)
|
|
|
|
if is_name_start(char):
|
|
return self.read_name(position)
|
|
|
|
if char == ".":
|
|
if body[position + 1 : position + 3] == "..":
|
|
return self.create_token(TokenKind.SPREAD, position, position + 3)
|
|
|
|
message = (
|
|
"Unexpected single quote character ('),"
|
|
' did you mean to use a double quote (")?'
|
|
if char == "'"
|
|
else (
|
|
f"Unexpected character: {self.print_code_point_at(position)}."
|
|
if is_unicode_scalar_value(char)
|
|
or is_supplementary_code_point(body, position)
|
|
else f"Invalid character: {self.print_code_point_at(position)}."
|
|
)
|
|
)
|
|
|
|
raise GraphQLSyntaxError(self.source, position, message)
|
|
|
|
return self.create_token(TokenKind.EOF, body_length, body_length)
|
|
|
|
def read_comment(self, start: int) -> Token:
|
|
"""Read a comment token from the source file."""
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
|
|
position = start + 1
|
|
while position < body_length:
|
|
char = body[position]
|
|
if char in "\r\n":
|
|
break
|
|
if is_unicode_scalar_value(char):
|
|
position += 1
|
|
elif is_supplementary_code_point(body, position):
|
|
position += 2
|
|
else:
|
|
break # pragma: no cover
|
|
|
|
return self.create_token(
|
|
TokenKind.COMMENT,
|
|
start,
|
|
position,
|
|
body[start + 1 : position],
|
|
)
|
|
|
|
def read_number(self, start: int, first_char: str) -> Token:
|
|
"""Reads a number token from the source file.
|
|
|
|
This can be either a FloatValue or an IntValue,
|
|
depending on whether a FractionalPart or ExponentPart is encountered.
|
|
"""
|
|
body = self.source.body
|
|
position = start
|
|
char = first_char
|
|
is_float = False
|
|
|
|
if char == "-":
|
|
position += 1
|
|
char = body[position : position + 1]
|
|
if char == "0":
|
|
position += 1
|
|
char = body[position : position + 1]
|
|
if is_digit(char):
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
"Invalid number, unexpected digit after 0:"
|
|
f" {self.print_code_point_at(position)}.",
|
|
)
|
|
else:
|
|
position = self.read_digits(position, char)
|
|
char = body[position : position + 1]
|
|
if char == ".":
|
|
is_float = True
|
|
position += 1
|
|
char = body[position : position + 1]
|
|
position = self.read_digits(position, char)
|
|
char = body[position : position + 1]
|
|
if char and char in "Ee":
|
|
is_float = True
|
|
position += 1
|
|
char = body[position : position + 1]
|
|
if char and char in "+-":
|
|
position += 1
|
|
char = body[position : position + 1]
|
|
position = self.read_digits(position, char)
|
|
char = body[position : position + 1]
|
|
|
|
# Numbers cannot be followed by . or NameStart
|
|
if char and (char == "." or is_name_start(char)):
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
"Invalid number, expected digit but got:"
|
|
f" {self.print_code_point_at(position)}.",
|
|
)
|
|
|
|
return self.create_token(
|
|
TokenKind.FLOAT if is_float else TokenKind.INT,
|
|
start,
|
|
position,
|
|
body[start:position],
|
|
)
|
|
|
|
def read_digits(self, start: int, first_char: str) -> int:
|
|
"""Return the new position in the source after reading one or more digits."""
|
|
if not is_digit(first_char):
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
start,
|
|
"Invalid number, expected digit but got:"
|
|
f" {self.print_code_point_at(start)}.",
|
|
)
|
|
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
position = start + 1
|
|
while position < body_length and is_digit(body[position]):
|
|
position += 1
|
|
return position
|
|
|
|
def read_string(self, start: int) -> Token:
|
|
"""Read a single-quote string token from the source file."""
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
position = start + 1
|
|
chunk_start = position
|
|
value: List[str] = []
|
|
append = value.append
|
|
|
|
while position < body_length:
|
|
char = body[position]
|
|
|
|
if char == '"':
|
|
append(body[chunk_start:position])
|
|
return self.create_token(
|
|
TokenKind.STRING,
|
|
start,
|
|
position + 1,
|
|
"".join(value),
|
|
)
|
|
|
|
if char == "\\":
|
|
append(body[chunk_start:position])
|
|
escape = (
|
|
(
|
|
self.read_escaped_unicode_variable_width(position)
|
|
if body[position + 2 : position + 3] == "{"
|
|
else self.read_escaped_unicode_fixed_width(position)
|
|
)
|
|
if body[position + 1 : position + 2] == "u"
|
|
else self.read_escaped_character(position)
|
|
)
|
|
append(escape.value)
|
|
position += escape.size
|
|
chunk_start = position
|
|
continue
|
|
|
|
if char in "\r\n":
|
|
break
|
|
|
|
if is_unicode_scalar_value(char):
|
|
position += 1
|
|
elif is_supplementary_code_point(body, position):
|
|
position += 2
|
|
else:
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
"Invalid character within String:"
|
|
f" {self.print_code_point_at(position)}.",
|
|
)
|
|
|
|
raise GraphQLSyntaxError(self.source, position, "Unterminated string.")
|
|
|
|
def read_escaped_unicode_variable_width(self, position: int) -> EscapeSequence:
|
|
body = self.source.body
|
|
point = 0
|
|
size = 3
|
|
max_size = min(12, len(body) - position)
|
|
# Cannot be larger than 12 chars (\u{00000000}).
|
|
while size < max_size:
|
|
char = body[position + size]
|
|
size += 1
|
|
if char == "}":
|
|
# Must be at least 5 chars (\u{0}) and encode a Unicode scalar value.
|
|
if size < 5 or not (
|
|
0 <= point <= 0xD7FF or 0xE000 <= point <= 0x10FFFF
|
|
):
|
|
break
|
|
return EscapeSequence(chr(point), size)
|
|
# Append this hex digit to the code point.
|
|
point = (point << 4) | read_hex_digit(char)
|
|
if point < 0:
|
|
break
|
|
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
f"Invalid Unicode escape sequence: '{body[position: position + size]}'.",
|
|
)
|
|
|
|
def read_escaped_unicode_fixed_width(self, position: int) -> EscapeSequence:
|
|
body = self.source.body
|
|
code = read_16_bit_hex_code(body, position + 2)
|
|
|
|
if 0 <= code <= 0xD7FF or 0xE000 <= code <= 0x10FFFF:
|
|
return EscapeSequence(chr(code), 6)
|
|
|
|
# GraphQL allows JSON-style surrogate pair escape sequences, but only when
|
|
# a valid pair is formed.
|
|
if 0xD800 <= code <= 0xDBFF:
|
|
if body[position + 6 : position + 8] == "\\u":
|
|
trailing_code = read_16_bit_hex_code(body, position + 8)
|
|
if 0xDC00 <= trailing_code <= 0xDFFF:
|
|
return EscapeSequence(
|
|
(chr(code) + chr(trailing_code))
|
|
.encode("utf-16", "surrogatepass")
|
|
.decode("utf-16"),
|
|
12,
|
|
)
|
|
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
f"Invalid Unicode escape sequence: '{body[position: position + 6]}'.",
|
|
)
|
|
|
|
def read_escaped_character(self, position: int) -> EscapeSequence:
|
|
body = self.source.body
|
|
value = _ESCAPED_CHARS.get(body[position + 1])
|
|
if value:
|
|
return EscapeSequence(value, 2)
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
f"Invalid character escape sequence: '{body[position: position + 2]}'.",
|
|
)
|
|
|
|
def read_block_string(self, start: int) -> Token:
|
|
"""Read a block string token from the source file."""
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
line_start = self.line_start
|
|
|
|
position = start + 3
|
|
chunk_start = position
|
|
current_line = ""
|
|
|
|
block_lines = []
|
|
while position < body_length:
|
|
char = body[position]
|
|
|
|
if char == '"' and body[position + 1 : position + 3] == '""':
|
|
current_line += body[chunk_start:position]
|
|
block_lines.append(current_line)
|
|
|
|
token = self.create_token(
|
|
TokenKind.BLOCK_STRING,
|
|
start,
|
|
position + 3,
|
|
# return a string of the lines joined with new lines
|
|
"\n".join(dedent_block_string_lines(block_lines)),
|
|
)
|
|
|
|
self.line += len(block_lines) - 1
|
|
self.line_start = line_start
|
|
return token
|
|
|
|
if char == "\\" and body[position + 1 : position + 4] == '"""':
|
|
current_line += body[chunk_start:position]
|
|
chunk_start = position + 1 # skip only slash
|
|
position += 4
|
|
continue
|
|
|
|
if char in "\r\n":
|
|
current_line += body[chunk_start:position]
|
|
block_lines.append(current_line)
|
|
|
|
if char == "\r" and body[position + 1 : position + 2] == "\n":
|
|
position += 2
|
|
else:
|
|
position += 1
|
|
|
|
current_line = ""
|
|
chunk_start = line_start = position
|
|
continue
|
|
|
|
if is_unicode_scalar_value(char):
|
|
position += 1
|
|
elif is_supplementary_code_point(body, position):
|
|
position += 2
|
|
else:
|
|
raise GraphQLSyntaxError(
|
|
self.source,
|
|
position,
|
|
"Invalid character within String:"
|
|
f" {self.print_code_point_at(position)}.",
|
|
)
|
|
|
|
raise GraphQLSyntaxError(self.source, position, "Unterminated string.")
|
|
|
|
def read_name(self, start: int) -> Token:
|
|
"""Read an alphanumeric + underscore name from the source."""
|
|
body = self.source.body
|
|
body_length = len(body)
|
|
position = start + 1
|
|
|
|
while position < body_length:
|
|
char = body[position]
|
|
if not is_name_continue(char):
|
|
break
|
|
position += 1
|
|
|
|
return self.create_token(TokenKind.NAME, start, position, body[start:position])
|
|
|
|
|
|
_punctuator_token_kinds = frozenset(
|
|
[
|
|
TokenKind.BANG,
|
|
TokenKind.DOLLAR,
|
|
TokenKind.AMP,
|
|
TokenKind.PAREN_L,
|
|
TokenKind.PAREN_R,
|
|
TokenKind.SPREAD,
|
|
TokenKind.COLON,
|
|
TokenKind.EQUALS,
|
|
TokenKind.AT,
|
|
TokenKind.BRACKET_L,
|
|
TokenKind.BRACKET_R,
|
|
TokenKind.BRACE_L,
|
|
TokenKind.PIPE,
|
|
TokenKind.BRACE_R,
|
|
]
|
|
)
|
|
|
|
|
|
def is_punctuator_token_kind(kind: TokenKind) -> bool:
|
|
"""Check whether the given token kind corresponds to a punctuator.
|
|
|
|
For internal use only.
|
|
"""
|
|
return kind in _punctuator_token_kinds
|
|
|
|
|
|
_KIND_FOR_PUNCT = {
|
|
"!": TokenKind.BANG,
|
|
"$": TokenKind.DOLLAR,
|
|
"&": TokenKind.AMP,
|
|
"(": TokenKind.PAREN_L,
|
|
")": TokenKind.PAREN_R,
|
|
":": TokenKind.COLON,
|
|
"=": TokenKind.EQUALS,
|
|
"@": TokenKind.AT,
|
|
"[": TokenKind.BRACKET_L,
|
|
"]": TokenKind.BRACKET_R,
|
|
"{": TokenKind.BRACE_L,
|
|
"}": TokenKind.BRACE_R,
|
|
"|": TokenKind.PIPE,
|
|
}
|
|
|
|
|
|
_ESCAPED_CHARS = {
|
|
'"': '"',
|
|
"/": "/",
|
|
"\\": "\\",
|
|
"b": "\b",
|
|
"f": "\f",
|
|
"n": "\n",
|
|
"r": "\r",
|
|
"t": "\t",
|
|
}
|
|
|
|
|
|
def read_16_bit_hex_code(body: str, position: int) -> int:
|
|
"""Read a 16bit hexadecimal string and return its positive integer value (0-65535).
|
|
|
|
Reads four hexadecimal characters and returns the positive integer that 16bit
|
|
hexadecimal string represents. For example, "000f" will return 15, and "dead"
|
|
will return 57005.
|
|
|
|
Returns a negative number if any char was not a valid hexadecimal digit.
|
|
"""
|
|
# read_hex_digit() returns -1 on error. ORing a negative value with any other
|
|
# value always produces a negative value.
|
|
return (
|
|
read_hex_digit(body[position]) << 12
|
|
| read_hex_digit(body[position + 1]) << 8
|
|
| read_hex_digit(body[position + 2]) << 4
|
|
| read_hex_digit(body[position + 3])
|
|
)
|
|
|
|
|
|
def read_hex_digit(char: str) -> int:
|
|
"""Read a hexadecimal character and returns its positive integer value (0-15).
|
|
|
|
'0' becomes 0, '9' becomes 9
|
|
'A' becomes 10, 'F' becomes 15
|
|
'a' becomes 10, 'f' becomes 15
|
|
|
|
Returns -1 if the provided character code was not a valid hexadecimal digit.
|
|
"""
|
|
if "0" <= char <= "9":
|
|
return ord(char) - 48
|
|
elif "A" <= char <= "F":
|
|
return ord(char) - 55
|
|
elif "a" <= char <= "f":
|
|
return ord(char) - 87
|
|
return -1
|
|
|
|
|
|
def is_unicode_scalar_value(char: str) -> bool:
|
|
"""Check whether this is a Unicode scalar value.
|
|
|
|
A Unicode scalar value is any Unicode code point except surrogate code
|
|
points. In other words, the inclusive ranges of values 0x0000 to 0xD7FF and
|
|
0xE000 to 0x10FFFF.
|
|
"""
|
|
return "\x00" <= char <= "\ud7ff" or "\ue000" <= char <= "\U0010ffff"
|
|
|
|
|
|
def is_supplementary_code_point(body: str, location: int) -> bool:
|
|
"""
|
|
Check whether the current location is a supplementary code point.
|
|
|
|
The GraphQL specification defines source text as a sequence of unicode scalar
|
|
values (which Unicode defines to exclude surrogate code points).
|
|
"""
|
|
try:
|
|
return (
|
|
"\ud800" <= body[location] <= "\udbff"
|
|
and "\udc00" <= body[location + 1] <= "\udfff"
|
|
)
|
|
except IndexError:
|
|
return False
|