You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
74 lines
1.9 KiB
Python
74 lines
1.9 KiB
Python
2 years ago
|
from typing import TypeVar, Generic, Iterable, Iterator, List
|
||
|
|
from .defs import Token, TokenType
|
||
|
|
|
||
|
|
|
||
|
|
T = TypeVar("T")
|
||
|
|
class PeekableIterator(Iterator[T]):
|
||
|
|
_peeked: List[T]
|
||
|
|
last_item: T | None
|
||
|
|
|
||
|
|
def __init__(self, iterable: Iterable[T]) -> None:
|
||
|
|
self.iter = iterable
|
||
|
|
self._peeked = list()
|
||
|
|
self.last_item = None
|
||
|
|
|
||
|
|
def peek(self, offset: int = 0):
|
||
|
|
while len(self._peeked) <= offset:
|
||
|
|
try:
|
||
|
|
self._peeked.append(next(self.iter))
|
||
|
|
except StopIteration:
|
||
|
|
return None
|
||
|
|
|
||
|
|
return self._peeked[offset]
|
||
|
|
|
||
|
|
def __next__(self) -> T:
|
||
|
|
if len(self._peeked) > 0:
|
||
|
|
item = self._peeked.pop(0)
|
||
|
|
else:
|
||
|
|
item = next(self.iter)
|
||
|
|
self.last_item = item
|
||
|
|
return item
|
||
|
|
|
||
|
|
def __iter__(self) -> Iterator[T]:
|
||
|
|
return self
|
||
|
|
|
||
|
|
def next(self) -> T:
|
||
|
|
try:
|
||
|
|
return next(self)
|
||
|
|
except StopIteration:
|
||
|
|
return None
|
||
|
|
|
||
|
|
def has_next(self):
|
||
|
|
return self.peek() is not None
|
||
|
|
|
||
|
|
|
||
|
|
class ParserIterator(PeekableIterator[Token]):
|
||
|
|
def __init__(self, iterable: Iterable[Token]) -> None:
|
||
|
|
super().__init__(t for t in iterable if t.kind not in (TokenType.LineComment, TokenType.MultiComment))
|
||
|
|
self.ignore_newline = False
|
||
|
|
|
||
|
|
def peek(self, offset: int = 0):
|
||
|
|
while len(self._peeked) <= offset:
|
||
|
|
try:
|
||
|
|
self._peeked.append(next(self.iter))
|
||
|
|
except StopIteration:
|
||
|
|
return None
|
||
|
|
|
||
|
|
token = self._peeked[offset]
|
||
|
|
|
||
|
|
if self.ignore_newline and token.kind == TokenType.EOL:
|
||
|
|
return self.peek(offset=offset+1)
|
||
|
|
|
||
|
|
return token
|
||
|
|
|
||
|
|
def __next__(self) -> T:
|
||
|
|
if len(self._peeked) > 0:
|
||
|
|
item = self._peeked.pop(0)
|
||
|
|
else:
|
||
|
|
item = next(self.iter)
|
||
|
|
self.last_item = item
|
||
|
|
|
||
|
|
if self.ignore_newline and item.kind == TokenType.EOL:
|
||
|
|
return next(self)
|
||
|
|
|
||
|
|
return item
|