src/ail_ref/lexer.py

Read-only source from the unchanged 0.3.4 kit. Existing license terms apply.

from __future__ import annotations

from dataclasses import dataclass


class LexError(ValueError):
    pass


@dataclass(frozen=True)
class Token:
    kind: str
    value: str
    offset: int


SYMBOLS = set("{}()[]=;,~")


def _is_identifier_start(ch: str) -> bool:
    return ch.isalpha()


def _is_identifier_part(ch: str) -> bool:
    return ch.isalnum() or ch in "_:-./"


def tokenize(text: str) -> list[Token]:
    tokens: list[Token] = []
    i = 0
    n = len(text)
    while i < n:
        ch = text[i]
        if ch.isspace():
            i += 1
            continue

        if text.startswith("AIL-PASSPORT/0.2", i):
            tokens.append(Token("HEADER", "AIL-PASSPORT/0.2", i))
            i += len("AIL-PASSPORT/0.2")
            continue
        if text.startswith("AIL/0.2", i):
            tokens.append(Token("HEADER", "AIL/0.2", i))
            i += len("AIL/0.2")
            continue

        if ch == '"':
            start = i
            i += 1
            chars: list[str] = []
            while i < n:
                if text[i] == '"':
                    i += 1
                    break
                if text[i] == "\\":
                    i += 1
                    if i >= n:
                        raise LexError(f"unterminated escape at offset {start}")
                    escapes = {"n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"}
                    chars.append(escapes.get(text[i], text[i]))
                    i += 1
                    continue
                chars.append(text[i])
                i += 1
            else:
                raise LexError(f"unterminated string at offset {start}")
            tokens.append(Token("STRING", "".join(chars), start))
            continue

        if ch == "$":
            start = i
            i += 1
            if i >= n or not _is_identifier_start(text[i]):
                raise LexError(f"invalid variable at offset {start}")
            j = i + 1
            while j < n and _is_identifier_part(text[j]):
                j += 1
            tokens.append(Token("VARIABLE", text[i:j], start))
            i = j
            continue

        if ch == "_":
            tokens.append(Token("UNKNOWN", ch, i))
            i += 1
            continue

        if ch in SYMBOLS:
            tokens.append(Token(ch, ch, i))
            i += 1
            continue

        if ch.isdigit() or (ch in "+-." and i + 1 < n and text[i + 1].isdigit()):
            start = i
            j = i
            if text[j] in "+-":
                j += 1
            while j < n and text[j].isdigit():
                j += 1
            if j < n and text[j] == ".":
                j += 1
                while j < n and text[j].isdigit():
                    j += 1
            if j < n and text[j] in "eE":
                j += 1
                if j < n and text[j] in "+-":
                    j += 1
                exp_start = j
                while j < n and text[j].isdigit():
                    j += 1
                if j == exp_start:
                    raise LexError(f"invalid exponent at offset {start}")
            tokens.append(Token("NUMBER", text[start:j], start))
            i = j
            continue

        if _is_identifier_start(ch):
            start = i
            j = i + 1
            while j < n and _is_identifier_part(text[j]):
                j += 1
            value = text[start:j]
            kind = "BOOLEAN" if value in {"TRUE", "FALSE"} else "IDENT"
            tokens.append(Token(kind, value, start))
            i = j
            continue

        raise LexError(f"unexpected character {ch!r} at offset {i}")

    tokens.append(Token("EOF", "", n))
    return tokens