Trait Lexer

pub trait Lexer<'a, TokenAndSpan>: Tokens<TokenAndSpan> + Sized {
    type State: State;
    type Token: TokenFactory<'a, TokenAndSpan, Self, Lexer = Self>;
    type CommentsBuffer: CommentsBufferTrait;

    // Required methods
    fn input(&self) -> &StringInput<'a>;
    fn input_mut(&mut self) -> &mut StringInput<'a>;
    fn state(&self) -> &Self::State;
    fn state_mut(&mut self) -> &mut Self::State;
    fn comments(&self) -> Option<&'a dyn Comments>;
    fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>;
    fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>;
    unsafe fn input_slice(&mut self, start: BytePos, end: BytePos) -> &'a str;
    fn input_uncons_while(&mut self, f: impl FnMut(char) -> bool) -> &'a str;
    fn atom<'b>(&self, s: impl Into<Cow<'b, str>>) -> Atom;
    fn push_error(&mut self, error: Error);

    // Provided methods
    fn had_line_break_before_last(&self) -> bool { ... }
    fn span(&self, start: BytePos) -> Span { ... }
    fn bump(&mut self) { ... }
    fn is(&self, c: u8) -> bool { ... }
    fn is_str(&self, s: &str) -> bool { ... }
    fn eat(&mut self, c: u8) -> bool { ... }
    fn cur(&self) -> Option<char> { ... }
    fn peek(&self) -> Option<char> { ... }
    fn peek_ahead(&self) -> Option<char> { ... }
    fn cur_pos(&self) -> BytePos { ... }
    fn last_pos(&self) -> BytePos { ... }
    fn error<T>(&self, start: BytePos, kind: SyntaxError) -> LexResult<T> { ... }
    fn error_span<T>(&self, span: Span, kind: SyntaxError) -> LexResult<T> { ... }
    fn emit_error(&mut self, start: BytePos, kind: SyntaxError) { ... }
    fn emit_error_span(&mut self, span: Span, kind: SyntaxError) { ... }
    fn emit_strict_mode_error(&mut self, start: BytePos, kind: SyntaxError) { ... }
    fn emit_module_mode_error(&mut self, start: BytePos, kind: SyntaxError) { ... }
    fn skip_line_comment(&mut self, start_skip: usize) { ... }
    fn skip_block_comment(&mut self) { ... }
    fn skip_space<const LEX_COMMENTS: bool>(&mut self) { ... }
    fn ensure_not_ident(&mut self) -> LexResult<()> { ... }
    fn make_legacy_octal(&mut self, start: BytePos, val: f64) -> LexResult<f64> { ... }
    fn read_digits<F, Ret, const RADIX: u8>(
        &mut self,
        op: F,
        allow_num_separator: bool,
        has_underscore: &mut bool,
    ) -> LexResult<Ret>
       where F: FnMut(Ret, u8, u32) -> LexResult<(Ret, bool)>,
             Ret: Copy + Default { ... }
    fn read_number_no_dot_as_str<const RADIX: u8>(
        &mut self,
    ) -> LexResult<LazyInteger> { ... }
    fn read_number<const START_WITH_DOT: bool, const START_WITH_ZERO: bool>(
        &mut self,
    ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>> { ... }
    fn read_int_u32<const RADIX: u8>(
        &mut self,
        len: u8,
    ) -> LexResult<Option<u32>> { ... }
    fn read_radix_number<const RADIX: u8>(
        &mut self,
    ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>> { ... }
    fn consume_pending_comments(&mut self) { ... }
    fn read_jsx_word(&mut self) -> LexResult<Self::Token> { ... }
    fn read_jsx_entity(&mut self) -> LexResult<(char, String)> { ... }
    fn read_jsx_new_line(
        &mut self,
        normalize_crlf: bool,
    ) -> LexResult<Either<&'static str, char>> { ... }
    fn read_jsx_str(&mut self, quote: char) -> LexResult<Self::Token> { ... }
    fn read_unicode_escape(&mut self) -> LexResult<Vec<Char>> { ... }
    fn read_shebang(&mut self) -> LexResult<Option<Atom>> { ... }
    fn read_tmpl_token(
        &mut self,
        start_of_tpl: BytePos,
    ) -> LexResult<Self::Token> { ... }
    fn read_escaped_char(
        &mut self,
        in_template: bool,
    ) -> LexResult<Option<Vec<Char>>> { ... }
    fn read_regexp(&mut self, start: BytePos) -> LexResult<Self::Token> { ... }
    fn read_word_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)> { ... }
    fn read_word_as_str_with_slow_path(
        &mut self,
        slice_start: BytePos,
    ) -> LexResult<(Cow<'a, str>, bool)> { ... }
    fn read_token_number_sign(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_dot(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_question_mark(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_colon(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_zero(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_logical<const C: u8>(&mut self) -> LexResult<Self::Token> { ... }
    fn read_token_mul_mod(&mut self, is_mul: bool) -> LexResult<Self::Token> { ... }
    fn read_slash(&mut self) -> LexResult<Self::Token> { ... }
    fn read_ident_unknown(&mut self) -> LexResult<Self::Token> { ... }
    fn read_str_lit(&mut self) -> LexResult<Self::Token> { ... }
    fn read_keyword_with(
        &mut self,
        convert: &dyn Fn(&str) -> Option<Self::Token>,
    ) -> LexResult<Self::Token> { ... }
    fn read_keyword_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)> { ... }
}

Required Associated Types

type State: State

type Token: TokenFactory<'a, TokenAndSpan, Self, Lexer = Self>

type CommentsBuffer: CommentsBufferTrait

Required Methods

fn input(&self) -> &StringInput<'a>

fn input_mut(&mut self) -> &mut StringInput<'a>

fn state(&self) -> &Self::State

fn state_mut(&mut self) -> &mut Self::State

fn comments(&self) -> Option<&'a dyn Comments>

fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>

fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>

unsafe fn input_slice(&mut self, start: BytePos, end: BytePos) -> &'a str

Safety

We know that the start and the end are valid

fn input_uncons_while(&mut self, f: impl FnMut(char) -> bool) -> &'a str

fn atom<'b>(&self, s: impl Into<Cow<'b, str>>) -> Atom

fn push_error(&mut self, error: Error)

Provided Methods

fn had_line_break_before_last(&self) -> bool

fn span(&self, start: BytePos) -> Span

fn bump(&mut self)

fn is(&self, c: u8) -> bool

fn is_str(&self, s: &str) -> bool

fn eat(&mut self, c: u8) -> bool

fn cur(&self) -> Option<char>

fn peek(&self) -> Option<char>

fn peek_ahead(&self) -> Option<char>

fn cur_pos(&self) -> BytePos

fn last_pos(&self) -> BytePos

fn error<T>(&self, start: BytePos, kind: SyntaxError) -> LexResult<T>

Shorthand for let span = self.span(start); self.error_span(span)

fn error_span<T>(&self, span: Span, kind: SyntaxError) -> LexResult<T>

fn emit_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_error_span(&mut self, span: Span, kind: SyntaxError)

fn emit_strict_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_module_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn skip_line_comment(&mut self, start_skip: usize)

fn skip_block_comment(&mut self)

Expects current char to be ‘/’ and next char to be ‘*’.

fn skip_space<const LEX_COMMENTS: bool>(&mut self)

Skip comments or whitespaces.

See https://tc39.github.io/ecma262/#sec-white-space

fn ensure_not_ident(&mut self) -> LexResult<()>

Ensure that ident cannot directly follow numbers.

fn make_legacy_octal(&mut self, start: BytePos, val: f64) -> LexResult<f64>

fn read_digits<F, Ret, const RADIX: u8>( &mut self, op: F, allow_num_separator: bool, has_underscore: &mut bool, ) -> LexResult<Ret>

where F: FnMut(Ret, u8, u32) -> LexResult<(Ret, bool)>, Ret: Copy + Default,

op- |total, radix, value| -> (total * radix + value, continue)

fn read_number_no_dot_as_str<const RADIX: u8>( &mut self, ) -> LexResult<LazyInteger>

This can read long integers like “13612536612375123612312312312312312312312”.

• Returned bool is true is there was 8 or 9.

fn read_number<const START_WITH_DOT: bool, const START_WITH_ZERO: bool>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Reads an integer, octal integer, or floating-point number

fn read_int_u32<const RADIX: u8>(&mut self, len: u8) -> LexResult<Option<u32>>

fn read_radix_number<const RADIX: u8>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Returns Left(value) or Right(BigInt)

fn consume_pending_comments(&mut self)

Consume pending comments.

This is called when the input is exhausted.

fn read_jsx_word(&mut self) -> LexResult<Self::Token>

Read a JSX identifier (valid tag or attribute name).

Optimized version since JSX identifiers can“t contain escape characters and so can be read as single slice. Also assumes that first character was already checked by isIdentifierStart in readToken.

fn read_jsx_entity(&mut self) -> LexResult<(char, String)>

fn read_jsx_new_line( &mut self, normalize_crlf: bool, ) -> LexResult<Either<&'static str, char>>

fn read_jsx_str(&mut self, quote: char) -> LexResult<Self::Token>

fn read_unicode_escape(&mut self) -> LexResult<Vec<Char>>

fn read_shebang(&mut self) -> LexResult<Option<Atom>>

fn read_tmpl_token(&mut self, start_of_tpl: BytePos) -> LexResult<Self::Token>

fn read_escaped_char( &mut self, in_template: bool, ) -> LexResult<Option<Vec<Char>>>

Read an escaped character for string literal.

In template literal, we should preserve raw string.

fn read_regexp(&mut self, start: BytePos) -> LexResult<Self::Token>

Expects current char to be ‘/’

fn read_word_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This method is optimized for texts without escape sequences.

fn read_word_as_str_with_slow_path( &mut self, slice_start: BytePos, ) -> LexResult<(Cow<'a, str>, bool)>

Slow path for identifier parsing that handles Unicode and escapes

fn read_token_number_sign(&mut self) -> LexResult<Self::Token>

#

fn read_token_dot(&mut self) -> LexResult<Self::Token>

Read a token given ..

This is extracted as a method to reduce size of read_token.

fn read_token_question_mark(&mut self) -> LexResult<Self::Token>

Read a token given ?.

This is extracted as a method to reduce size of read_token.

fn read_token_colon(&mut self) -> LexResult<Self::Token>

Read a token given :.

This is extracted as a method to reduce size of read_token.

fn read_token_zero(&mut self) -> LexResult<Self::Token>

Read a token given 0.

This is extracted as a method to reduce size of read_token.

fn read_token_logical<const C: u8>(&mut self) -> LexResult<Self::Token>

Read a token given | or &.

This is extracted as a method to reduce size of read_token.

fn read_token_mul_mod(&mut self, is_mul: bool) -> LexResult<Self::Token>

Read a token given * or %.

This is extracted as a method to reduce size of read_token.

fn read_slash(&mut self) -> LexResult<Self::Token>

fn read_ident_unknown(&mut self) -> LexResult<Self::Token>

This can be used if there’s no keyword starting with the first character.

fn read_str_lit(&mut self) -> LexResult<Self::Token>

See https://tc39.github.io/ecma262/#sec-literals-string-literals

fn read_keyword_with( &mut self, convert: &dyn Fn(&str) -> Option<Self::Token>, ) -> LexResult<Self::Token>

fn read_keyword_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This is a performant version of Lexer::read_word_as_str_with for reading keywords. We should make sure the first byte is a valid ASCII.

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors

impl<'a> Lexer<'a, TokenAndSpan> for Lexer<'a>

type CommentsBuffer = CommentsBuffer

type State = State

type Token = Token