Struct TokenAndSpan

pub struct TokenAndSpan {
    pub token: Token,
    pub had_line_break: bool,
    pub span: Span,
}

Fields

token: Token

had_line_break: bool

Had a line break before this token?

span: Span

Trait Implementations

impl Clone for TokenAndSpan

fn clone(&self) -> TokenAndSpan

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenAndSpan

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a> Lexer<'a, TokenAndSpan> for Lexer<'a>

type CommentsBuffer = CommentsBuffer

type State = State

type Token = Token

fn input(&self) -> &StringInput<'a>

fn input_mut(&mut self) -> &mut StringInput<'a>

fn push_error(&mut self, error: Error)

fn state(&self) -> &Self::State

fn state_mut(&mut self) -> &mut Self::State

fn comments(&self) -> Option<&'a dyn Comments>

fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>

fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>

unsafe fn input_slice(&mut self, start: BytePos, end: BytePos) -> &'a str

Safety

fn input_uncons_while(&mut self, f: impl FnMut(char) -> bool) -> &'a str

fn atom<'b>(&self, s: impl Into<Cow<'b, str>>) -> Atom

fn had_line_break_before_last(&self) -> bool

fn span(&self, start: BytePos) -> Span

fn bump(&mut self)

fn is(&self, c: u8) -> bool

fn is_str(&self, s: &str) -> bool

fn eat(&mut self, c: u8) -> bool

fn cur(&self) -> Option<char>

fn peek(&self) -> Option<char>

fn peek_ahead(&self) -> Option<char>

fn cur_pos(&self) -> BytePos

fn last_pos(&self) -> BytePos

fn error<T>(&self, start: BytePos, kind: SyntaxError) -> LexResult<T>

Shorthand for let span = self.span(start); self.error_span(span)

fn error_span<T>(&self, span: Span, kind: SyntaxError) -> LexResult<T>

fn emit_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_error_span(&mut self, span: Span, kind: SyntaxError)

fn emit_strict_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_module_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn skip_line_comment(&mut self, start_skip: usize)

fn skip_block_comment(&mut self)

Expects current char to be ‘/’ and next char to be ‘*’.

fn skip_space<const LEX_COMMENTS: bool>(&mut self)

Skip comments or whitespaces.

fn ensure_not_ident(&mut self) -> LexResult<()>

Ensure that ident cannot directly follow numbers.

fn make_legacy_octal(&mut self, start: BytePos, val: f64) -> LexResult<f64>

fn read_digits<F, Ret, const RADIX: u8>( &mut self, op: F, allow_num_separator: bool, has_underscore: &mut bool, ) -> LexResult<Ret>

where F: FnMut(Ret, u8, u32) -> LexResult<(Ret, bool)>, Ret: Copy + Default,

op- |total, radix, value| -> (total * radix + value, continue)

fn read_number_no_dot_as_str<const RADIX: u8>( &mut self, ) -> LexResult<LazyInteger>

This can read long integers like “13612536612375123612312312312312312312312”.

fn read_number<const START_WITH_DOT: bool, const START_WITH_ZERO: bool>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Reads an integer, octal integer, or floating-point number

fn read_int_u32<const RADIX: u8>(&mut self, len: u8) -> LexResult<Option<u32>>

fn read_radix_number<const RADIX: u8>( &mut self, ) -> LexResult<Either<(f64, Atom), (Box<BigIntValue>, Atom)>>

Returns Left(value) or Right(BigInt)

fn consume_pending_comments(&mut self)

Consume pending comments.

fn read_jsx_word(&mut self) -> LexResult<Self::Token>

Read a JSX identifier (valid tag or attribute name).

fn read_jsx_entity(&mut self) -> LexResult<(char, String)>

fn read_jsx_new_line( &mut self, normalize_crlf: bool, ) -> LexResult<Either<&'static str, char>>

fn read_jsx_str(&mut self, quote: char) -> LexResult<Self::Token>

fn read_unicode_escape(&mut self) -> LexResult<Vec<Char>>

fn read_shebang(&mut self) -> LexResult<Option<Atom>>

fn read_tmpl_token(&mut self, start_of_tpl: BytePos) -> LexResult<Self::Token>

fn read_escaped_char( &mut self, in_template: bool, ) -> LexResult<Option<Vec<Char>>>

Read an escaped character for string literal.

fn read_regexp(&mut self, start: BytePos) -> LexResult<Self::Token>

Expects current char to be ‘/’

fn read_word_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This method is optimized for texts without escape sequences.

fn read_word_as_str_with_slow_path( &mut self, slice_start: BytePos, ) -> LexResult<(Cow<'a, str>, bool)>

Slow path for identifier parsing that handles Unicode and escapes

fn read_token_number_sign(&mut self) -> LexResult<Self::Token>

#

fn read_token_dot(&mut self) -> LexResult<Self::Token>

Read a token given ..

fn read_token_question_mark(&mut self) -> LexResult<Self::Token>

Read a token given ?.

fn read_token_colon(&mut self) -> LexResult<Self::Token>

Read a token given :.

fn read_token_zero(&mut self) -> LexResult<Self::Token>

Read a token given 0.

fn read_token_logical<const C: u8>(&mut self) -> LexResult<Self::Token>

Read a token given | or &.

fn read_token_mul_mod(&mut self, is_mul: bool) -> LexResult<Self::Token>

Read a token given * or %.

fn read_slash(&mut self) -> LexResult<Self::Token>

fn read_ident_unknown(&mut self) -> LexResult<Self::Token>

This can be used if there’s no keyword starting with the first character.

fn read_str_lit(&mut self) -> LexResult<Self::Token>

See https://tc39.github.io/ecma262/#sec-literals-string-literals

fn read_keyword_with( &mut self, convert: &dyn Fn(&str) -> Option<Self::Token>, ) -> LexResult<Self::Token>

fn read_keyword_as_str_with(&mut self) -> LexResult<(Cow<'a, str>, bool)>

This is a performant version of Lexer::read_word_as_str_with for reading keywords. We should make sure the first byte is a valid ASCII.

impl NextTokenAndSpan for TokenAndSpan

type Token = Token

fn token(&self) -> &Self::Token

fn span(&self) -> Span

fn had_line_break(&self) -> bool

impl PartialEq for TokenAndSpan

fn eq(&self, other: &TokenAndSpan) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Spanned for TokenAndSpan

fn span(&self) -> Span

Get span of self.

fn span_lo(&self) -> BytePos

fn span_hi(&self) -> BytePos

impl TokenAndSpan for TokenAndSpan

type Token = Token

fn new(token: Token, span: Span, had_line_break: bool) -> Self

fn token(&self) -> &Token

fn take_token(self) -> Token

fn had_line_break(&self) -> bool

fn span(&self) -> Span

impl<'a, I: Tokens<TokenAndSpan>> TokenFactory<'a, TokenAndSpan, I> for Token

const ABSTRACT: Self

const ACCESSOR: Self

const ANY: Self

const ARROW: Self = Self::Arrow

const AS: Self

const ASSERT: Self

const ASSERTS: Self

const ASYNC: Self

const AT: Self = Self::At

const AWAIT: Self

const BACKQUOTE: Self = Self::BackQuote

const BANG: Self = Self::Bang

const BIGINT: Self

const BIT_AND: Self

const BIT_AND_EQ: Self

const BIT_OR: Self

const BIT_OR_EQ: Self

const BOOLEAN: Self

const BREAK: Self

const CASE: Self

const CATCH: Self

const CLASS: Self

const COLON: Self = Self::Colon

const COMMA: Self = Self::Comma

const CONST: Self

const CONTINUE: Self

const DEBUGGER: Self

const DECLARE: Self

const DEFAULT: Self

const DELETE: Self

const DIV: Self

const DIV_EQ: Self

const DO: Self

const DOLLAR_LBRACE: Self = Self::DollarLBrace

const DOT: Self = Self::Dot

const DOTDOTDOT: Self = Self::DotDotDot

const ELSE: Self

const ENUM: Self

const EOF: Self = Token::Eof

const EQUAL: Self

const EXP: Self

const EXPORT: Self

const EXP_EQ: Self

const EXTENDS: Self

const FALSE: Self

const FINALLY: Self

const FOR: Self

const FROM: Self

const FUNCTION: Self

const GET: Self

const GLOBAL: Self

const GREATER: Self

const GREATER_EQ: Self

const HASH: Self = Self::Hash

const IF: Self

const IMPLEMENTS: Self

const IMPORT: Self

const IN: Self

const INFER: Self

const INSTANCEOF: Self

const INTERFACE: Self

const INTRINSIC: Self

const IS: Self

const JSX_TAG_END: Self = Self::JSXTagEnd

const JSX_TAG_START: Self = Self::JSXTagStart

const KEYOF: Self

const LBRACE: Self = Self::LBrace

const LBRACKET: Self = Self::LBracket

const LESS: Self

const LESS_EQ: Self

const LET: Self

const LOGICAL_AND: Self

const LOGICAL_AND_EQ: Self

const LOGICAL_OR: Self

const LOGICAL_OR_EQ: Self

const LPAREN: Self = Self::LParen

const LSHIFT: Self

const LSHIFT_EQ: Self

const MINUS: Self

const MINUS_MINUS: Self = Self::MinusMinus

const MOD: Self

const MOD_EQ: Self

const MUL: Self

const MUL_EQ: Self

const NAMESPACE: Self

const NEVER: Self

const NEW: Self

const NULL: Self

const NULLISH_ASSIGN: Self

const NULLISH_COALESCING: Self

const NUMBER: Self

const OBJECT: Self

const OF: Self

const PACKAGE: Self

const PLUS: Self

const PLUS_PLUS: Self = Self::PlusPlus

const PRIVATE: Self

const PROTECTED: Self

const PUBLIC: Self

const QUESTION: Self = Token::QuestionMark

const RBRACE: Self = Self::RBrace

const RBRACKET: Self = Self::RBracket

const READONLY: Self

const REQUIRE: Self

const RETURN: Self

const RPAREN: Self = Self::RParen

const RSHIFT: Self

const RSHIFT_EQ: Self

const SATISFIES: Self

const SEMI: Self = Self::Semi

const SET: Self

const STATIC: Self

const STRING: Self

const SUPER: Self

const SWITCH: Self

const SYMBOL: Self

const TARGET: Self

const THIS: Self

const THROW: Self

const TILDE: Self = Self::Tilde

const TRUE: Self

const TRY: Self

const TYPE: Self

const TYPEOF: Self

const UNDEFINED: Self

const UNIQUE: Self

const UNKNOWN: Self

const USING: Self

const VAR: Self

const VOID: Self

const WHILE: Self

const WITH: Self

const YIELD: Self

const ZERO_FILL_RSHIFT: Self

const ZERO_FILL_RSHIFT_EQ: Self

type Buffer = Buffer<I>

type Lexer = Lexer<'a>

fn jsx_name(name: &'a str, lexer: &mut Lexer<'a>) -> Self

fn is_jsx_name(&self) -> bool

fn take_jsx_name(self, _: &mut Self::Buffer) -> Atom

fn str(value: Atom, raw: Atom, _: &mut Lexer<'a>) -> Self

fn template(cooked: LexResult<Atom>, raw: Atom, _: &mut Lexer<'a>) -> Self

fn regexp(content: Atom, flags: Atom, _: &mut Lexer<'a>) -> Self

fn num(value: f64, raw: Atom, _: &mut Lexer<'a>) -> Self

fn bigint(value: Box<BigIntValue>, raw: Atom, _: &mut Lexer<'a>) -> Self

fn unknown_ident(value: Atom, _: &mut Lexer<'a>) -> Self

fn is_word(&self) -> bool

fn is_reserved(&self, ctx: Context) -> bool

fn into_atom(self, _: &mut Lexer<'a>) -> Option<Atom>

fn is_error(&self) -> bool

fn take_error(self, _: &mut Self::Buffer) -> Error

fn is_str(&self) -> bool

fn is_str_raw_content(&self, content: &str, _: &Self::Buffer) -> bool

fn take_str(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn is_num(&self) -> bool

fn take_num(self, _: &mut Self::Buffer) -> (f64, Atom)

fn is_bigint(&self) -> bool

fn take_bigint(self, _: &mut Self::Buffer) -> (Box<BigIntValue>, Atom)

fn take_word(self, _: &Self::Buffer) -> Option<Atom>

fn is_unknown_ident(&self) -> bool

fn take_unknown_ident(self, _: &mut Self::Buffer) -> Atom

fn take_unknown_ident_ref<'b>(&'b self, _: &'b Self::Buffer) -> &'b Atom

fn is_keyword(&self) -> bool

fn is_known_ident(&self) -> bool

fn take_known_ident(&self) -> Atom

fn is_regexp(&self) -> bool

fn is_template(&self) -> bool

fn take_template(self, _: &mut Self::Buffer) -> (LexResult<Atom>, Atom)

fn jsx_text(value: Atom, raw: Atom, _: &mut Self::Lexer) -> Self

fn is_jsx_text(&self) -> bool

fn take_jsx_text(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn starts_expr(&self) -> bool

fn to_string(&self, _: &Self::Buffer) -> String

fn is_bin_op(&self) -> bool

fn as_assign_op(&self) -> Option<AssignOp>

fn as_bin_op(&self) -> Option<BinaryOp>

fn follows_keyword_let(&self) -> bool

fn is_assign_op(&self) -> bool

fn take_regexp(self, _: &mut Self::Buffer) -> (Atom, Atom)

fn shebang(value: Atom, _: &mut Self::Lexer) -> Self

fn is_shebang(&self) -> bool

fn take_shebang(self, _: &mut Self::Buffer) -> Atom

fn is_no_substitution_template_literal(&self) -> bool

fn is_template_head(&self) -> bool

fn is_less(&self) -> bool

fn is_less_eq(&self) -> bool

fn is_greater(&self) -> bool

fn is_colon(&self) -> bool

fn is_comma(&self) -> bool

fn is_equal(&self) -> bool

fn is_question(&self) -> bool

fn is_null(&self) -> bool

fn is_lshift(&self) -> bool

fn is_rshift(&self) -> bool

fn is_rshift_eq(&self) -> bool

fn is_greater_eq(&self) -> bool

fn is_true(&self) -> bool

fn is_false(&self) -> bool

fn is_enum(&self) -> bool

fn is_yield(&self) -> bool

fn is_let(&self) -> bool

fn is_var(&self) -> bool

fn is_static(&self) -> bool

fn is_extends(&self) -> bool

fn is_implements(&self) -> bool

fn is_interface(&self) -> bool

fn is_type(&self) -> bool

fn is_package(&self) -> bool

fn is_private(&self) -> bool

fn is_protected(&self) -> bool

fn is_public(&self) -> bool

fn is_readonly(&self) -> bool

fn is_await(&self) -> bool

fn is_break(&self) -> bool

fn is_continue(&self) -> bool

fn is_arrow(&self) -> bool

fn is_this(&self) -> bool

fn is_super(&self) -> bool

fn is_using(&self) -> bool

fn is_backquote(&self) -> bool

fn is_lparen(&self) -> bool

fn is_rparen(&self) -> bool

fn is_lbracket(&self) -> bool

fn is_rbracket(&self) -> bool

fn is_lbrace(&self) -> bool

fn is_rbrace(&self) -> bool

fn is_function(&self) -> bool

fn is_class(&self) -> bool

fn is_if(&self) -> bool

fn is_return(&self) -> bool

fn is_switch(&self) -> bool

fn is_throw(&self) -> bool

fn is_catch(&self) -> bool

fn is_finally(&self) -> bool

fn is_try(&self) -> bool

fn is_with(&self) -> bool

fn is_while(&self) -> bool

fn is_new(&self) -> bool

fn is_ident_ref(&self, ctx: Context) -> bool

fn is_import(&self) -> bool

fn is_export(&self) -> bool

fn is_dot(&self) -> bool

fn is_do(&self) -> bool

fn is_for(&self) -> bool

fn is_from(&self) -> bool

fn is_dotdotdot(&self) -> bool

fn is_plus(&self) -> bool

fn is_minus(&self) -> bool

fn is_bang(&self) -> bool

fn is_tilde(&self) -> bool

fn is_plus_plus(&self) -> bool

fn is_minus_minus(&self) -> bool

fn is_delete(&self) -> bool

fn is_typeof(&self) -> bool

fn is_of(&self) -> bool

fn is_void(&self) -> bool

fn is_hash(&self) -> bool

fn is_in(&self) -> bool

fn is_const(&self) -> bool

fn is_star(&self) -> bool

fn is_mod(&self) -> bool

fn is_semi(&self) -> bool

fn is_slash(&self) -> bool

fn is_slash_eq(&self) -> bool

fn is_jsx_tag_start(&self) -> bool

fn is_jsx_tag_end(&self) -> bool

fn is_asserts(&self) -> bool

fn is_is(&self) -> bool

fn is_as(&self) -> bool

fn is_satisfies(&self) -> bool

fn is_instanceof(&self) -> bool

fn is_async(&self) -> bool

fn is_case(&self) -> bool

fn is_default(&self) -> bool

fn is_debugger(&self) -> bool

fn is_bit_and(&self) -> bool

fn is_bit_or(&self) -> bool

fn is_exp(&self) -> bool

fn is_eof(&self) -> bool

impl<I: Tokens<TokenAndSpan>> Tokens<TokenAndSpan> for Capturing<I>

type Checkpoint = <I as Tokens<TokenAndSpan>>::Checkpoint

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx_mut(&mut self) -> &mut Context

fn ctx(&self) -> Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, allow: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl Tokens<TokenAndSpan> for Lexer<'_>

type Checkpoint = Lexer<'_>

fn set_ctx(&mut self, ctx: Context)

fn ctx(&self) -> Context

fn ctx_mut(&mut self) -> &mut Context

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, allow: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl Tokens<TokenAndSpan> for TokensInput

type Checkpoint = TokensInput

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx_mut(&mut self) -> &mut Context

fn ctx(&self) -> Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, _: bool)

fn set_next_regexp(&mut self, _: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, _: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl StructuralPartialEq for TokenAndSpan