Struct TokenAndSpan

pub struct TokenAndSpan {
    pub token: Token,
    pub had_line_break: bool,
    pub span: Span,
}

Fields

token: Token

had_line_break: bool

Had a line break before this token?

span: Span

Trait Implementations

impl Clone for TokenAndSpan

fn clone(&self) -> TokenAndSpan

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenAndSpan

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a> Lexer<'a, TokenAndSpan> for Lexer<'a>

type CommentsBuffer = CommentsBuffer

type State = State

type Token = Token

fn input(&self) -> &StringInput<'a>

fn input_mut(&mut self) -> &mut StringInput<'a>

fn push_error(&mut self, error: Error)

fn state(&self) -> &Self::State

fn state_mut(&mut self) -> &mut Self::State

fn comments(&self) -> Option<&'a dyn Comments>

fn comments_buffer(&self) -> Option<&Self::CommentsBuffer>

fn comments_buffer_mut(&mut self) -> Option<&mut Self::CommentsBuffer>

unsafe fn input_slice(&mut self, start: BytePos, end: BytePos) -> &'a str

Safety

fn input_uncons_while(&mut self, f: impl FnMut(char) -> bool) -> &'a str

fn atom<'b>(&self, s: impl Into<Cow<'b, str>>) -> Atom

fn had_line_break_before_last(&self) -> bool

fn span(&self, start: BytePos) -> Span

fn bump(&mut self)

fn is(&self, c: u8) -> bool

fn is_str(&self, s: &str) -> bool

fn eat(&mut self, c: u8) -> bool

fn cur(&self) -> Option<char>

fn peek(&self) -> Option<char>

fn peek_ahead(&self) -> Option<char>

fn cur_pos(&self) -> BytePos

fn last_pos(&self) -> BytePos

fn error<T>(&self, start: BytePos, kind: SyntaxError) -> Result<T, Error>

Shorthand for let span = self.span(start); self.error_span(span)

fn error_span<T>(&self, span: Span, kind: SyntaxError) -> Result<T, Error>

fn emit_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_error_span(&mut self, span: Span, kind: SyntaxError)

fn emit_strict_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn emit_module_mode_error(&mut self, start: BytePos, kind: SyntaxError)

fn skip_line_comment(&mut self, start_skip: usize)

fn skip_block_comment(&mut self)

Expects current char to be ‘/’ and next char to be ‘*’.

fn skip_space<const LEX_COMMENTS: bool>(&mut self)

Skip comments or whitespaces.

fn ensure_not_ident(&mut self) -> Result<(), Error>

Ensure that ident cannot directly follow numbers.

fn make_legacy_octal(&mut self, start: BytePos, val: f64) -> Result<f64, Error>

fn read_digits<F, Ret, const RADIX: u8>( &mut self, op: F, allow_num_separator: bool, has_underscore: &mut bool, ) -> Result<Ret, Error>

where F: FnMut(Ret, u8, u32) -> Result<(Ret, bool), Error>, Ret: Copy + Default,

op- |total, radix, value| -> (total * radix + value, continue)

fn read_number_no_dot_as_str<const RADIX: u8>( &mut self, ) -> Result<LazyInteger, Error>

This can read long integers like “13612536612375123612312312312312312312312”.

fn read_number<const START_WITH_DOT: bool, const START_WITH_ZERO: bool>( &mut self, ) -> Result<Either<(f64, Atom), (Box<BigInt>, Atom)>, Error>

Reads an integer, octal integer, or floating-point number

fn read_int_u32<const RADIX: u8>( &mut self, len: u8, ) -> Result<Option<u32>, Error>

fn read_radix_number<const RADIX: u8>( &mut self, ) -> Result<Either<(f64, Atom), (Box<BigInt>, Atom)>, Error>

Returns Left(value) or Right(BigInt)

fn consume_pending_comments(&mut self)

Consume pending comments.

fn read_jsx_word(&mut self) -> Result<Self::Token, Error>

Read a JSX identifier (valid tag or attribute name).

fn read_jsx_entity(&mut self) -> Result<(char, String), Error>

fn read_jsx_new_line( &mut self, normalize_crlf: bool, ) -> Result<Either<&'static str, char>, Error>

fn read_jsx_str(&mut self, quote: char) -> Result<Self::Token, Error>

fn read_unicode_escape(&mut self) -> Result<Vec<Char>, Error>

fn read_shebang(&mut self) -> Result<Option<Atom>, Error>

fn read_tmpl_token( &mut self, start_of_tpl: BytePos, ) -> Result<Self::Token, Error>

fn read_escaped_char( &mut self, in_template: bool, ) -> Result<Option<Vec<Char>>, Error>

Read an escaped character for string literal.

fn read_regexp(&mut self, start: BytePos) -> Result<Self::Token, Error>

Expects current char to be ‘/’

fn read_word_as_str_with(&mut self) -> Result<(Cow<'a, str>, bool), Error>

This method is optimized for texts without escape sequences.

fn read_word_as_str_with_slow_path( &mut self, slice_start: BytePos, ) -> Result<(Cow<'a, str>, bool), Error>

Slow path for identifier parsing that handles Unicode and escapes

fn read_token_number_sign(&mut self) -> Result<Self::Token, Error>

#

fn read_token_dot(&mut self) -> Result<Self::Token, Error>

Read a token given ..

fn read_token_question_mark(&mut self) -> Result<Self::Token, Error>

Read a token given ?.

fn read_token_colon(&mut self) -> Result<Self::Token, Error>

Read a token given :.

fn read_token_zero(&mut self) -> Result<Self::Token, Error>

Read a token given 0.

fn read_token_logical<const C: u8>(&mut self) -> Result<Self::Token, Error>

Read a token given | or &.

fn read_token_mul_mod(&mut self, is_mul: bool) -> Result<Self::Token, Error>

Read a token given * or %.

fn read_slash(&mut self) -> Result<Self::Token, Error>

fn read_ident_unknown(&mut self) -> Result<Self::Token, Error>

This can be used if there’s no keyword starting with the first character.

fn read_str_lit(&mut self) -> Result<Self::Token, Error>

See https://tc39.github.io/ecma262/#sec-literals-string-literals

fn read_keyword_with( &mut self, convert: &dyn Fn(&str) -> Option<Self::Token>, ) -> Result<Self::Token, Error>

fn read_keyword_as_str_with(&mut self) -> Result<(Cow<'a, str>, bool), Error>

This is a performant version of [Lexer::read_word_as_str_with] for reading keywords. We should make sure the first byte is a valid ASCII.

impl TokenAndSpan for TokenAndSpan

type Token = Token

fn new(token: Token, span: Span, had_line_break: bool) -> Self

fn token(&self) -> &Token

fn take_token(self) -> Token

fn had_line_break(&self) -> bool

fn span(&self) -> Span

impl<'a, I: Tokens> TokenFactory<'a, TokenAndSpan, I> for Token

const ABSTRACT: Self = Token::Abstract

const ACCESSOR: Self = Token::Accessor

const ANY: Self = Token::Any

const ARROW: Self = Token::Arrow

const AS: Self = Token::As

const ASSERT: Self = Token::Assert

const ASSERTS: Self = Token::Asserts

const ASYNC: Self = Token::Async

const AT: Self = Token::At

const AWAIT: Self = Token::Await

const BACKQUOTE: Self = Token::BackQuote

const BANG: Self = Self::Bang

const BIGINT: Self = Token::Bigint

const BIT_AND: Self = Self::Ampersand

const BIT_AND_EQ: Self = Self::BitAndEq

const BIT_OR: Self = Self::Pipe

const BIT_OR_EQ: Self = Self::BitOrEq

const BOOLEAN: Self = Token::Boolean

const BREAK: Self = Token::Break

const CASE: Self = Token::Case

const CATCH: Self = Token::Catch

const CLASS: Self = Self::Class

const COLON: Self = Self::Colon

const COMMA: Self = Token::Comma

const CONST: Self = Self::Const

const CONTINUE: Self = Token::Continue

const DEBUGGER: Self = Token::Debugger

const DECLARE: Self = Token::Declare

const DEFAULT: Self = Token::Default

const DELETE: Self = Self::Delete

const DIV: Self = Token::Slash

const DIV_EQ: Self = Token::DivEq

const DO: Self = Token::Do

const DOLLAR_LBRACE: Self = Token::DollarLBrace

const DOT: Self = Self::Dot

const DOTDOTDOT: Self = Self::DotDotDot

const ELSE: Self = Self::Else

const ENUM: Self = Token::Enum

const EOF: Self = Token::Eof

const EQUAL: Self = Token::Eq

const EXP: Self = Token::Exp

const EXPORT: Self = Token::Export

const EXP_EQ: Self = Token::ExpEq

const EXTENDS: Self = Token::Extends

const FALSE: Self = Token::False

const FINALLY: Self = Token::Finally

const FOR: Self = Token::For

const FROM: Self = Token::From

const FUNCTION: Self = Self::Function

const GET: Self = Token::Get

const GLOBAL: Self = Token::Global

const GREATER: Self = Token::Gt

const GREATER_EQ: Self = Token::GtEq

const HASH: Self = Self::Hash

const IF: Self = Self::If

const IMPLEMENTS: Self = Token::Implements

const IMPORT: Self = Self::Import

const IN: Self = Self::In

const INFER: Self = Token::Infer

const INSTANCEOF: Self = Token::InstanceOf

const INTERFACE: Self = Token::Interface

const INTRINSIC: Self = Token::Intrinsic

const IS: Self = Token::Is

const JSX_TAG_END: Self = Token::JSXTagEnd

const JSX_TAG_START: Self = Token::JSXTagStart

const KEYOF: Self = Token::Keyof

const LBRACE: Self = Self::LBrace

const LBRACKET: Self = Self::LBracket

const LESS: Self = Token::Lt

const LESS_EQ: Self = Token::LtEq

const LET: Self = Token::Let

const LOGICAL_AND: Self = Token::LogicalAnd

const LOGICAL_AND_EQ: Self = Self::LogicalAndEq

const LOGICAL_OR: Self = Token::LogicalOr

const LOGICAL_OR_EQ: Self = Self::LogicalOrEq

const LPAREN: Self = Self::LParen

const LSHIFT: Self = Token::LShift

const LSHIFT_EQ: Self = Token::LShiftEq

const MINUS: Self = Self::Minus

const MINUS_MINUS: Self = Self::MinusMinus

const MOD: Self = Token::Percent

const MOD_EQ: Self = Token::ModEq

const MUL: Self = Token::Asterisk

const MUL_EQ: Self = Token::MulEq

const NAMESPACE: Self = Token::Namespace

const NEVER: Self = Token::Never

const NEW: Self = Self::New

const NULL: Self = Token::Null

const NULLISH_ASSIGN: Self = Token::NullishEq

const NULLISH_COALESCING: Self = Token::NullishCoalescing

const NUMBER: Self = Token::Number

const OBJECT: Self = Token::Object

const OF: Self = Token::Of

const PACKAGE: Self = Token::Package

const PLUS: Self = Self::Plus

const PLUS_PLUS: Self = Self::PlusPlus

const PRIVATE: Self = Token::Private

const PROTECTED: Self = Token::Protected

const PUBLIC: Self = Token::Public

const QUESTION: Self = Token::QuestionMark

const RBRACE: Self = Self::RBrace

const RBRACKET: Self = Self::RBracket

const READONLY: Self = Token::Readonly

const REQUIRE: Self = Token::Require

const RETURN: Self = Token::Return

const RPAREN: Self = Self::RParen

const RSHIFT: Self = Token::RShift

const RSHIFT_EQ: Self = Token::RShiftEq

const SATISFIES: Self = Token::Satisfies

const SEMI: Self = Token::Semi

const SET: Self = Token::Set

const STATIC: Self = Token::Static

const STRING: Self = Token::String

const SUPER: Self = Self::Super

const SWITCH: Self = Token::Switch

const SYMBOL: Self = Token::Symbol

const TARGET: Self = Token::Target

const THIS: Self = Token::This

const THROW: Self = Token::Throw

const TILDE: Self = Self::Tilde

const TRUE: Self = Token::True

const TRY: Self = Token::Try

const TYPE: Self = Token::Type

const TYPEOF: Self = Self::TypeOf

const UNDEFINED: Self = Token::Undefined

const UNIQUE: Self = Token::Unique

const UNKNOWN: Self = Token::Unknown

const USING: Self = Self::Using

const VAR: Self = Self::Var

const VOID: Self = Self::Void

const WHILE: Self = Token::While

const WITH: Self = Token::With

const YIELD: Self = Token::Yield

const ZERO_FILL_RSHIFT: Self = Token::ZeroFillRShift

const ZERO_FILL_RSHIFT_EQ: Self = Token::ZeroFillRShiftEq

type Buffer = Buffer<I>

type Lexer = Lexer<'a>

fn jsx_name(name: &str, lexer: &mut Lexer<'_>) -> Self

fn is_jsx_name(&self) -> bool

fn take_jsx_name(self, buffer: &mut Self::Buffer) -> Atom

fn str(value: Atom, raw: Atom, lexer: &mut Lexer<'a>) -> Self

fn template(cooked: LexResult<Atom>, raw: Atom, lexer: &mut Lexer<'a>) -> Self

fn regexp(content: Atom, flags: Atom, lexer: &mut Lexer<'a>) -> Self

fn num(value: f64, raw: Atom, lexer: &mut Lexer<'a>) -> Self

fn bigint(value: Box<BigInt>, raw: Atom, lexer: &mut Lexer<'a>) -> Self

fn unknown_ident(value: Atom, lexer: &mut Lexer<'a>) -> Self

fn is_reserved(&self, ctx: Context) -> bool

fn into_atom(self, lexer: &mut Lexer<'a>) -> Option<Atom>

fn is_error(&self) -> bool

fn take_error(self, buffer: &mut Self::Buffer) -> Error

fn is_str(&self) -> bool

fn is_str_raw_content(&self, content: &str, buffer: &Self::Buffer) -> bool

fn take_str(self, buffer: &mut Self::Buffer) -> (Atom, Atom)

fn is_num(&self) -> bool

fn take_num(self, buffer: &mut Self::Buffer) -> (f64, Atom)

fn is_bigint(&self) -> bool

fn take_bigint(self, buffer: &mut Self::Buffer) -> (Box<BigInt>, Atom)

fn is_word(&self) -> bool

fn take_word(self, buffer: &Self::Buffer) -> Option<Atom>

fn is_unknown_ident(&self) -> bool

fn take_unknown_ident(self, buffer: &mut Self::Buffer) -> Atom

fn is_keyword(&self) -> bool

fn is_known_ident(&self) -> bool

fn take_known_ident(&self) -> Atom

fn is_regexp(&self) -> bool

fn take_unknown_ident_ref<'b>(&'b self, buffer: &'b Self::Buffer) -> &'b Atom

fn is_template(&self) -> bool

fn take_template(self, buffer: &mut Self::Buffer) -> (LexResult<Atom>, Atom)

fn jsx_text(value: Atom, raw: Atom, lexer: &mut Self::Lexer) -> Self

fn is_jsx_text(&self) -> bool

fn take_jsx_text(self, buffer: &mut Self::Buffer) -> (Atom, Atom)

fn starts_expr(&self) -> bool

fn to_string(&self, buffer: &Self::Buffer) -> String

fn is_bin_op(&self) -> bool

fn as_assign_op(&self) -> Option<AssignOp>

fn as_bin_op(&self) -> Option<BinaryOp>

fn follows_keyword_let(&self) -> bool

fn is_assign_op(&self) -> bool

fn take_regexp(self, buffer: &mut Self::Buffer) -> (Atom, Atom)

fn shebang(value: Atom, lexer: &mut Self::Lexer) -> Self

fn is_shebang(&self) -> bool

fn take_shebang(self, buffer: &mut Self::Buffer) -> Atom

fn is_no_substitution_template_literal(&self) -> bool

fn is_template_head(&self) -> bool

fn is_less(&self) -> bool

fn is_less_eq(&self) -> bool

fn is_greater(&self) -> bool

fn is_colon(&self) -> bool

fn is_comma(&self) -> bool

fn is_equal(&self) -> bool

fn is_question(&self) -> bool

fn is_null(&self) -> bool

fn is_lshift(&self) -> bool

fn is_rshift(&self) -> bool

fn is_rshift_eq(&self) -> bool

fn is_greater_eq(&self) -> bool

fn is_true(&self) -> bool

fn is_false(&self) -> bool

fn is_enum(&self) -> bool

fn is_yield(&self) -> bool

fn is_let(&self) -> bool

fn is_var(&self) -> bool

fn is_static(&self) -> bool

fn is_extends(&self) -> bool

fn is_implements(&self) -> bool

fn is_interface(&self) -> bool

fn is_type(&self) -> bool

fn is_package(&self) -> bool

fn is_private(&self) -> bool

fn is_protected(&self) -> bool

fn is_public(&self) -> bool

fn is_readonly(&self) -> bool

fn is_await(&self) -> bool

fn is_break(&self) -> bool

fn is_continue(&self) -> bool

fn is_arrow(&self) -> bool

fn is_this(&self) -> bool

fn is_super(&self) -> bool

fn is_using(&self) -> bool

fn is_backquote(&self) -> bool

fn is_lparen(&self) -> bool

fn is_rparen(&self) -> bool

fn is_lbracket(&self) -> bool

fn is_rbracket(&self) -> bool

fn is_lbrace(&self) -> bool

fn is_rbrace(&self) -> bool

fn is_function(&self) -> bool

fn is_class(&self) -> bool

fn is_if(&self) -> bool

fn is_return(&self) -> bool

fn is_switch(&self) -> bool

fn is_throw(&self) -> bool

fn is_catch(&self) -> bool

fn is_finally(&self) -> bool

fn is_try(&self) -> bool

fn is_with(&self) -> bool

fn is_while(&self) -> bool

fn is_new(&self) -> bool

fn is_ident_ref(&self, ctx: Context) -> bool

fn is_import(&self) -> bool

fn is_export(&self) -> bool

fn is_dot(&self) -> bool

fn is_do(&self) -> bool

fn is_for(&self) -> bool

fn is_from(&self) -> bool

fn is_dotdotdot(&self) -> bool

fn is_plus(&self) -> bool

fn is_minus(&self) -> bool

fn is_bang(&self) -> bool

fn is_tilde(&self) -> bool

fn is_plus_plus(&self) -> bool

fn is_minus_minus(&self) -> bool

fn is_delete(&self) -> bool

fn is_typeof(&self) -> bool

fn is_of(&self) -> bool

fn is_void(&self) -> bool

fn is_hash(&self) -> bool

fn is_in(&self) -> bool

fn is_const(&self) -> bool

fn is_star(&self) -> bool

fn is_mod(&self) -> bool

fn is_semi(&self) -> bool

fn is_slash(&self) -> bool

fn is_slash_eq(&self) -> bool

fn is_jsx_tag_start(&self) -> bool

fn is_jsx_tag_end(&self) -> bool

fn is_asserts(&self) -> bool

fn is_is(&self) -> bool

fn is_as(&self) -> bool

fn is_satisfies(&self) -> bool

fn is_instanceof(&self) -> bool

fn is_async(&self) -> bool

fn is_case(&self) -> bool

fn is_default(&self) -> bool

fn is_debugger(&self) -> bool

fn is_bit_and(&self) -> bool

fn is_bit_or(&self) -> bool

fn is_exp(&self) -> bool

fn is_eof(&self) -> bool

impl<I: Tokens<TokenAndSpan>> Tokens<TokenAndSpan> for Capturing<I>

type Checkpoint = CapturingCheckpoint<I>

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx(&self) -> Context

fn ctx_mut(&mut self) -> &mut Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn set_expr_allowed(&mut self, allow: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, c: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn end_pos(&self) -> BytePos

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn update_token_flags(&mut self, f: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

fn start_pos(&self) -> BytePos

impl<'a> Tokens<TokenAndSpan> for Lexer<'a>

type Checkpoint = LexerCheckpoint

fn checkpoint_save(&self) -> Self::Checkpoint

fn checkpoint_load(&mut self, checkpoint: Self::Checkpoint)

fn set_ctx(&mut self, ctx: Context)

fn ctx(&self) -> Context

fn ctx_mut(&mut self) -> &mut Context

fn syntax(&self) -> SyntaxFlags

fn target(&self) -> EsVersion

fn start_pos(&self) -> BytePos

fn set_expr_allowed(&mut self, _: bool)

fn set_next_regexp(&mut self, start: Option<BytePos>)

fn token_context(&self) -> &TokenContexts

fn token_context_mut(&mut self) -> &mut TokenContexts

fn set_token_context(&mut self, _: TokenContexts)

fn add_error(&mut self, error: Error)

Implementors should use Rc<RefCell<Vec>>.

fn add_module_mode_error(&mut self, error: Error)

Add an error which is valid syntax in script mode.

fn take_errors(&mut self) -> Vec<Error>

fn take_script_module_errors(&mut self) -> Vec<Error>

If the program was parsed as a script, this contains the module errors should the program be identified as a module in the future.

fn end_pos(&self) -> BytePos

fn update_token_flags(&mut self, f: impl FnOnce(&mut TokenFlags))

fn token_flags(&self) -> TokenFlags

impl Copy for TokenAndSpan