swc_ecma_parser/lexer/
token.rs

1use num_bigint::BigInt;
2use swc_atoms::{atom, Atom, Wtf8Atom};
3use swc_common::Span;
4use swc_ecma_ast::AssignOp;
5
6use super::LexResult;
7use crate::{
8    error::Error,
9    input::{Buffer, Tokens},
10    Context, Lexer,
11};
12
13#[derive(Debug, Clone)]
14pub enum TokenValue {
15    /// unknown ident, jsx name and shebang
16    Word(Atom),
17    Template {
18        raw: Atom,
19        cooked: LexResult<Wtf8Atom>,
20    },
21    // string, jsx text
22    Str {
23        value: Wtf8Atom,
24        raw: Atom,
25    },
26    // regexp
27    Regex {
28        value: Atom,
29        flags: Atom,
30    },
31    Num {
32        value: f64,
33        raw: Atom,
34    },
35    BigInt {
36        value: Box<num_bigint::BigInt>,
37        raw: Atom,
38    },
39    Error(crate::error::Error),
40}
41
42#[derive(Clone, Copy, PartialEq, Eq)]
43#[repr(u8)]
44pub enum Token {
45    // Single character tokens
46    /// `(`
47    LParen,
48    /// `)`
49    RParen,
50    /// `{`
51    LBrace,
52    /// `}`
53    RBrace,
54    /// `[`
55    LBracket,
56    /// `]`
57    RBracket,
58    /// `;`
59    Semi,
60    /// `,`
61    Comma,
62    /// `.`
63    Dot,
64    /// `:`
65    Colon,
66    /// `?`
67    QuestionMark,
68    /// `!`
69    Bang,
70    /// `~`
71    Tilde,
72    /// `+`
73    Plus,
74    /// `-`
75    Minus,
76    /// `*`
77    Asterisk,
78    /// `/`
79    Slash,
80    /// `%`
81    Percent,
82    /// `<`
83    Lt,
84    /// `>`
85    Gt,
86    /// `|`
87    Pipe,
88    /// `^`
89    Caret,
90    /// `&`
91    Ampersand,
92    /// `=`
93    Eq,
94    /// `@`
95    At,
96    /// `#`
97    Hash,
98    /// '`'
99    BackQuote,
100    /// `=>`
101    Arrow,
102    /// `...`
103    DotDotDot,
104
105    // Compound operators
106    /// `++`
107    PlusPlus,
108    /// `--`
109    MinusMinus,
110    /// `+=`
111    PlusEq,
112    /// `-=`
113    MinusEq,
114
115    // More compound operators and keywords
116    /// `*=`
117    MulEq,
118    /// `/=`
119    DivEq,
120    /// `%=`
121    ModEq,
122    /// `<<=`
123    LShiftEq,
124    /// `>>=`
125    RShiftEq,
126    /// `>>>=`
127    ZeroFillRShiftEq,
128    /// `|=`
129    BitOrEq,
130    /// `^=`
131    BitXorEq,
132    /// `&=`
133    BitAndEq,
134    /// `**=`
135    ExpEq,
136    /// `||=`
137    LogicalOrEq,
138    /// `&&=`
139    LogicalAndEq,
140    /// `??=`
141    NullishEq,
142    /// `?.`
143    OptionalChain,
144
145    /// `==`
146    EqEq,
147    /// `!=`
148    NotEq,
149    /// `===`
150    EqEqEq,
151    /// `!==`
152    NotEqEq,
153
154    /// `<=`
155    LtEq,
156    /// `>=`
157    GtEq,
158    /// `<<`
159    LShift,
160    /// `>>`
161    RShift,
162    /// `>>>`
163    ZeroFillRShift,
164
165    /// `**`
166    Exp,
167    /// `||`
168    LogicalOr,
169    /// `&&`
170    LogicalAnd,
171    /// `??`
172    NullishCoalescing,
173
174    /// `</`
175    LessSlash,
176    /// `${`
177    DollarLBrace,
178
179    // JSX-related tokens
180    JSXTagStart,
181    JSXTagEnd,
182
183    // Literals
184    Str,
185    Num,
186    BigInt,
187    Regex,
188    Template,
189    NoSubstitutionTemplateLiteral,
190    TemplateHead,
191    TemplateMiddle,
192    TemplateTail,
193    JSXName,
194    JSXText,
195    // Identifiers and keyword
196    Ident,
197    // Reserved keyword tokens
198    Await,
199    Break,
200    Case,
201    Catch,
202    Class,
203    Const,
204    Continue,
205    Debugger,
206    Default,
207    Delete,
208    Do,
209    Else,
210    Export,
211    Extends,
212    False,
213    Finally,
214    For,
215    Function,
216    If,
217    Import,
218    In,
219    InstanceOf,
220    Let,
221    New,
222    Null,
223    Return,
224    Super,
225    Switch,
226    This,
227    Throw,
228    True,
229    Try,
230    TypeOf,
231    Var,
232    Void,
233    While,
234    With,
235    Yield,
236    Module,
237
238    // TypeScript-related keywords
239    Abstract,
240    Any,
241    As,
242    Asserts,
243    Assert,
244    Async,
245    Bigint,
246    Boolean,
247    Constructor,
248    Declare,
249    Enum,
250    From,
251    Get,
252    Global,
253    Implements,
254    Interface,
255    Intrinsic,
256    Is,
257    Keyof,
258    Namespace,
259    Never,
260    Number,
261    Object,
262    Of,
263    Out,
264    Override,
265    Package,
266    Private,
267    Protected,
268    Public,
269    Readonly,
270    Require,
271    Set,
272    Static,
273    String,
274    Symbol,
275    Type,
276    Undefined,
277    Unique,
278    Unknown,
279    Using,
280    Accessor,
281    Infer,
282    Satisfies,
283    Meta,
284    Target,
285
286    // Special tokens
287    Shebang,
288    Error,
289    Eof,
290}
291
292impl Token {
293    #[inline(always)]
294    pub const fn is_ident_ref(self, ctx: Context) -> bool {
295        self.is_word() && !self.is_reserved(ctx)
296    }
297
298    pub const fn is_other_and_before_expr_is_false(self) -> bool {
299        !self.is_keyword()
300            && !self.is_bin_op()
301            && !self.before_expr()
302            && !matches!(
303                self,
304                Token::Template
305                    | Token::Dot
306                    | Token::Colon
307                    | Token::LBrace
308                    | Token::RParen
309                    | Token::Semi
310                    | Token::JSXName
311                    | Token::JSXText
312                    | Token::JSXTagStart
313                    | Token::JSXTagEnd
314                    | Token::Arrow
315            )
316    }
317
318    #[inline(always)]
319    pub const fn is_other_and_can_have_trailing_comment(self) -> bool {
320        matches!(
321            self,
322            Token::Num
323                | Token::Str
324                | Token::Ident
325                | Token::DollarLBrace
326                | Token::Regex
327                | Token::BigInt
328                | Token::JSXText
329                | Token::RBrace
330        ) || self.is_known_ident()
331    }
332}
333
334impl<'a> Token {
335    #[inline(always)]
336    pub fn jsx_name(name: &str, lexer: &mut crate::Lexer) -> Self {
337        let name = lexer.atoms.atom(name);
338        lexer.set_token_value(Some(TokenValue::Word(name)));
339        Token::JSXName
340    }
341
342    #[inline(always)]
343    pub fn take_jsx_name<I: Tokens>(self, buffer: &mut Buffer<I>) -> Atom {
344        buffer.expect_word_token_value()
345    }
346
347    #[inline(always)]
348    pub fn str(value: Wtf8Atom, raw: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
349        lexer.set_token_value(Some(TokenValue::Str { value, raw }));
350        Token::Str
351    }
352
353    #[inline(always)]
354    pub fn template(cooked: LexResult<Wtf8Atom>, raw: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
355        lexer.set_token_value(Some(TokenValue::Template { cooked, raw }));
356        Token::Template
357    }
358
359    #[inline(always)]
360    pub fn regexp(content: Atom, flags: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
361        lexer.set_token_value(Some(TokenValue::Regex {
362            value: content,
363            flags,
364        }));
365        Token::Regex
366    }
367
368    #[inline(always)]
369    pub fn num(value: f64, raw: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
370        lexer.set_token_value(Some(TokenValue::Num { value, raw }));
371        Self::Num
372    }
373
374    #[inline(always)]
375    pub fn bigint(value: Box<num_bigint::BigInt>, raw: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
376        lexer.set_token_value(Some(TokenValue::BigInt { value, raw }));
377        Self::BigInt
378    }
379
380    #[inline(always)]
381    pub fn unknown_ident(value: Atom, lexer: &mut crate::Lexer<'a>) -> Self {
382        lexer.set_token_value(Some(TokenValue::Word(value)));
383        Token::Ident
384    }
385
386    #[inline(always)]
387    pub fn into_atom(self, lexer: &mut crate::Lexer<'a>) -> Option<Atom> {
388        let value = lexer.get_token_value();
389        self.as_word_atom(value)
390    }
391
392    #[inline(always)]
393    pub fn take_error<I: Tokens>(self, buffer: &mut Buffer<I>) -> Error {
394        buffer.expect_error_token_value()
395    }
396
397    #[inline(always)]
398    pub fn is_str_raw_content<I: Tokens>(self, content: &str, buffer: &Buffer<I>) -> bool {
399        self == Token::Str
400            && if let Some(TokenValue::Str { raw, .. }) = buffer.get_token_value() {
401                raw == content
402            } else {
403                unreachable!()
404            }
405    }
406
407    #[inline(always)]
408    pub fn take_str<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Wtf8Atom, Atom) {
409        buffer.expect_string_token_value()
410    }
411
412    #[inline(always)]
413    pub fn take_num<I: Tokens>(self, buffer: &mut Buffer<I>) -> (f64, Atom) {
414        buffer.expect_number_token_value()
415    }
416
417    #[inline(always)]
418    pub fn take_bigint<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Box<BigInt>, Atom) {
419        buffer.expect_bigint_token_value()
420    }
421
422    #[inline]
423    pub fn take_word<I: Tokens>(self, buffer: &Buffer<I>) -> Option<Atom> {
424        self.as_word_atom(buffer.get_token_value())
425    }
426
427    #[inline(always)]
428    pub fn take_unknown_ident<I: Tokens>(self, buffer: &mut Buffer<I>) -> Atom {
429        buffer.expect_word_token_value()
430    }
431
432    #[inline(always)]
433    pub fn take_known_ident(self) -> Atom {
434        self.as_known_ident_atom().unwrap()
435    }
436
437    #[inline(always)]
438    pub fn take_unknown_ident_ref<'b, I: Tokens>(&'b self, buffer: &'b Buffer<I>) -> &'b Atom {
439        buffer.expect_word_token_value_ref()
440    }
441
442    #[inline(always)]
443    pub fn take_template<I: Tokens>(self, buffer: &mut Buffer<I>) -> (LexResult<Wtf8Atom>, Atom) {
444        buffer.expect_template_token_value()
445    }
446
447    #[inline(always)]
448    pub fn jsx_text(value: Wtf8Atom, raw: Atom, lexer: &mut Lexer) -> Self {
449        lexer.set_token_value(Some(TokenValue::Str { value, raw }));
450        Token::JSXText
451    }
452
453    #[inline(always)]
454    pub fn take_jsx_text<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Atom, Atom) {
455        let (value, raw) = buffer.expect_string_token_value();
456        // SAFETY: We set value as Atom in `jsx_text` method.
457        (value.as_atom().cloned().unwrap(), raw)
458    }
459
460    #[inline(always)]
461    pub fn take_regexp<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Atom, Atom) {
462        buffer.expect_regex_token_value()
463    }
464
465    #[inline(always)]
466    pub fn shebang(value: Atom, lexer: &mut Lexer) -> Self {
467        lexer.set_token_value(Some(TokenValue::Word(value)));
468        Token::Shebang
469    }
470
471    #[inline(always)]
472    pub fn take_shebang<I: Tokens>(self, buffer: &mut Buffer<I>) -> Atom {
473        buffer.expect_word_token_value()
474    }
475}
476
477impl std::fmt::Debug for Token {
478    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
479        let s = match self {
480            Token::Str => "<string literal>",
481            Token::Num => "<number literal>",
482            Token::BigInt => "<bigint literal>",
483            Token::Regex => "<regexp literal>",
484            Token::Template | Token::NoSubstitutionTemplateLiteral => "<template literal>",
485            Token::TemplateHead => "<template head `...${ >",
486            Token::TemplateMiddle => "<template middle ...${ >",
487            Token::TemplateTail => "<template tail ` >",
488            Token::JSXName => "<jsx name>",
489            Token::JSXText => "<jsx text>",
490            Token::Ident => "<identifier>",
491            Token::Error => "<error>",
492            _ => &self.to_string(None),
493        };
494        f.write_str(s)
495    }
496}
497
498impl Token {
499    pub const fn is_reserved(self, ctx: Context) -> bool {
500        match self {
501            Token::Let | Token::Static => ctx.contains(Context::Strict),
502            Token::Await => {
503                ctx.contains(Context::InAsync)
504                    || ctx.contains(Context::InStaticBlock)
505                    || ctx.contains(Context::Strict)
506            }
507            Token::Yield => ctx.contains(Context::InGenerator) || ctx.contains(Context::Strict),
508
509            Token::Null
510            | Token::True
511            | Token::False
512            | Token::Break
513            | Token::Case
514            | Token::Catch
515            | Token::Continue
516            | Token::Debugger
517            | Token::Default
518            | Token::Do
519            | Token::Export
520            | Token::Else
521            | Token::Finally
522            | Token::For
523            | Token::Function
524            | Token::If
525            | Token::Return
526            | Token::Switch
527            | Token::Throw
528            | Token::Try
529            | Token::Var
530            | Token::Const
531            | Token::While
532            | Token::With
533            | Token::New
534            | Token::This
535            | Token::Super
536            | Token::Class
537            | Token::Extends
538            | Token::Import
539            | Token::In
540            | Token::InstanceOf
541            | Token::TypeOf
542            | Token::Void
543            | Token::Delete => true,
544
545            // Future reserved word
546            Token::Enum => true,
547
548            Token::Implements
549            | Token::Package
550            | Token::Protected
551            | Token::Interface
552            | Token::Private
553            | Token::Public
554                if ctx.contains(Context::Strict) =>
555            {
556                true
557            }
558
559            _ => false,
560        }
561    }
562
563    #[cold]
564    pub fn to_string(self, value: Option<&TokenValue>) -> String {
565        match self {
566            Token::LParen => "(",
567            Token::RParen => ")",
568            Token::LBrace => "{",
569            Token::RBrace => "}",
570            Token::LBracket => "[",
571            Token::RBracket => "]",
572            Token::Semi => ";",
573            Token::Comma => ",",
574            Token::Dot => ".",
575            Token::Colon => ":",
576            Token::QuestionMark => "?",
577            Token::Bang => "!",
578            Token::Tilde => "~",
579            Token::Plus => "+",
580            Token::Minus => "-",
581            Token::Asterisk => "*",
582            Token::Slash => "/",
583            Token::Percent => "%",
584            Token::Lt => "<",
585            Token::Gt => ">",
586            Token::Pipe => "|",
587            Token::Caret => "^",
588            Token::Ampersand => "&",
589            Token::Eq => "=",
590            Token::At => "@",
591            Token::Hash => "#",
592            Token::BackQuote => "`",
593            Token::Arrow => "=>",
594            Token::DotDotDot => "...",
595            Token::PlusPlus => "++",
596            Token::MinusMinus => "--",
597            Token::PlusEq => "+",
598            Token::MinusEq => "-",
599            Token::MulEq => "*",
600            Token::DivEq => "/=",
601            Token::ModEq => "%=",
602            Token::LShiftEq => "<<=",
603            Token::RShiftEq => ">>=",
604            Token::ZeroFillRShiftEq => ">>>=",
605            Token::BitOrEq => "|=",
606            Token::BitXorEq => "^=",
607            Token::BitAndEq => "&=",
608            Token::ExpEq => "**=",
609            Token::LogicalOrEq => "||=",
610            Token::LogicalAndEq => "&&=",
611            Token::NullishEq => "??=",
612            Token::OptionalChain => "?.",
613            Token::EqEq => "==",
614            Token::NotEq => "!=",
615            Token::EqEqEq => "===",
616            Token::NotEqEq => "!==",
617            Token::LtEq => "<=",
618            Token::GtEq => ">=",
619            Token::LShift => "<<",
620            Token::RShift => ">>",
621            Token::ZeroFillRShift => ">>>",
622            Token::Exp => "**",
623            Token::LogicalOr => "||",
624            Token::LogicalAnd => "&&",
625            Token::NullishCoalescing => "??",
626            Token::DollarLBrace => "${",
627            Token::JSXTagStart => "jsx tag start",
628            Token::JSXTagEnd => "jsx tag end",
629            Token::JSXText => {
630                let Some(TokenValue::Str { raw, .. }) = value else {
631                    unreachable!("{:#?}", value)
632                };
633                return format!("jsx text ({raw})");
634            }
635            Token::Str => {
636                let Some(TokenValue::Str { value, raw, .. }) = value else {
637                    unreachable!("{:#?}", value)
638                };
639                return format!("string literal ({value:?}, {raw})");
640            }
641            Token::Num => {
642                let Some(TokenValue::Num { value, raw, .. }) = value else {
643                    unreachable!("{:#?}", value)
644                };
645                return format!("numeric literal ({value}, {raw})");
646            }
647            Token::BigInt => {
648                let Some(TokenValue::BigInt { value, raw, .. }) = value else {
649                    unreachable!("{:#?}", value)
650                };
651                return format!("bigint literal ({value}, {raw})");
652            }
653            Token::Regex => {
654                let Some(TokenValue::Regex { value, flags, .. }) = value else {
655                    unreachable!("{:#?}", value)
656                };
657                return format!("regexp literal ({value}, {flags})");
658            }
659            Token::Template => {
660                let Some(TokenValue::Template { raw, .. }) = value else {
661                    unreachable!("{:#?}", value)
662                };
663                return format!("template token ({raw})");
664            }
665            Token::JSXName => {
666                let Some(TokenValue::Word(w)) = value else {
667                    unreachable!("{:#?}", value)
668                };
669                return format!("jsx name ({w})");
670            }
671            Token::Error => {
672                let Some(TokenValue::Error(e)) = value else {
673                    unreachable!("{:#?}", value)
674                };
675                return format!("<lexing error: {e:?}>");
676            }
677            Token::Ident => {
678                let Some(TokenValue::Word(w)) = value else {
679                    unreachable!("{:#?}", value)
680                };
681                w.as_ref()
682            }
683            Token::NoSubstitutionTemplateLiteral
684            | Token::TemplateHead
685            | Token::TemplateMiddle
686            | Token::TemplateTail => {
687                let Some(TokenValue::Template { raw, .. }) = value else {
688                    unreachable!("{:#?}", value)
689                };
690                raw
691            }
692            Token::Await => "await",
693            Token::Break => "break",
694            Token::Case => "case",
695            Token::Catch => "catch",
696            Token::Class => "class",
697            Token::Const => "const",
698            Token::Continue => "continue",
699            Token::Debugger => "debugger",
700            Token::Default => "default",
701            Token::Delete => "delete",
702            Token::Do => "do",
703            Token::Else => "else",
704            Token::Export => "export",
705            Token::Extends => "extends",
706            Token::False => "false",
707            Token::Finally => "finally",
708            Token::For => "for",
709            Token::Function => "function",
710            Token::If => "if",
711            Token::Import => "import",
712            Token::In => "in",
713            Token::InstanceOf => "instanceOf",
714            Token::Let => "let",
715            Token::New => "new",
716            Token::Null => "null",
717            Token::Return => "return",
718            Token::Super => "super",
719            Token::Switch => "switch",
720            Token::This => "this",
721            Token::Throw => "throw",
722            Token::True => "true",
723            Token::Try => "try",
724            Token::TypeOf => "typeOf",
725            Token::Var => "var",
726            Token::Void => "void",
727            Token::While => "while",
728            Token::With => "with",
729            Token::Yield => "yield",
730            Token::Module => "module",
731            Token::Abstract => "abstract",
732            Token::Any => "any",
733            Token::As => "as",
734            Token::Asserts => "asserts",
735            Token::Assert => "assert",
736            Token::Async => "async",
737            Token::Bigint => "bigint",
738            Token::Boolean => "boolean",
739            Token::Constructor => "constructor",
740            Token::Declare => "declare",
741            Token::Enum => "enum",
742            Token::From => "from",
743            Token::Get => "get",
744            Token::Global => "global",
745            Token::Implements => "implements",
746            Token::Interface => "interface",
747            Token::Intrinsic => "intrinsic",
748            Token::Is => "is",
749            Token::Keyof => "keyof",
750            Token::Namespace => "namespace",
751            Token::Never => "never",
752            Token::Number => "number",
753            Token::Object => "object",
754            Token::Of => "of",
755            Token::Out => "out",
756            Token::Override => "override",
757            Token::Package => "package",
758            Token::Private => "private",
759            Token::Protected => "protected",
760            Token::Public => "public",
761            Token::Readonly => "readonly",
762            Token::Require => "require",
763            Token::Set => "set",
764            Token::Static => "static",
765            Token::String => "string",
766            Token::Symbol => "symbol",
767            Token::Type => "type",
768            Token::Undefined => "undefined",
769            Token::Unique => "unique",
770            Token::Unknown => "unknown",
771            Token::Using => "using",
772            Token::Accessor => "accessor",
773            Token::Infer => "infer",
774            Token::Satisfies => "satisfies",
775            Token::Meta => "meta",
776            Token::Target => "target",
777            Token::Shebang => "#!",
778            Token::LessSlash => "</",
779            Token::Eof => "<eof>",
780        }
781        .to_string()
782    }
783
784    pub fn as_keyword_atom(self) -> Option<Atom> {
785        let atom = match self {
786            Token::Await => atom!("await"),
787            Token::Break => atom!("break"),
788            Token::Case => atom!("case"),
789            Token::Catch => atom!("catch"),
790            Token::Class => atom!("class"),
791            Token::Const => atom!("const"),
792            Token::Continue => atom!("continue"),
793            Token::Debugger => atom!("debugger"),
794            Token::Default => atom!("default"),
795            Token::Delete => atom!("delete"),
796            Token::Do => atom!("do"),
797            Token::Else => atom!("else"),
798            Token::Export => atom!("export"),
799            Token::Extends => atom!("extends"),
800            Token::Finally => atom!("finally"),
801            Token::For => atom!("for"),
802            Token::Function => atom!("function"),
803            Token::If => atom!("if"),
804            Token::Import => atom!("import"),
805            Token::In => atom!("in"),
806            Token::InstanceOf => atom!("instanceof"),
807            Token::Let => atom!("let"),
808            Token::New => atom!("new"),
809            Token::Return => atom!("return"),
810            Token::Super => atom!("super"),
811            Token::Switch => atom!("switch"),
812            Token::This => atom!("this"),
813            Token::Throw => atom!("throw"),
814            Token::Try => atom!("try"),
815            Token::TypeOf => atom!("typeof"),
816            Token::Var => atom!("var"),
817            Token::Void => atom!("void"),
818            Token::While => atom!("while"),
819            Token::With => atom!("with"),
820            Token::Yield => atom!("yield"),
821            Token::Module => atom!("module"),
822            _ => return None,
823        };
824        Some(atom)
825    }
826
827    #[inline(always)]
828    pub const fn is_keyword(self) -> bool {
829        let t = self as u8;
830        t >= Token::Await as u8 && t <= Token::Module as u8
831    }
832
833    pub fn as_known_ident_atom(self) -> Option<Atom> {
834        let atom = match self {
835            Token::Abstract => atom!("abstract"),
836            Token::Any => atom!("any"),
837            Token::As => atom!("as"),
838            Token::Asserts => atom!("asserts"),
839            Token::Assert => atom!("assert"),
840            Token::Async => atom!("async"),
841            Token::Bigint => atom!("bigint"),
842            Token::Boolean => atom!("boolean"),
843            Token::Constructor => atom!("constructor"),
844            Token::Declare => atom!("declare"),
845            Token::Enum => atom!("enum"),
846            Token::From => atom!("from"),
847            Token::Get => atom!("get"),
848            Token::Global => atom!("global"),
849            Token::Implements => atom!("implements"),
850            Token::Interface => atom!("interface"),
851            Token::Intrinsic => atom!("intrinsic"),
852            Token::Is => atom!("is"),
853            Token::Keyof => atom!("keyof"),
854            Token::Namespace => atom!("namespace"),
855            Token::Never => atom!("never"),
856            Token::Number => atom!("number"),
857            Token::Object => atom!("object"),
858            Token::Of => atom!("of"),
859            Token::Out => atom!("out"),
860            Token::Override => atom!("override"),
861            Token::Package => atom!("package"),
862            Token::Private => atom!("private"),
863            Token::Protected => atom!("protected"),
864            Token::Public => atom!("public"),
865            Token::Readonly => atom!("readonly"),
866            Token::Require => atom!("require"),
867            Token::Set => atom!("set"),
868            Token::Static => atom!("static"),
869            Token::String => atom!("string"),
870            Token::Symbol => atom!("symbol"),
871            Token::Type => atom!("type"),
872            Token::Undefined => atom!("undefined"),
873            Token::Unique => atom!("unique"),
874            Token::Unknown => atom!("unknown"),
875            Token::Using => atom!("using"),
876            Token::Accessor => atom!("accessor"),
877            Token::Infer => atom!("infer"),
878            Token::Satisfies => atom!("satisfies"),
879            Token::Meta => atom!("meta"),
880            Token::Target => atom!("target"),
881            _ => return None,
882        };
883        Some(atom)
884    }
885
886    #[inline(always)]
887    pub const fn is_known_ident(self) -> bool {
888        let t = self as u8;
889        t >= Token::Abstract as u8 && t <= Token::Target as u8
890    }
891
892    pub const fn is_word(self) -> bool {
893        matches!(
894            self,
895            Token::Null | Token::True | Token::False | Token::Ident
896        ) || self.is_known_ident()
897            || self.is_keyword()
898    }
899
900    pub fn as_word_atom(self, value: Option<&TokenValue>) -> Option<Atom> {
901        match self {
902            Token::Null => Some(atom!("null")),
903            Token::True => Some(atom!("true")),
904            Token::False => Some(atom!("false")),
905            Token::Ident => {
906                let Some(TokenValue::Word(w)) = value else {
907                    unreachable!("{:#?}", value)
908                };
909                Some(w.clone())
910            }
911            _ => self
912                .as_known_ident_atom()
913                .or_else(|| self.as_keyword_atom()),
914        }
915    }
916
917    #[inline(always)]
918    pub const fn is_bin_op(self) -> bool {
919        let t = self as u8;
920        (t >= Token::EqEq as u8 && t <= Token::NullishCoalescing as u8)
921            || (t >= Token::Plus as u8 && t <= Token::Ampersand as u8)
922    }
923
924    pub const fn as_bin_op(self) -> Option<swc_ecma_ast::BinaryOp> {
925        match self {
926            Token::EqEq => Some(swc_ecma_ast::BinaryOp::EqEq),
927            Token::NotEq => Some(swc_ecma_ast::BinaryOp::NotEq),
928            Token::EqEqEq => Some(swc_ecma_ast::BinaryOp::EqEqEq),
929            Token::NotEqEq => Some(swc_ecma_ast::BinaryOp::NotEqEq),
930            Token::Lt => Some(swc_ecma_ast::BinaryOp::Lt),
931            Token::LtEq => Some(swc_ecma_ast::BinaryOp::LtEq),
932            Token::Gt => Some(swc_ecma_ast::BinaryOp::Gt),
933            Token::GtEq => Some(swc_ecma_ast::BinaryOp::GtEq),
934            Token::LShift => Some(swc_ecma_ast::BinaryOp::LShift),
935            Token::RShift => Some(swc_ecma_ast::BinaryOp::RShift),
936            Token::ZeroFillRShift => Some(swc_ecma_ast::BinaryOp::ZeroFillRShift),
937            Token::Plus => Some(swc_ecma_ast::BinaryOp::Add),
938            Token::Minus => Some(swc_ecma_ast::BinaryOp::Sub),
939            Token::Asterisk => Some(swc_ecma_ast::BinaryOp::Mul),
940            Token::Slash => Some(swc_ecma_ast::BinaryOp::Div),
941            Token::Percent => Some(swc_ecma_ast::BinaryOp::Mod),
942            Token::Pipe => Some(swc_ecma_ast::BinaryOp::BitOr),
943            Token::Caret => Some(swc_ecma_ast::BinaryOp::BitXor),
944            Token::Ampersand => Some(swc_ecma_ast::BinaryOp::BitAnd),
945            Token::LogicalOr => Some(swc_ecma_ast::BinaryOp::LogicalOr),
946            Token::LogicalAnd => Some(swc_ecma_ast::BinaryOp::LogicalAnd),
947            // Token::In => Some(swc_ecma_ast::BinaryOp::In),
948            // Token::InstanceOf => Some(swc_ecma_ast::BinaryOp::InstanceOf),
949            Token::Exp => Some(swc_ecma_ast::BinaryOp::Exp),
950            Token::NullishCoalescing => Some(swc_ecma_ast::BinaryOp::NullishCoalescing),
951            _ => None,
952        }
953    }
954
955    #[inline(always)]
956    pub const fn is_assign_op(self) -> bool {
957        let t = self as u8;
958        matches!(self, Token::Eq) || (t >= Token::PlusEq as u8 && t <= Token::NullishEq as u8)
959    }
960
961    pub fn as_assign_op(self) -> Option<swc_ecma_ast::AssignOp> {
962        match self {
963            Self::Eq => Some(AssignOp::Assign),
964            Self::PlusEq => Some(AssignOp::AddAssign),
965            Self::MinusEq => Some(AssignOp::SubAssign),
966            Self::MulEq => Some(AssignOp::MulAssign),
967            Self::DivEq => Some(AssignOp::DivAssign),
968            Self::ModEq => Some(AssignOp::ModAssign),
969            Self::LShiftEq => Some(AssignOp::LShiftAssign),
970            Self::RShiftEq => Some(AssignOp::RShiftAssign),
971            Self::ZeroFillRShiftEq => Some(AssignOp::ZeroFillRShiftAssign),
972            Self::BitOrEq => Some(AssignOp::BitOrAssign),
973            Self::BitXorEq => Some(AssignOp::BitXorAssign),
974            Self::BitAndEq => Some(AssignOp::BitAndAssign),
975            Self::ExpEq => Some(AssignOp::ExpAssign),
976            Self::LogicalAndEq => Some(AssignOp::AndAssign),
977            Self::LogicalOrEq => Some(AssignOp::OrAssign),
978            Self::NullishEq => Some(AssignOp::NullishAssign),
979            _ => None,
980        }
981    }
982
983    #[inline(always)]
984    pub const fn before_expr(self) -> bool {
985        match self {
986            Self::Await
987            | Self::Case
988            | Self::Default
989            | Self::Do
990            | Self::Else
991            | Self::Return
992            | Self::Throw
993            | Self::New
994            | Self::Extends
995            | Self::Yield
996            | Self::In
997            | Self::InstanceOf
998            | Self::TypeOf
999            | Self::Void
1000            | Self::Delete
1001            | Self::Arrow
1002            | Self::DotDotDot
1003            | Self::Bang
1004            | Self::LParen
1005            | Self::LBrace
1006            | Self::LBracket
1007            | Self::Semi
1008            | Self::Comma
1009            | Self::Colon
1010            | Self::DollarLBrace
1011            | Self::QuestionMark
1012            | Self::PlusPlus
1013            | Self::MinusMinus
1014            | Self::Tilde
1015            | Self::JSXText => true,
1016            _ => self.is_bin_op() || self.is_assign_op(),
1017        }
1018    }
1019
1020    #[inline(always)]
1021    pub const fn starts_expr(self) -> bool {
1022        matches!(
1023            self,
1024            Self::Ident
1025                | Self::JSXName
1026                | Self::Plus
1027                | Self::Minus
1028                | Self::Bang
1029                | Self::LParen
1030                | Self::LBrace
1031                | Self::LBracket
1032                | Self::TemplateHead
1033                | Self::NoSubstitutionTemplateLiteral
1034                | Self::DollarLBrace
1035                | Self::PlusPlus
1036                | Self::MinusMinus
1037                | Self::Tilde
1038                | Self::Str
1039                | Self::Regex
1040                | Self::Num
1041                | Self::BigInt
1042                | Self::JSXTagStart
1043                | Self::Await
1044                | Self::Function
1045                | Self::Throw
1046                | Self::New
1047                | Self::This
1048                | Self::Super
1049                | Self::Class
1050                | Self::Import
1051                | Self::Yield
1052                | Self::TypeOf
1053                | Self::Void
1054                | Self::Delete
1055                | Self::Null
1056                | Self::True
1057                | Self::False
1058                | Self::BackQuote
1059        ) || self.is_known_ident()
1060    }
1061
1062    #[inline(always)]
1063    pub const fn follows_keyword_let(self) -> bool {
1064        match self {
1065            Token::Let
1066            | Token::LBrace
1067            | Token::LBracket
1068            | Token::Ident
1069            | Token::Yield
1070            | Token::Await => true,
1071            _ if self.is_known_ident() => true,
1072            _ => false,
1073        }
1074    }
1075
1076    #[inline(always)]
1077    pub const fn should_rescan_into_gt_in_jsx(self) -> bool {
1078        matches!(
1079            self,
1080            Token::GtEq
1081                | Token::RShift
1082                | Token::RShiftEq
1083                | Token::ZeroFillRShift
1084                | Token::ZeroFillRShiftEq
1085        )
1086    }
1087}
1088
1089#[derive(Clone, Copy, Debug)]
1090pub struct TokenAndSpan {
1091    pub token: Token,
1092    /// Had a line break before this token?
1093    pub had_line_break: bool,
1094    pub span: Span,
1095}
1096
1097impl TokenAndSpan {
1098    #[inline(always)]
1099    pub fn new(token: Token, span: Span, had_line_break: bool) -> Self {
1100        Self {
1101            token,
1102            had_line_break,
1103            span,
1104        }
1105    }
1106}
1107
1108#[derive(Clone)]
1109pub struct NextTokenAndSpan {
1110    pub token_and_span: TokenAndSpan,
1111    pub value: Option<TokenValue>,
1112}
1113
1114impl NextTokenAndSpan {
1115    #[inline(always)]
1116    pub fn token(&self) -> Token {
1117        self.token_and_span.token
1118    }
1119
1120    #[inline(always)]
1121    pub fn span(&self) -> Span {
1122        self.token_and_span.span
1123    }
1124
1125    #[inline(always)]
1126    pub fn had_line_break(&self) -> bool {
1127        self.token_and_span.had_line_break
1128    }
1129}
swc_ecma_parser/lexer/token.rs

swc_ecma_parser/lexer/
token.rs
