1pub mod token {
3 use std::{
4 borrow::Cow,
5 fmt::{self, Debug, Display, Formatter},
6 };
7
8 use swc_atoms::{atom, Atom, AtomStore, Wtf8Atom};
9 use swc_common::{Span, Spanned};
10 use swc_ecma_ast::{AssignOp, BigIntValue, BinaryOp};
11
12 use self::{Keyword::*, Token::*};
13 use crate::{error::Error, lexer::LexResult};
14
15 macro_rules! define_known_ident {
16 (
17 $(
18 $name:ident => $value:tt,
19 )*
20 ) => {
21 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
22 #[non_exhaustive]
23 pub enum KnownIdent {
24 $(
25 $name
26 ),*
27 }
28
29 #[macro_export]
30 macro_rules! known_ident_token {
31 $(
32 ($value) => {
33 $crate::token::TokenKind::Word($crate::token::WordKind::Ident(
34 $crate::token::IdentKind::Known($crate::token::KnownIdent::$name),
35 ))
36 };
37 )*
38 }
39
40 #[macro_export]
41 macro_rules! known_ident {
42 $(
43 ($value) => {
44 $crate::token::KnownIdent::$name
45 };
46 )*
47 }
48 #[macro_export]
49 macro_rules! ident_like {
50 $(
51 ($value) => {
52 $crate::token::IdentLike::Known(
53 $crate::token::KnownIdent::$name
54 )
55 };
56 )*
57 }
58
59 static STR_TO_KNOWN_IDENT: phf::Map<&'static str, KnownIdent> = phf::phf_map! {
60 $(
61 $value => KnownIdent::$name,
62 )*
63 };
64
65 impl From<KnownIdent> for swc_atoms::Atom {
66
67 fn from(s: KnownIdent) -> Self {
68 match s {
69 $(
70 KnownIdent::$name => atom!($value),
71 )*
72 }
73 }
74 }
75 impl From<KnownIdent> for &'static str {
76
77 fn from(s: KnownIdent) -> Self {
78 match s {
79 $(
80 KnownIdent::$name => $value,
81 )*
82 }
83 }
84 }
85 };
86}
87
88 define_known_ident!(
89 Abstract => "abstract",
90 As => "as",
91 Async => "async",
92 From => "from",
93 Of => "of",
94 Type => "type",
95 Global => "global",
96 Static => "static",
97 Using => "using",
98 Readonly => "readonly",
99 Unique => "unique",
100 Keyof => "keyof",
101 Declare => "declare",
102 Enum => "enum",
103 Is => "is",
104 Infer => "infer",
105 Symbol => "symbol",
106 Undefined => "undefined",
107 Interface => "interface",
108 Implements => "implements",
109 Asserts => "asserts",
110 Require => "require",
111 Get => "get",
112 Set => "set",
113 Any => "any",
114 Intrinsic => "intrinsic",
115 Unknown => "unknown",
116 String => "string",
117 Object => "object",
118 Number => "number",
119 Bigint => "bigint",
120 Boolean => "boolean",
121 Never => "never",
122 Assert => "assert",
123 Namespace => "namespace",
124 Accessor => "accessor",
125 Meta => "meta",
126 Target => "target",
127 Satisfies => "satisfies",
128 Package => "package",
129 Protected => "protected",
130 Private => "private",
131 Public => "public",
132 );
133
134 impl std::str::FromStr for KnownIdent {
135 type Err = ();
136
137 fn from_str(s: &str) -> Result<Self, Self::Err> {
138 STR_TO_KNOWN_IDENT.get(s).cloned().ok_or(())
139 }
140 }
141
142 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
143 pub enum WordKind {
144 Keyword(Keyword),
145
146 Null,
147 True,
148 False,
149
150 Ident(IdentKind),
151 }
152
153 impl From<Keyword> for WordKind {
154 #[inline(always)]
155 fn from(kwd: Keyword) -> Self {
156 Self::Keyword(kwd)
157 }
158 }
159
160 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
161 pub enum IdentKind {
162 Known(KnownIdent),
163 Other,
164 }
165
166 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
167 pub enum TokenKind {
168 Word(WordKind),
169 Arrow,
170 Hash,
171 At,
172 Dot,
173 DotDotDot,
174 Bang,
175 LParen,
176 RParen,
177 LBracket,
178 RBracket,
179 LBrace,
180 RBrace,
181 Semi,
182 Comma,
183 BackQuote,
184 Template,
185 Colon,
186 BinOp(BinOpToken),
187 AssignOp(AssignOp),
188 DollarLBrace,
189 QuestionMark,
190 PlusPlus,
191 MinusMinus,
192 Tilde,
193 Str,
194 Regex,
196 Num,
197 BigInt,
198
199 JSXName,
200 JSXText,
201 JSXTagStart,
202 JSXTagEnd,
203
204 Shebang,
205 Error,
206 }
207
208 #[derive(Clone, PartialEq)]
209 pub enum Token {
210 Word(Word),
214
215 Arrow,
217
218 Hash,
220
221 At,
223 Dot,
225
226 DotDotDot,
228 Bang,
230
231 LParen,
233 RParen,
235 LBracket,
237 RBracket,
239 LBrace,
241 RBrace,
243
244 Semi,
246 Comma,
248
249 BackQuote,
251 Template {
252 raw: Atom,
253 cooked: LexResult<Wtf8Atom>,
254 },
255 Colon,
257 BinOp(BinOpToken),
258 AssignOp(AssignOp),
259
260 DollarLBrace,
262
263 QuestionMark,
265
266 PlusPlus,
268 MinusMinus,
270
271 Tilde,
273
274 Str {
276 value: Wtf8Atom,
277 raw: Atom,
278 },
279
280 Regex(Atom, Atom),
282
283 Num {
285 value: f64,
286 raw: Atom,
287 },
288
289 BigInt {
290 value: Box<BigIntValue>,
291 raw: Atom,
292 },
293
294 JSXName {
295 name: Atom,
296 },
297 JSXText {
298 value: Atom,
299 raw: Atom,
300 },
301 JSXTagStart,
302 JSXTagEnd,
303
304 Shebang(Atom),
305 Error(Error),
306 Eof,
307 }
308
309 impl Token {
310 pub fn kind(&self) -> TokenKind {
311 match self {
312 Self::Arrow => TokenKind::Arrow,
313 Self::Hash => TokenKind::Hash,
314 Self::At => TokenKind::At,
315 Self::Dot => TokenKind::Dot,
316 Self::DotDotDot => TokenKind::DotDotDot,
317 Self::Bang => TokenKind::Bang,
318 Self::LParen => TokenKind::LParen,
319 Self::RParen => TokenKind::RParen,
320 Self::LBracket => TokenKind::LBracket,
321 Self::RBracket => TokenKind::RBracket,
322 Self::LBrace => TokenKind::LBrace,
323 Self::RBrace => TokenKind::RBrace,
324 Self::Semi => TokenKind::Semi,
325 Self::Comma => TokenKind::Comma,
326 Self::BackQuote => TokenKind::BackQuote,
327 Self::Template { .. } => TokenKind::Template,
328 Self::Colon => TokenKind::Colon,
329 Self::BinOp(op) => TokenKind::BinOp(*op),
330 Self::AssignOp(op) => TokenKind::AssignOp(*op),
331 Self::DollarLBrace => TokenKind::DollarLBrace,
332 Self::QuestionMark => TokenKind::QuestionMark,
333 Self::PlusPlus => TokenKind::PlusPlus,
334 Self::MinusMinus => TokenKind::MinusMinus,
335 Self::Tilde => TokenKind::Tilde,
336 Self::Str { .. } => TokenKind::Str,
337 Self::Regex(..) => TokenKind::Regex,
338 Self::Num { .. } => TokenKind::Num,
339 Self::BigInt { .. } => TokenKind::BigInt,
340 Self::JSXName { .. } => TokenKind::JSXName,
341 Self::JSXText { .. } => TokenKind::JSXText,
342 Self::JSXTagStart => TokenKind::JSXTagStart,
343 Self::JSXTagEnd => TokenKind::JSXTagEnd,
344 Self::Shebang(..) => TokenKind::Shebang,
345 Self::Error(..) => TokenKind::Error,
346 Self::Eof => TokenKind::Error,
347 Self::Word(w) => TokenKind::Word(w.kind()),
348 }
349 }
350 }
351
352 impl TokenKind {
353 pub const fn before_expr(self) -> bool {
354 match self {
355 Self::Word(w) => w.before_expr(),
356 Self::BinOp(w) => w.before_expr(),
357 Self::Arrow
358 | Self::DotDotDot
359 | Self::Bang
360 | Self::LParen
361 | Self::LBrace
362 | Self::LBracket
363 | Self::Semi
364 | Self::Comma
365 | Self::Colon
366 | Self::AssignOp(..)
367 | Self::DollarLBrace
368 | Self::QuestionMark
369 | Self::PlusPlus
370 | Self::MinusMinus
371 | Self::Tilde
372 | Self::JSXText { .. } => true,
373 _ => false,
374 }
375 }
376
377 pub const fn starts_expr(self) -> bool {
378 match self {
379 Self::Word(w) => w.starts_expr(),
380 Self::BinOp(w) => w.starts_expr(),
381 Self::Bang
382 | Self::LParen
383 | Self::LBrace
384 | Self::LBracket
385 | Self::BackQuote
386 | Self::DollarLBrace
387 | Self::PlusPlus
388 | Self::MinusMinus
389 | Self::Tilde
390 | Self::Str
391 | Self::Regex
392 | Self::Num
393 | Self::BigInt
394 | Self::JSXTagStart => true,
395 _ => false,
396 }
397 }
398 }
399
400 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
401 pub enum BinOpToken {
402 EqEq,
404 NotEq,
406 EqEqEq,
408 NotEqEq,
410 Lt,
412 LtEq,
414 Gt,
416 GtEq,
418 LShift,
420 RShift,
422 ZeroFillRShift,
424
425 Add,
427 Sub,
429 Mul,
431 Div,
433 Mod,
435
436 BitOr,
438 BitXor,
440 BitAnd,
442
443 Exp,
451
452 LogicalOr,
454 LogicalAnd,
456
457 NullishCoalescing,
459 }
460
461 impl BinOpToken {
462 pub(crate) const fn starts_expr(self) -> bool {
463 matches!(self, Self::Add | Self::Sub)
464 }
465
466 pub const fn before_expr(self) -> bool {
467 true
468 }
469 }
470
471 #[derive(Debug, Clone, PartialEq)]
472 pub struct TokenAndSpan {
473 pub token: Token,
474 pub had_line_break: bool,
476 pub span: Span,
477 }
478
479 impl Spanned for TokenAndSpan {
480 #[inline]
481 fn span(&self) -> Span {
482 self.span
483 }
484 }
485
486 #[derive(Clone, PartialEq, Eq, Hash)]
487 pub enum Word {
488 Keyword(Keyword),
489
490 Null,
491 True,
492 False,
493
494 Ident(IdentLike),
495 }
496
497 #[derive(Clone, PartialEq, Eq, Hash)]
498 pub enum IdentLike {
499 Known(KnownIdent),
500 Other(Atom),
501 }
502
503 impl From<&'_ str> for IdentLike {
504 fn from(s: &str) -> Self {
505 s.parse::<KnownIdent>()
506 .map(Self::Known)
507 .unwrap_or_else(|_| Self::Other(s.into()))
508 }
509 }
510
511 impl IdentLike {
512 pub(crate) fn from_str(atoms: &mut AtomStore, s: &str) -> IdentLike {
513 s.parse::<KnownIdent>()
514 .map(Self::Known)
515 .unwrap_or_else(|_| Self::Other(atoms.atom(s)))
516 }
517 }
518
519 impl Word {
520 pub fn from_str(atoms: &mut AtomStore, s: &str) -> Self {
521 match s {
522 "null" => Word::Null,
523 "true" => Word::True,
524 "false" => Word::False,
525 "await" => Await.into(),
526 "break" => Break.into(),
527 "case" => Case.into(),
528 "catch" => Catch.into(),
529 "continue" => Continue.into(),
530 "debugger" => Debugger.into(),
531 "default" => Default_.into(),
532 "do" => Do.into(),
533 "export" => Export.into(),
534 "else" => Else.into(),
535 "finally" => Finally.into(),
536 "for" => For.into(),
537 "function" => Function.into(),
538 "if" => If.into(),
539 "return" => Return.into(),
540 "switch" => Switch.into(),
541 "throw" => Throw.into(),
542 "try" => Try.into(),
543 "var" => Var.into(),
544 "let" => Let.into(),
545 "const" => Const.into(),
546 "while" => While.into(),
547 "with" => With.into(),
548 "new" => New.into(),
549 "this" => This.into(),
550 "super" => Super.into(),
551 "class" => Class.into(),
552 "extends" => Extends.into(),
553 "import" => Import.into(),
554 "yield" => Yield.into(),
555 "in" => In.into(),
556 "instanceof" => InstanceOf.into(),
557 "typeof" => TypeOf.into(),
558 "void" => Void.into(),
559 "delete" => Delete.into(),
560 _ => Word::Ident(IdentLike::from_str(atoms, s)),
561 }
562 }
563
564 pub(crate) fn kind(&self) -> WordKind {
565 match self {
566 Word::Keyword(k) => WordKind::Keyword(*k),
567 Word::Null => WordKind::Null,
568 Word::True => WordKind::True,
569 Word::False => WordKind::False,
570 Word::Ident(IdentLike::Known(i)) => WordKind::Ident(IdentKind::Known(*i)),
571 Word::Ident(IdentLike::Other(..)) => WordKind::Ident(IdentKind::Other),
572 }
573 }
574 }
575
576 impl WordKind {
577 pub(crate) const fn before_expr(self) -> bool {
578 match self {
579 Self::Keyword(k) => k.before_expr(),
580 _ => false,
581 }
582 }
583
584 pub(crate) const fn starts_expr(self) -> bool {
585 match self {
586 Self::Keyword(k) => k.starts_expr(),
587 _ => true,
588 }
589 }
590 }
591
592 impl AsRef<str> for IdentLike {
593 fn as_ref(&self) -> &str {
594 match self {
595 IdentLike::Known(k) => (*k).into(),
596 IdentLike::Other(s) => s.as_ref(),
597 }
598 }
599 }
600
601 impl From<Keyword> for Word {
602 fn from(kwd: Keyword) -> Self {
603 Word::Keyword(kwd)
604 }
605 }
606
607 impl From<Word> for Atom {
608 fn from(w: Word) -> Self {
609 match w {
610 Word::Keyword(k) => match k {
611 Await => "await",
612 Break => "break",
613 Case => "case",
614 Catch => "catch",
615 Continue => "continue",
616 Debugger => "debugger",
617 Default_ => "default",
618 Do => "do",
619 Else => "else",
620
621 Finally => "finally",
622 For => "for",
623
624 Function => "function",
625
626 If => "if",
627
628 Return => "return",
629
630 Switch => "switch",
631
632 Throw => "throw",
633
634 Try => "try",
635 Var => "var",
636 Let => "let",
637 Const => "const",
638 While => "while",
639 With => "with",
640
641 New => "new",
642 This => "this",
643 Super => "super",
644
645 Class => "class",
646
647 Extends => "extends",
648
649 Export => "export",
650 Import => "import",
651
652 Yield => "yield",
653
654 In => "in",
655 InstanceOf => "instanceof",
656
657 TypeOf => "typeof",
658
659 Void => "void",
660
661 Delete => "delete",
662 }
663 .into(),
664
665 Word::Null => atom!("null"),
666 Word::True => atom!("true"),
667 Word::False => atom!("false"),
668
669 Word::Ident(w) => w.into(),
670 }
671 }
672 }
673
674 impl From<IdentLike> for Atom {
675 fn from(i: IdentLike) -> Self {
676 match i {
677 IdentLike::Known(i) => i.into(),
678 IdentLike::Other(i) => i,
679 }
680 }
681 }
682
683 impl Debug for Word {
684 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
685 match *self {
686 Word::Ident(ref s) => Display::fmt(s, f),
687 _ => {
688 let s: Atom = self.clone().into();
689 Display::fmt(&s, f)
690 }
691 }
692 }
693 }
694
695 impl Display for IdentLike {
696 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
697 match *self {
698 IdentLike::Known(ref s) => Display::fmt(s, f),
699 IdentLike::Other(ref s) => Display::fmt(s, f),
700 }
701 }
702 }
703
704 impl Display for KnownIdent {
705 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
706 let s: &'static str = (*self).into();
707
708 Display::fmt(s, f)
709 }
710 }
711
712 #[macro_export]
713 macro_rules! declare_keyword {
714 ($(
715 $name:ident => $value:tt,
716 )*) => {
717 impl Keyword {
718 pub fn into_atom(self) -> Atom {
719 match self {
720 $(Keyword::$name => atom!($value),)*
721 }
722 }
723 }
724 };
725}
726
727 declare_keyword!(
728 Await => "await",
729 Break => "break",
730 Case => "case",
731 Catch => "catch",
732 Continue => "continue",
733 Debugger => "debugger",
734 Default_ => "default",
735 Do => "do",
736 Else => "else",
737
738 Finally => "finally",
739 For => "for",
740
741 Function => "function",
742
743 If => "if",
744
745 Return => "return",
746
747 Switch => "switch",
748
749 Throw => "throw",
750
751 Try => "try",
752 Var => "var",
753 Let => "let",
754 Const => "const",
755 While => "while",
756 With => "with",
757
758 New => "new",
759 This => "this",
760 Super => "super",
761
762 Class => "class",
763
764 Extends => "extends",
765
766 Export => "export",
767 Import => "import",
768
769 Yield => "yield",
770
771 In => "in",
772 InstanceOf => "instanceof",
773
774 TypeOf => "typeof",
775
776 Void => "void",
777
778 Delete => "delete",
779 );
780
781 #[derive(Clone, Copy, PartialEq, Eq, Hash)]
783 pub enum Keyword {
784 Await,
786 Break,
787 Case,
788 Catch,
789 Continue,
790 Debugger,
791 Default_,
792 Do,
793 Else,
794
795 Finally,
796 For,
797
798 Function,
799
800 If,
801
802 Return,
803
804 Switch,
805
806 Throw,
807
808 Try,
809 Var,
810 Let,
811 Const,
812 While,
813 With,
814
815 New,
816 This,
817 Super,
818
819 Class,
820
821 Extends,
822
823 Export,
824 Import,
825
826 Yield,
828
829 In,
830 InstanceOf,
831 TypeOf,
832 Void,
833 Delete,
834 }
835
836 impl Keyword {
837 pub const fn before_expr(self) -> bool {
838 matches!(
839 self,
840 Self::Await
841 | Self::Case
842 | Self::Default_
843 | Self::Do
844 | Self::Else
845 | Self::Return
846 | Self::Throw
847 | Self::New
848 | Self::Extends
849 | Self::Yield
850 | Self::In
851 | Self::InstanceOf
852 | Self::TypeOf
853 | Self::Void
854 | Self::Delete
855 )
856 }
857
858 pub(crate) const fn starts_expr(self) -> bool {
859 matches!(
860 self,
861 Self::Await
862 | Self::Function
863 | Self::Throw
864 | Self::New
865 | Self::This
866 | Self::Super
867 | Self::Class
868 | Self::Import
869 | Self::Yield
870 | Self::TypeOf
871 | Self::Void
872 | Self::Delete
873 )
874 }
875 }
876
877 impl Debug for Keyword {
878 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
879 write!(f, "keyword '{}'", self.into_atom())?;
880
881 Ok(())
882 }
883 }
884
885 impl From<BinOpToken> for BinaryOp {
886 fn from(t: BinOpToken) -> Self {
887 use self::BinaryOp::*;
888 match t {
889 BinOpToken::EqEq => EqEq,
890 BinOpToken::NotEq => NotEq,
891 BinOpToken::EqEqEq => EqEqEq,
892 BinOpToken::NotEqEq => NotEqEq,
893 BinOpToken::Lt => Lt,
894 BinOpToken::LtEq => LtEq,
895 BinOpToken::Gt => Gt,
896 BinOpToken::GtEq => GtEq,
897 BinOpToken::LShift => LShift,
898 BinOpToken::RShift => RShift,
899 BinOpToken::ZeroFillRShift => ZeroFillRShift,
900 BinOpToken::Add => Add,
901 BinOpToken::Sub => Sub,
902 BinOpToken::Mul => Mul,
903 BinOpToken::Div => Div,
904 BinOpToken::Mod => Mod,
905 BinOpToken::BitOr => BitOr,
906 BinOpToken::BitXor => BitXor,
907 BinOpToken::BitAnd => BitAnd,
908 BinOpToken::LogicalOr => LogicalOr,
909 BinOpToken::LogicalAnd => LogicalAnd,
910 BinOpToken::Exp => Exp,
911 BinOpToken::NullishCoalescing => NullishCoalescing,
912 }
913 }
914 }
915
916 impl TokenKind {
917 pub fn follows_keyword_let(self, _strict: bool) -> bool {
921 match self {
922 Self::Word(WordKind::Keyword(Keyword::Let))
923 | TokenKind::LBrace
924 | TokenKind::LBracket
925 | Self::Word(WordKind::Ident(..))
926 | TokenKind::Word(WordKind::Keyword(Keyword::Yield))
927 | TokenKind::Word(WordKind::Keyword(Keyword::Await)) => true,
928 _ => false,
929 }
930 }
931 }
932
933 impl Word {
934 pub fn cow(&self) -> Cow<Atom> {
935 match self {
936 Word::Keyword(k) => Cow::Owned(k.into_atom()),
937 Word::Ident(IdentLike::Known(w)) => Cow::Owned((*w).into()),
938 Word::Ident(IdentLike::Other(w)) => Cow::Borrowed(w),
939 Word::False => Cow::Owned(atom!("false")),
940 Word::True => Cow::Owned(atom!("true")),
941 Word::Null => Cow::Owned(atom!("null")),
942 }
943 }
944 }
945
946 impl Debug for Token {
947 #[cold]
949 #[inline(never)]
950 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
951 match self {
952 Word(w) => write!(f, "{w:?}")?,
953 Arrow => write!(f, "=>")?,
954 Hash => write!(f, "#")?,
955 At => write!(f, "@")?,
956 Dot => write!(f, ".")?,
957 DotDotDot => write!(f, "...")?,
958 Bang => write!(f, "!")?,
959 LParen => write!(f, "(")?,
960 RParen => write!(f, ")")?,
961 LBracket => write!(f, "[")?,
962 RBracket => write!(f, "]")?,
963 LBrace => write!(f, "{{")?,
964 RBrace => write!(f, "}}")?,
965 Semi => write!(f, ";")?,
966 Comma => write!(f, ",")?,
967 BackQuote => write!(f, "`")?,
968 Template { raw, .. } => write!(f, "template token ({raw})")?,
969 Colon => write!(f, ":")?,
970 BinOp(op) => write!(f, "{}", BinaryOp::from(*op).as_str())?,
971 AssignOp(op) => write!(f, "{}", op.as_str())?,
972 DollarLBrace => write!(f, "${{")?,
973 QuestionMark => write!(f, "?")?,
974 PlusPlus => write!(f, "++")?,
975 MinusMinus => write!(f, "--")?,
976 Tilde => write!(f, "~")?,
977 Str { value, raw } => write!(f, "string literal ({value:?}, {raw})")?,
978 Regex(exp, flags) => write!(f, "regexp literal ({exp}, {flags})")?,
979 Num { value, raw, .. } => write!(f, "numeric literal ({value}, {raw})")?,
980 BigInt { value, raw } => write!(f, "bigint literal ({value}, {raw})")?,
981 JSXName { name } => write!(f, "jsx name ({name})")?,
982 JSXText { raw, .. } => write!(f, "jsx text ({raw})")?,
983 JSXTagStart => write!(f, "< (jsx tag start)")?,
984 JSXTagEnd => write!(f, "> (jsx tag end)")?,
985 Shebang(_) => write!(f, "#!")?,
986 Error(e) => write!(f, "<lexing error: {e:?}>")?,
987 Eof => write!(f, "<eof>")?,
988 }
989
990 Ok(())
991 }
992 }
993}