swc_ecma_lexer/
lib.rs

1//! # swc_ecma_lexer
2//!
3//! This crate provides a lexer for ECMAScript and TypeScript. It can ensure
4//! these tokens are correctly parsed.
5
6#![cfg_attr(docsrs, feature(doc_cfg))]
7#![cfg_attr(test, feature(test))]
8#![deny(clippy::all)]
9#![deny(unused)]
10#![allow(clippy::nonminimal_bool)]
11#![allow(clippy::too_many_arguments)]
12#![allow(clippy::unnecessary_unwrap)]
13#![allow(clippy::vec_box)]
14#![allow(clippy::wrong_self_convention)]
15#![allow(clippy::match_like_matches_macro)]
16#![allow(unexpected_cfgs)]
17
18pub mod common;
19pub mod lexer;
20mod parser;
21#[macro_use]
22pub mod token;
23pub mod error;
24pub mod input;
25mod utils;
26
27use common::parser::{buffer::Buffer, Parser as ParserTrait};
28pub use swc_common::input::StringInput;
29
30use self::common::{context::Context, parser::PResult};
31pub use self::{
32    common::syntax::{EsSyntax, Syntax, SyntaxFlags, TsSyntax},
33    input::Capturing,
34    lexer::{Lexer, TokenContext, TokenContexts, TokenType},
35    parser::Parser,
36};
37
38#[cfg(test)]
39fn with_test_sess<F, Ret>(src: &str, f: F) -> Result<Ret, ::testing::StdErr>
40where
41    F: FnOnce(&swc_common::errors::Handler, swc_common::input::StringInput<'_>) -> Result<Ret, ()>,
42{
43    use swc_common::FileName;
44
45    ::testing::run_test(false, |cm, handler| {
46        let fm = cm.new_source_file(FileName::Real("testing".into()).into(), src.to_string());
47
48        f(handler, (&*fm).into())
49    })
50}
51
52#[macro_export]
53macro_rules! tok {
54    ('`') => {
55        $crate::token::Token::BackQuote
56    };
57    // (';') => { Token::Semi };
58    ('@') => {
59        $crate::token::Token::At
60    };
61    ('#') => {
62        $crate::token::Token::Hash
63    };
64
65    ('&') => {
66        $crate::token::Token::BinOp($crate::token::BinOpToken::BitAnd)
67    };
68    ('|') => {
69        $crate::token::Token::BinOp($crate::token::BinOpToken::BitOr)
70    };
71    ('^') => {
72        $crate::token::Token::BinOp($crate::token::BinOpToken::BitXor)
73    };
74    ('+') => {
75        $crate::token::Token::BinOp($crate::token::BinOpToken::Add)
76    };
77    ('-') => {
78        $crate::token::Token::BinOp($crate::token::BinOpToken::Sub)
79    };
80    ("??") => {
81        $crate::token::Token::BinOp($crate::token::BinOpToken::NullishCoalescing)
82    };
83    ('~') => {
84        $crate::token::Token::Tilde
85    };
86    ('!') => {
87        $crate::token::Token::Bang
88    };
89    ("&&") => {
90        $crate::token::Token::BinOp($crate::token::BinOpToken::LogicalAnd)
91    };
92    ("||") => {
93        $crate::token::Token::BinOp($crate::token::BinOpToken::LogicalOr)
94    };
95    ("&&=") => {
96        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::AndAssign)
97    };
98    ("||=") => {
99        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::OrAssign)
100    };
101    ("??=") => {
102        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::NullishAssign)
103    };
104
105    ("==") => {
106        $crate::token::Token::BinOp($crate::token::BinOpToken::EqEq)
107    };
108    ("===") => {
109        $crate::token::Token::BinOp($crate::token::BinOpToken::EqEqEq)
110    };
111    ("!=") => {
112        $crate::token::Token::BinOp($crate::token::BinOpToken::NotEq)
113    };
114    ("!==") => {
115        $crate::token::Token::BinOp($crate::token::BinOpToken::NotEqEq)
116    };
117
118    (',') => {
119        $crate::token::Token::Comma
120    };
121    ('?') => {
122        $crate::token::Token::QuestionMark
123    };
124    (':') => {
125        $crate::token::Token::Colon
126    };
127    ('.') => {
128        $crate::token::Token::Dot
129    };
130    ("=>") => {
131        $crate::token::Token::Arrow
132    };
133    ("...") => {
134        $crate::token::Token::DotDotDot
135    };
136    ("${") => {
137        $crate::token::Token::DollarLBrace
138    };
139
140    ('+') => {
141        $crate::token::Token::BinOp($crate::token::BinOpToken::Add)
142    };
143    ('-') => {
144        $crate::token::Token::BinOp($crate::token::BinOpToken::Sub)
145    };
146    ('*') => {
147        $crate::token::Token::BinOp($crate::token::BinOpToken::Mul)
148    };
149    ('/') => {
150        $crate::token::Token::BinOp($crate::token::BinOpToken::Div)
151    };
152    ("/=") => {
153        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::DivAssign)
154    };
155    ('%') => {
156        $crate::token::Token::BinOp($crate::token::BinOpToken::Mod)
157    };
158    ('~') => {
159        $crate::token::Token::Tilde
160    };
161    ('<') => {
162        $crate::token::Token::BinOp($crate::token::BinOpToken::Lt)
163    };
164    ("<<") => {
165        $crate::token::Token::BinOp($crate::token::BinOpToken::LShift)
166    };
167    ("<=") => {
168        $crate::token::Token::BinOp($crate::token::BinOpToken::LtEq)
169    };
170    ("<<=") => {
171        $crate::token::Token::AssignOp($crate::token::AssignOp::LShiftAssign)
172    };
173    ('>') => {
174        $crate::token::Token::BinOp($crate::token::BinOpToken::Gt)
175    };
176    (">>") => {
177        $crate::token::Token::BinOp($crate::token::BinOpToken::RShift)
178    };
179    (">>>") => {
180        $crate::token::Token::BinOp($crate::token::BinOpToken::ZeroFillRShift)
181    };
182    (">=") => {
183        $crate::token::Token::BinOp($crate::token::BinOpToken::GtEq)
184    };
185    (">>=") => {
186        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::RShiftAssign)
187    };
188    (">>>=") => {
189        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::ZeroFillRShiftAssign)
190    };
191
192    ("++") => {
193        $crate::token::Token::PlusPlus
194    };
195    ("--") => {
196        $crate::token::Token::MinusMinus
197    };
198
199    ('=') => {
200        $crate::token::Token::AssignOp(swc_ecma_ast::AssignOp::Assign)
201    };
202
203    ('(') => {
204        $crate::token::Token::LParen
205    };
206    (')') => {
207        $crate::token::Token::RParen
208    };
209    ('{') => {
210        $crate::token::Token::LBrace
211    };
212    ('}') => {
213        $crate::token::Token::RBrace
214    };
215    ('[') => {
216        $crate::token::Token::LBracket
217    };
218    (']') => {
219        $crate::token::Token::RBracket
220    };
221
222    ("await") => {
223        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Await))
224    };
225    ("break") => {
226        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Break))
227    };
228    ("case") => {
229        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Case))
230    };
231    ("catch") => {
232        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Catch))
233    };
234    ("class") => {
235        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Class))
236    };
237    ("const") => {
238        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Const))
239    };
240    ("continue") => {
241        $crate::token::Token::Word($crate::token::Word::Keyword(
242            $crate::token::Keyword::Continue,
243        ))
244    };
245    ("debugger") => {
246        $crate::token::Token::Word($crate::token::Word::Keyword(
247            $crate::token::Keyword::Debugger,
248        ))
249    };
250    ("default") => {
251        $crate::token::Token::Word($crate::token::Word::Keyword(
252            $crate::token::Keyword::Default_,
253        ))
254    };
255    ("delete") => {
256        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Delete))
257    };
258    ("do") => {
259        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Do))
260    };
261    ("else") => {
262        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Else))
263    };
264    ("export") => {
265        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Export))
266    };
267    ("extends") => {
268        $crate::token::Token::Word($crate::token::Word::Keyword(
269            $crate::token::Keyword::Extends,
270        ))
271    };
272    ("false") => {
273        $crate::token::Token::Word($crate::token::Word::False)
274    };
275    ("finally") => {
276        $crate::token::Token::Word($crate::token::Word::Keyword(
277            $crate::token::Keyword::Finally,
278        ))
279    };
280    ("for") => {
281        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::For))
282    };
283    ("function") => {
284        $crate::token::Token::Word($crate::token::Word::Keyword(
285            $crate::token::Keyword::Function,
286        ))
287    };
288    ("if") => {
289        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::If))
290    };
291    ("in") => {
292        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::In))
293    };
294    ("instanceof") => {
295        $crate::token::Token::Word($crate::token::Word::Keyword(
296            $crate::token::Keyword::InstanceOf,
297        ))
298    };
299    ("import") => {
300        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Import))
301    };
302    ("let") => {
303        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Let))
304    };
305    ("new") => {
306        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::New))
307    };
308    ("null") => {
309        $crate::token::Token::Word($crate::token::Word::Null)
310    };
311
312    ("return") => {
313        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Return))
314    };
315    ("super") => {
316        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Super))
317    };
318    ("switch") => {
319        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Switch))
320    };
321    ("this") => {
322        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::This))
323    };
324    ("throw") => {
325        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Throw))
326    };
327    ("true") => {
328        $crate::token::Token::Word($crate::token::Word::True)
329    };
330    ("try") => {
331        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Try))
332    };
333    ("typeof") => {
334        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::TypeOf))
335    };
336    ("var") => {
337        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Var))
338    };
339    ("void") => {
340        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Void))
341    };
342    ("while") => {
343        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::While))
344    };
345    ("with") => {
346        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::With))
347    };
348    ("yield") => {
349        $crate::token::Token::Word($crate::token::Word::Keyword($crate::token::Keyword::Yield))
350    };
351
352    // ----------
353    // JSX
354    // ----------
355    (JSXTagStart) => {
356        $crate::token::Token::JSXTagStart
357    };
358
359    (JSXTagEnd) => {
360        $crate::token::Token::JSXTagEnd
361    };
362
363    ($tt:tt) => {
364        $crate::token::Token::Word($crate::token::Word::Ident($crate::token::IdentLike::Known(
365            known_ident!($tt),
366        )))
367    };
368}
369
370#[inline(always)]
371#[cfg(any(
372    target_arch = "wasm32",
373    target_arch = "arm",
374    not(feature = "stacker"),
375    // miri does not work with stacker
376    miri
377))]
378fn maybe_grow<R, F: FnOnce() -> R>(_red_zone: usize, _stack_size: usize, callback: F) -> R {
379    callback()
380}
381
382#[inline(always)]
383#[cfg(all(
384    not(any(target_arch = "wasm32", target_arch = "arm", miri)),
385    feature = "stacker"
386))]
387fn maybe_grow<R, F: FnOnce() -> R>(red_zone: usize, stack_size: usize, callback: F) -> R {
388    stacker::maybe_grow(red_zone, stack_size, callback)
389}
390
391pub fn lexer(input: Lexer) -> PResult<Vec<token::TokenAndSpan>> {
392    let capturing = input::Capturing::new(input);
393    let mut parser = parser::Parser::new_from(capturing);
394    let _ = parser.parse_module()?;
395    Ok(parser.input_mut().iter_mut().take())
396}
swc_ecma_lexer/lib.rs

swc_ecma_lexer/
lib.rs
