swc_ecma_parser/parser/
input.rs

1use swc_atoms::Atom;
2use swc_common::{BytePos, Span};
3use swc_ecma_lexer::common::{lexer::LexResult, parser::buffer::Buffer as BufferTrait};
4
5use crate::lexer::{NextTokenAndSpan, Token, TokenAndSpan, TokenValue};
6
7/// Clone should be cheap if you are parsing typescript because typescript
8/// syntax requires backtracking.
9pub trait Tokens: swc_ecma_lexer::common::input::Tokens<TokenAndSpan> {
10    fn clone_token_value(&self) -> Option<TokenValue>;
11    fn take_token_value(&mut self) -> Option<TokenValue>;
12    fn get_token_value(&self) -> Option<&TokenValue>;
13    fn set_token_value(&mut self, token_value: Option<TokenValue>);
14
15    fn scan_jsx_token(&mut self, allow_multiline_jsx_text: bool) -> Option<TokenAndSpan>;
16    fn scan_jsx_open_el_terminal_token(&mut self) -> Option<TokenAndSpan>;
17    fn rescan_jsx_open_el_terminal_token(&mut self, reset: BytePos) -> Option<TokenAndSpan>;
18    fn rescan_jsx_token(
19        &mut self,
20        allow_multiline_jsx_text: bool,
21        reset: BytePos,
22    ) -> Option<TokenAndSpan>;
23    fn scan_jsx_identifier(&mut self, start: BytePos) -> TokenAndSpan;
24    fn scan_jsx_attribute_value(&mut self) -> Option<TokenAndSpan>;
25    fn rescan_template_token(
26        &mut self,
27        start: BytePos,
28        start_with_back_tick: bool,
29    ) -> Option<TokenAndSpan>;
30}
31
32/// This struct is responsible for managing current token and peeked token.
33#[derive(Clone)]
34pub struct Buffer<I> {
35    pub iter: I,
36    /// Span of the previous token.
37    pub prev_span: Span,
38    pub cur: Option<TokenAndSpan>,
39    /// Peeked token
40    pub next: Option<NextTokenAndSpan>,
41}
42
43impl<I: Tokens> Buffer<I> {
44    pub fn expect_word_token_value(&mut self) -> Atom {
45        let Some(crate::lexer::TokenValue::Word(word)) = self.iter.take_token_value() else {
46            unreachable!()
47        };
48        word
49    }
50
51    pub fn expect_word_token_value_ref(&self) -> &Atom {
52        let Some(crate::lexer::TokenValue::Word(word)) = self.iter.get_token_value() else {
53            unreachable!("token_value: {:?}", self.iter.get_token_value())
54        };
55        word
56    }
57
58    pub fn expect_number_token_value(&mut self) -> (f64, Atom) {
59        let Some(crate::lexer::TokenValue::Num { value, raw }) = self.iter.take_token_value()
60        else {
61            unreachable!()
62        };
63        (value, raw)
64    }
65
66    pub fn expect_string_token_value(&mut self) -> (Atom, Atom) {
67        let Some(crate::lexer::TokenValue::Str { value, raw }) = self.iter.take_token_value()
68        else {
69            unreachable!()
70        };
71        (value, raw)
72    }
73
74    pub fn expect_bigint_token_value(&mut self) -> (Box<num_bigint::BigInt>, Atom) {
75        let Some(crate::lexer::TokenValue::BigInt { value, raw }) = self.iter.take_token_value()
76        else {
77            unreachable!()
78        };
79        (value, raw)
80    }
81
82    pub fn expect_regex_token_value(&mut self) -> (Atom, Atom) {
83        let Some(crate::lexer::TokenValue::Regex { value, flags }) = self.iter.take_token_value()
84        else {
85            unreachable!()
86        };
87        (value, flags)
88    }
89
90    pub fn expect_template_token_value(&mut self) -> (LexResult<Atom>, Atom) {
91        let Some(crate::lexer::TokenValue::Template { cooked, raw }) = self.iter.take_token_value()
92        else {
93            unreachable!()
94        };
95        (cooked, raw)
96    }
97
98    pub fn expect_error_token_value(&mut self) -> swc_ecma_lexer::error::Error {
99        let Some(crate::lexer::TokenValue::Error(error)) = self.iter.take_token_value() else {
100            unreachable!()
101        };
102        error
103    }
104
105    pub fn get_token_value(&self) -> Option<&TokenValue> {
106        self.iter.get_token_value()
107    }
108
109    pub fn scan_jsx_token(&mut self, allow_multiline_jsx_text: bool) {
110        if let Some(t) = self.iter_mut().scan_jsx_token(allow_multiline_jsx_text) {
111            self.set_cur(t);
112        }
113    }
114
115    fn scan_jsx_open_el_terminal_token(&mut self) {
116        if let Some(t) = self.iter_mut().scan_jsx_open_el_terminal_token() {
117            self.set_cur(t);
118        }
119    }
120
121    pub fn rescan_jsx_open_el_terminal_token(&mut self) {
122        // rescan `>=`, `>>`, `>>=`, `>>>`, `>>>=` into `>`
123        let cur = match self.cur.as_ref() {
124            Some(cur) => cur,
125            None => {
126                return self.scan_jsx_open_el_terminal_token();
127            }
128        };
129        if !matches!(
130            cur.token,
131            Token::GtEq
132                | Token::RShift
133                | Token::RShiftEq
134                | Token::ZeroFillRShift
135                | Token::ZeroFillRShiftEq
136        ) {
137            return;
138        }
139        let start = cur.span.lo;
140        if let Some(t) = self.iter_mut().rescan_jsx_open_el_terminal_token(start) {
141            self.set_cur(t);
142        }
143    }
144
145    pub fn rescan_jsx_token(&mut self, allow_multiline_jsx_text: bool) {
146        let start = match self.cur.as_ref() {
147            Some(cur) => cur.span.lo,
148            None => {
149                return self.scan_jsx_token(allow_multiline_jsx_text);
150            }
151        };
152        if let Some(t) = self
153            .iter_mut()
154            .rescan_jsx_token(allow_multiline_jsx_text, start)
155        {
156            self.set_cur(t);
157        }
158    }
159
160    pub fn scan_jsx_identifier(&mut self) {
161        let Some(cur) = self.cur() else {
162            return;
163        };
164        if !cur.is_word() {
165            return;
166        }
167        let start = self.cur.as_ref().unwrap().span.lo;
168        let cur = self.iter_mut().scan_jsx_identifier(start);
169        debug_assert!(cur.token == Token::JSXName);
170        self.set_cur(cur);
171    }
172
173    pub fn scan_jsx_attribute_value(&mut self) {
174        if let Some(t) = self.iter_mut().scan_jsx_attribute_value() {
175            self.set_cur(t);
176        }
177    }
178
179    pub fn rescan_template_token(&mut self, start_with_back_tick: bool) {
180        debug_assert!(self.cur.is_some());
181        let start = self.cur_pos();
182        self.cur = self
183            .iter_mut()
184            .rescan_template_token(start, start_with_back_tick);
185    }
186}
187
188impl<'a, I: Tokens> swc_ecma_lexer::common::parser::buffer::Buffer<'a> for Buffer<I> {
189    type I = I;
190    type Lexer = super::super::lexer::Lexer<'a>;
191    type Next = NextTokenAndSpan;
192    type Token = super::super::lexer::Token;
193    type TokenAndSpan = TokenAndSpan;
194
195    fn new(lexer: I) -> Self {
196        let start_pos = lexer.start_pos();
197        Buffer {
198            iter: lexer,
199            cur: None,
200            prev_span: Span::new(start_pos, start_pos),
201            next: None,
202        }
203    }
204
205    #[cold]
206    #[inline(never)]
207    fn dump_cur(&mut self) -> String {
208        match self.cur() {
209            Some(v) => v.to_string(self.get_token_value()),
210            None => "<eof>".to_string(),
211        }
212    }
213
214    fn set_cur(&mut self, token: TokenAndSpan) {
215        self.cur = Some(token);
216    }
217
218    fn next(&self) -> Option<&Self::Next> {
219        self.next.as_ref()
220    }
221
222    fn set_next(&mut self, token: Option<Self::Next>) {
223        self.next = token;
224    }
225
226    fn next_mut(&mut self) -> &mut Option<Self::Next> {
227        &mut self.next
228    }
229
230    fn cur(&mut self) -> Option<&super::super::lexer::Token> {
231        if self.cur.is_none() {
232            // If we have peeked a token, take it instead of calling lexer.next()
233            if let Some(next) = self.next.take() {
234                self.cur = Some(next.token_and_span);
235                self.iter.set_token_value(next.value);
236            } else {
237                self.cur = self.iter.next()
238            }
239        }
240
241        self.cur.as_ref().map(|v| &v.token)
242    }
243
244    fn get_cur(&self) -> Option<&TokenAndSpan> {
245        self.cur.as_ref()
246    }
247
248    fn get_cur_mut(&mut self) -> &mut Option<TokenAndSpan> {
249        &mut self.cur
250    }
251
252    fn prev_span(&self) -> Span {
253        self.prev_span
254    }
255
256    fn set_prev_span(&mut self, span: Span) {
257        self.prev_span = span;
258    }
259
260    fn iter(&self) -> &I {
261        &self.iter
262    }
263
264    fn iter_mut(&mut self) -> &mut I {
265        &mut self.iter
266    }
267
268    fn peek<'b>(&'b mut self) -> Option<&'b super::super::lexer::Token>
269    where
270        TokenAndSpan: 'b,
271    {
272        debug_assert!(
273            self.cur.is_some(),
274            "parser should not call peek() without knowing current token"
275        );
276
277        if self.next.is_none() {
278            let old = self.iter.take_token_value();
279            let next_token = self.iter.next();
280            self.next = next_token.map(|t| NextTokenAndSpan {
281                token_and_span: t,
282                value: self.iter.take_token_value(),
283            });
284            self.iter.set_token_value(old);
285        }
286
287        self.next.as_ref().map(|ts| &ts.token_and_span.token)
288    }
289}
swc_ecma_parser/parser/input.rs

swc_ecma_parser/parser/
input.rs
