swc_ecma_parser/parser/
input.rs

1use swc_atoms::Atom;
2use swc_common::{BytePos, Span};
3use swc_ecma_lexer::common::{
4    lexer::{token::TokenFactory, LexResult},
5    parser::{buffer::Buffer as BufferTrait, token_and_span::TokenAndSpan as TokenAndSpanTrait},
6};
7
8use crate::lexer::{NextTokenAndSpan, Token, TokenAndSpan, TokenValue};
9
10/// Clone should be cheap if you are parsing typescript because typescript
11/// syntax requires backtracking.
12pub trait Tokens: swc_ecma_lexer::common::input::Tokens<TokenAndSpan> {
13    fn clone_token_value(&self) -> Option<TokenValue>;
14    fn take_token_value(&mut self) -> Option<TokenValue>;
15    fn get_token_value(&self) -> Option<&TokenValue>;
16    fn set_token_value(&mut self, token_value: Option<TokenValue>);
17
18    fn scan_jsx_token(&mut self, allow_multiline_jsx_text: bool) -> TokenAndSpan;
19    fn scan_jsx_open_el_terminal_token(&mut self) -> TokenAndSpan;
20    fn rescan_jsx_open_el_terminal_token(&mut self, reset: BytePos) -> TokenAndSpan;
21    fn rescan_jsx_token(&mut self, allow_multiline_jsx_text: bool, reset: BytePos) -> TokenAndSpan;
22    fn scan_jsx_identifier(&mut self, start: BytePos) -> TokenAndSpan;
23    fn scan_jsx_attribute_value(&mut self) -> TokenAndSpan;
24    fn rescan_template_token(&mut self, start: BytePos, start_with_back_tick: bool)
25        -> TokenAndSpan;
26}
27
28/// This struct is responsible for managing current token and peeked token.
29#[derive(Clone)]
30pub struct Buffer<I> {
31    pub iter: I,
32    /// Span of the previous token.
33    pub prev_span: Span,
34    pub cur: TokenAndSpan,
35    /// Peeked token
36    pub next: Option<NextTokenAndSpan>,
37}
38
39impl<I: Tokens> Buffer<I> {
40    pub fn expect_word_token_value(&mut self) -> Atom {
41        let Some(crate::lexer::TokenValue::Word(word)) = self.iter.take_token_value() else {
42            unreachable!()
43        };
44        word
45    }
46
47    pub fn expect_word_token_value_ref(&self) -> &Atom {
48        let Some(crate::lexer::TokenValue::Word(word)) = self.iter.get_token_value() else {
49            unreachable!("token_value: {:?}", self.iter.get_token_value())
50        };
51        word
52    }
53
54    pub fn expect_number_token_value(&mut self) -> (f64, Atom) {
55        let Some(crate::lexer::TokenValue::Num { value, raw }) = self.iter.take_token_value()
56        else {
57            unreachable!()
58        };
59        (value, raw)
60    }
61
62    pub fn expect_string_token_value(&mut self) -> (Atom, Atom) {
63        let Some(crate::lexer::TokenValue::Str { value, raw }) = self.iter.take_token_value()
64        else {
65            unreachable!()
66        };
67        (value, raw)
68    }
69
70    pub fn expect_bigint_token_value(&mut self) -> (Box<num_bigint::BigInt>, Atom) {
71        let Some(crate::lexer::TokenValue::BigInt { value, raw }) = self.iter.take_token_value()
72        else {
73            unreachable!()
74        };
75        (value, raw)
76    }
77
78    pub fn expect_regex_token_value(&mut self) -> (Atom, Atom) {
79        let Some(crate::lexer::TokenValue::Regex { value, flags }) = self.iter.take_token_value()
80        else {
81            unreachable!()
82        };
83        (value, flags)
84    }
85
86    pub fn expect_template_token_value(&mut self) -> (LexResult<Atom>, Atom) {
87        let Some(crate::lexer::TokenValue::Template { cooked, raw }) = self.iter.take_token_value()
88        else {
89            unreachable!()
90        };
91        (cooked, raw)
92    }
93
94    pub fn expect_error_token_value(&mut self) -> swc_ecma_lexer::error::Error {
95        let Some(crate::lexer::TokenValue::Error(error)) = self.iter.take_token_value() else {
96            unreachable!()
97        };
98        error
99    }
100
101    pub fn get_token_value(&self) -> Option<&TokenValue> {
102        self.iter.get_token_value()
103    }
104
105    pub fn scan_jsx_token(&mut self, allow_multiline_jsx_text: bool) {
106        let prev = self.cur;
107        let t = self.iter.scan_jsx_token(allow_multiline_jsx_text);
108        self.prev_span = prev.span;
109        self.set_cur(t);
110    }
111
112    #[allow(unused)]
113    fn scan_jsx_open_el_terminal_token(&mut self) {
114        let prev = self.cur;
115        let t = self.iter.scan_jsx_open_el_terminal_token();
116        self.prev_span = prev.span;
117        self.set_cur(t);
118    }
119
120    pub fn rescan_jsx_open_el_terminal_token(&mut self) {
121        if !self.cur().should_rescan_into_gt_in_jsx() {
122            return;
123        }
124        // rescan `>=`, `>>`, `>>=`, `>>>`, `>>>=` into `>`
125        let start = self.cur.span.lo;
126        let t = self.iter.rescan_jsx_open_el_terminal_token(start);
127        self.set_cur(t);
128    }
129
130    pub fn rescan_jsx_token(&mut self, allow_multiline_jsx_text: bool) {
131        let start = self.cur.span.lo;
132        let t = self.iter.rescan_jsx_token(allow_multiline_jsx_text, start);
133        self.set_cur(t);
134    }
135
136    pub fn scan_jsx_identifier(&mut self) {
137        if !(*self.cur()).is_word() {
138            return;
139        }
140        let start = self.cur.span.lo;
141        let cur = self.iter.scan_jsx_identifier(start);
142        debug_assert!(cur.token == Token::JSXName);
143        self.set_cur(cur);
144    }
145
146    pub fn scan_jsx_attribute_value(&mut self) {
147        self.cur = self.iter.scan_jsx_attribute_value();
148    }
149
150    pub fn rescan_template_token(&mut self, start_with_back_tick: bool) {
151        let start = self.cur_pos();
152        self.cur = self.iter.rescan_template_token(start, start_with_back_tick);
153    }
154}
155
156impl<'a, I: Tokens> swc_ecma_lexer::common::parser::buffer::Buffer<'a> for Buffer<I> {
157    type I = I;
158    type Next = NextTokenAndSpan;
159    type Token = super::super::lexer::Token;
160    type TokenAndSpan = TokenAndSpan;
161
162    fn new(lexer: I) -> Self {
163        let start_pos = lexer.start_pos();
164        let prev_span = Span::new_with_checked(start_pos, start_pos);
165        Buffer {
166            iter: lexer,
167            cur: TokenAndSpan::new(Token::Eof, prev_span, false),
168            prev_span,
169            next: None,
170        }
171    }
172
173    #[inline(always)]
174    fn set_cur(&mut self, token: TokenAndSpan) {
175        self.cur = token
176    }
177
178    #[inline(always)]
179    fn next(&self) -> Option<&Self::Next> {
180        self.next.as_ref()
181    }
182
183    #[inline(always)]
184    fn set_next(&mut self, token: Option<Self::Next>) {
185        self.next = token;
186    }
187
188    #[inline(always)]
189    fn next_mut(&mut self) -> &mut Option<Self::Next> {
190        &mut self.next
191    }
192
193    #[inline(always)]
194    fn cur(&self) -> &super::super::lexer::Token {
195        &self.cur.token
196    }
197
198    #[inline(always)]
199    fn get_cur(&self) -> &TokenAndSpan {
200        &self.cur
201    }
202
203    #[inline(always)]
204    fn prev_span(&self) -> Span {
205        self.prev_span
206    }
207
208    #[inline(always)]
209    fn iter(&self) -> &I {
210        &self.iter
211    }
212
213    #[inline(always)]
214    fn iter_mut(&mut self) -> &mut I {
215        &mut self.iter
216    }
217
218    fn peek<'b>(&'b mut self) -> Option<&'b super::super::lexer::Token>
219    where
220        TokenAndSpan: 'b,
221    {
222        debug_assert!(
223            self.cur.token != Token::Eof,
224            "parser should not call peek() without knowing current token"
225        );
226
227        if self.next.is_none() {
228            let old = self.iter.take_token_value();
229            let next_token = self.iter.next();
230            self.next = next_token.map(|t| NextTokenAndSpan {
231                token_and_span: t,
232                value: self.iter.take_token_value(),
233            });
234            self.iter.set_token_value(old);
235        }
236
237        self.next.as_ref().map(|ts| &ts.token_and_span.token)
238    }
239
240    fn bump(&mut self) {
241        let next = if let Some(next) = self.next.take() {
242            self.iter.set_token_value(next.value);
243            next.token_and_span
244        } else if let Some(next) = self.iter.next() {
245            next
246        } else {
247            let eof_pos = self.cur.span().hi;
248            let eof_span = Span::new_with_checked(eof_pos, eof_pos);
249            TokenAndSpan::new(Token::Eof, eof_span, true)
250        };
251        self.prev_span = self.cur.span();
252        self.set_cur(next);
253    }
254
255    fn expect_word_token_and_bump(&mut self) -> Atom {
256        let cur = *self.cur();
257        let word = cur.take_word(self).unwrap();
258        self.bump();
259        word
260    }
261
262    fn expect_shebang_token_and_bump(&mut self) -> swc_atoms::Atom {
263        let cur = *self.cur();
264        let ret = cur.take_shebang(self);
265        self.bump();
266        ret
267    }
268
269    fn expect_jsx_name_token_and_bump(&mut self) -> Atom {
270        let cur = *self.cur();
271        let word = cur.take_jsx_name(self);
272        self.bump();
273        word
274    }
275
276    fn expect_jsx_text_token_and_bump(&mut self) -> (Atom, Atom) {
277        let cur = *self.cur();
278        let ret = cur.take_jsx_text(self);
279        self.bump();
280        ret
281    }
282
283    fn expect_number_token_and_bump(&mut self) -> (f64, Atom) {
284        let cur = *self.cur();
285        let ret = cur.take_num(self);
286        self.bump();
287        ret
288    }
289
290    fn expect_string_token_and_bump(&mut self) -> (Atom, Atom) {
291        let cur = *self.cur();
292        let ret = cur.take_str(self);
293        self.bump();
294        ret
295    }
296
297    fn expect_bigint_token_and_bump(&mut self) -> (Box<num_bigint::BigInt>, Atom) {
298        let cur = *self.cur();
299        let ret = cur.take_bigint(self);
300        self.bump();
301        ret
302    }
303
304    fn expect_regex_token_and_bump(&mut self) -> (Atom, Atom) {
305        let cur = *self.cur();
306        let ret = cur.take_regexp(self);
307        self.bump();
308        ret
309    }
310
311    fn expect_template_token_and_bump(&mut self) -> (LexResult<Atom>, Atom) {
312        let cur = *self.cur();
313        let ret = cur.take_template(self);
314        self.bump();
315        ret
316    }
317
318    fn expect_error_token_and_bump(&mut self) -> swc_ecma_lexer::error::Error {
319        let cur = *self.cur();
320        let ret = cur.take_error(self);
321        self.bump();
322        ret
323    }
324
325    #[cold]
326    #[inline(never)]
327    fn dump_cur(&self) -> String {
328        let cur = self.cur();
329        cur.to_string(self)
330    }
331}
swc_ecma_parser/parser/input.rs

swc_ecma_parser/parser/
input.rs
