diff --git a/core/parser/src/parser/cursor/buffered_lexer/mod.rs b/core/parser/src/parser/cursor/buffered_lexer/mod.rs index 67ba7aee596..f932adc3fa6 100644 --- a/core/parser/src/parser/cursor/buffered_lexer/mod.rs +++ b/core/parser/src/parser/cursor/buffered_lexer/mod.rs @@ -30,6 +30,7 @@ pub(super) struct BufferedLexer { read_index: usize, write_index: usize, last_linear_pos: LinearPosition, + start_of_file: bool, } impl From> for BufferedLexer @@ -53,6 +54,7 @@ where read_index: 0, write_index: 0, last_linear_pos: LinearPosition::default(), + start_of_file: true, } } } @@ -134,9 +136,12 @@ where let previous_index = self.write_index.checked_sub(1).unwrap_or(PEEK_BUF_SIZE - 1); - if let Some(ref token) = self.peeked[previous_index] - && token.kind() == &TokenKind::LineTerminator - { + let previous_token = self.peeked[previous_index].as_ref(); + let is_line_term = + previous_token.is_some_and(|token| token.kind() == &TokenKind::LineTerminator); + + if is_line_term { + self.start_of_file = false; // We don't want to have multiple contiguous line terminators in the buffer, since // they have no meaning. let next = loop { @@ -153,6 +158,24 @@ where } }; + self.peeked[self.write_index] = next; + } else if self.start_of_file { + self.start_of_file = false; + // At the start of the file, HTML close comments (`-->`) are allowed by Annex B. + // Whitespace and single-line block comments may precede them. + let next = loop { + self.lexer.skip_html_close(interner)?; + let next = self.lexer.next_no_skip(interner)?; + if let Some(ref token) = next { + match token.kind() { + TokenKind::Comment => self.lexer.skip_html_close(interner)?, + _ => break next, + } + } else { + break None; + } + }; + self.peeked[self.write_index] = next; } else { self.peeked[self.write_index] = self.lexer.next(interner)?; diff --git a/core/parser/src/parser/cursor/buffered_lexer/tests.rs b/core/parser/src/parser/cursor/buffered_lexer/tests.rs index 7dc8c58ef5b..e803e06a1b2 100644 --- a/core/parser/src/parser/cursor/buffered_lexer/tests.rs +++ b/core/parser/src/parser/cursor/buffered_lexer/tests.rs @@ -287,3 +287,57 @@ fn issue_1768() { assert!(cur.peek(3, true, interner).unwrap().is_none()); } + +#[test] +#[cfg(feature = "annex-b")] +fn html_close_comment_first_line() { + let mut cur = BufferedLexer::from(&b"--> comment\nx"[..]); + let interner = &mut Interner::default(); + + assert_eq!( + *cur.peek(0, true, interner) + .unwrap() + .expect("Token expected") + .kind(), + TokenKind::identifier(interner.get_or_intern_static("x", utf16!("x"))) + ); +} + +#[test] +#[cfg(feature = "annex-b")] +fn html_close_comment_first_line_with_spaces() { + let mut cur = BufferedLexer::from(&b" --> comment\nx"[..]); + let interner = &mut Interner::default(); + + assert_eq!( + *cur.peek(0, true, interner) + .unwrap() + .expect("Token expected") + .kind(), + TokenKind::identifier(interner.get_or_intern_static("x", utf16!("x"))) + ); +} + +#[test] +#[cfg(feature = "annex-b")] +fn html_close_comment_first_line_with_block_comments() { + let mut cur = BufferedLexer::from(&b"/* comment */ /* another */ --> comment\nx"[..]); + let interner = &mut Interner::default(); + + assert_eq!( + *cur.peek(0, true, interner) + .unwrap() + .expect("Token expected") + .kind(), + TokenKind::identifier(interner.get_or_intern_static("x", utf16!("x"))) + ); +} + +#[test] +#[cfg(feature = "annex-b")] +fn html_close_comment_first_line_eof() { + let mut cur = BufferedLexer::from(&b"--> comment"[..]); + let interner = &mut Interner::default(); + + assert!(cur.peek(0, true, interner).unwrap().is_none()); +}