diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b53e140..f8548c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -258,7 +258,7 @@ jobs: mv /tmp/artifacts/milo-parser-bin-macos-intel/milo-parser-bin-macos-intel.tar.gz /tmp/release/ - name: Push release commit and tag run: | - git add -f package.json CHANGELOG.md parser/Cargo.toml parser/Cargo.lock macros/Cargo.toml macros/Cargo.lock references/rust/Cargo.toml references/rust/Cargo.lock parser/src/wasm/package.json + git add -f package.json CHANGELOG.md parser/Cargo.toml parser/Cargo.lock macros/Cargo.toml macros/Cargo.lock references/rust/Cargo.toml references/rust/Cargo.lock parser/wasm/src/package.json git commit -m "chore: Updated version." git tag -f "v${{ inputs.version }}" - name: Publish macros on crates.io diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..fbd2390 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,67 @@ +# Milo development guide + +## Parser internals + +Milo leverages Rust's [procedural macros](https://doc.rust-lang.org/reference/procedural-macros.html), [syn](https://crates.io/crates/syn), and [quote](https://crates.io/crates/quote) crates to define actions and matchers for the parser. + +See the [macros](./macros/README.md) internal crate for more information. + +## Build WebAssembly and C++ locally + +Required tools: + +- [cargo-make](https://github.com/sagiegurari/cargo-make). +- The pinned Rust nightly toolchain, installed via [rustup](https://rustup.rs/). +- [rust-cbindgen](https://github.com/mozilla/cbindgen). +- [Binaryen](https://github.com/WebAssembly/binaryen), providing `wasm-opt`. + +Install the pinned toolchain, the Rust sources required by the WebAssembly release build, and the WebAssembly target: + +```sh +rustup toolchain install nightly-2026-07-29 --component rust-src +rustup target add wasm32-unknown-unknown +``` + +Run from the repository root: + +```sh +makers +``` + +This produces debug and release builds for each language in the top-level `dist` folder. + +Build tooling is compiled from `scripts` into standalone Rust binaries. Node.js and npm dependencies are not required to build the parser or generate its C++ and WebAssembly packages. + +The WebAssembly release build uses immediate-abort panics to keep the artifact smaller. Panics trap without unwinding or rich panic messages. The debug build also enables the `on_state_change` callback and provides more detailed WebAssembly errors. + +For JavaScript linting and formatting, install the development dependencies with `pnpm install`. + +## Run tests + +Run `makers test` from the repository root for the Rust and WebAssembly suites, or `makers test:wasm` to build and test only WebAssembly. + +The WebAssembly suite uses Node.js's built-in test runner and tests the release SIMD package by default. Set `MILO_VARIANT=no-simd` to select the non-SIMD package instead. After building, run `pnpm test:wasm` to rerun it without rebuilding. + +Tests live in `parser/wasm/test` and mirror the Rust integration tests in `parser/tests` with the same case names: basic, benchmark, compliance, issue, undici, and upgrade. Issue regressions use the `issue___` naming convention. The llhttp suite is not yet ported. + +## Build WebAssembly with Docker + +The repository includes a Docker image for building the WebAssembly packages without changing the working tree. Build the image from the repository root, then mount the sources read-only and choose a host directory for the generated artifacts: + +```sh +docker build -t milo-wasm . +mkdir -p /path/to/milo-wasm-output +docker run --rm \ + -v "$PWD:/src:ro" \ + -v "/path/to/milo-wasm-output:/output" \ + milo-wasm +``` + +The container builds both the debug and release profiles in its temporary workspace. The output directory receives the resulting `debug` and `release` packages; the mounted source tree remains read-only. + +## Contributing + +- Check the latest default branch to make sure the feature hasn't been implemented or the bug hasn't been fixed yet. +- Check the issue tracker for existing requests and contributions. +- The contribution workflow uses a fork and a feature or bugfix branch, followed by commits and a push when the contribution is ready. +- Add tests for changes to prevent regressions. diff --git a/Makefile.toml b/Makefile.toml index c2186cf..aa5527b 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -8,8 +8,14 @@ script = ["cd parser", "makers build"] [tasks.test] + dependencies = ["test:rust", "test:wasm"] + +[tasks."test:rust"] script = ["cd parser", "cargo test"] +[tasks."test:wasm"] + script = ["cd parser", "makers test:wasm"] + [tasks.format] dependencies = ["format:rust", "format:js", "format:cpp"] diff --git a/README.md b/README.md index 7057968..86a4ae4 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,8 @@ milo.dealloc(ptr, message.length) The default JavaScript entry point uses the SIMD WebAssembly build. Use `@perseveranza-pets/milo/no-simd` when SIMD is not available, and add `/unbundled` to either entry point to load the external `.wasm` file instead of the bundled JavaScript module. +The WebAssembly release build uses immediate-abort panics: panics trap without unwinding or rich panic messages. The debug build also enables the `on_state_change` callback and provides more detailed WebAssembly errors. + CommonJS projects can use the same entry points from `@perseveranza-pets/milo-cjs`: ```javascript @@ -233,42 +235,6 @@ clang++ -std=c++11 -o example main.cc libmilo.a # Pos=38 Body: abc ``` -### Build milo (WebAssembly and C++) locally - -If you want to build it locally, you need the following tools: - -- [cargo-make][cargo-make] -- Rust toolchain - You can install it via [rustup]. -- [rust-cbindgen](https://github.com/mozilla/cbindgen) - -Make sure you have the pinned nightly toolchain installed locally: - -```bash -rustup toolchain install nightly-2026-07-29 -``` - -Make sure you have the `wasm32-unknown-unknown` target: - -```bash -rustup target add wasm32-unknown-unknown -``` - -After all the requirements are met, you can then run: - -```bash -makers -``` - -The command above will produce debug and release builds for each language in the top-level `dist` folder. - -Build tooling is compiled from `scripts` into standalone Rust binaries. Node.js and npm dependencies are not required to build the parser or generate its C++ and WebAssembly packages. - -For JavaScript linting and formatting, install the development dependencies with `pnpm install`. - -The WebAssembly release build uses immediate-abort panics to keep the artifact smaller. Panics trap without unwinding or rich panic messages. - -The debug build also enables the `on_state_change` callback and is more verbose in case of WebAssembly errors. - ## How to use it (CLI) Install it from crates.io: @@ -336,11 +302,7 @@ Milo validates HTTP/1.1 syntax, message framing, protocol switching, connection ## How it works? -Milo leverages Rust's [procedural macro], [syn] and [quote] crates to allow an easy definition of actions and matchers for the parser. - -See the [macros](./macros/README.md) internal crate for more information. - -The resulting parser is a simple state machine which copies data in only one optional case: automatically handling the unconsumed portion of the input data. +Milo is a simple state machine which copies data in only one optional case: automatically handling the unconsumed portion of the input data. In all other cases, no data is copied and the memory footprint is very small as only a few dozen `bool`, `uintptr_t`, or `uint64_t` fields can represent the entire parser state. @@ -358,33 +320,13 @@ To see the rationale behind the replacement of llhttp, check Paolo's talk at [Va To see the initial disclosure of milo, check Paolo's talk at [NodeConf EU 2023][nodeconf-talk] in November 2023 ([slides][slides]). -## Building WebAssembly with Docker - -The repository includes a Docker image for building the WebAssembly packages without changing the working tree. Build the image from the repository root, then mount the sources read-only and choose a host directory for the generated artifacts: - -```sh -docker build -t milo-wasm . -mkdir -p /path/to/milo-wasm-output -docker run --rm \ - -v "$PWD:/src:ro" \ - -v "/path/to/milo-wasm-output:/output" \ - milo-wasm -``` - -The container builds both the debug and release profiles in its temporary workspace. The output directory receives the resulting `debug` and `release` packages; the mounted source tree remains read-only. - ## Sponsored by [![NearForm](https://raw.githubusercontent.com/ShogunPanda/milo/main/docs/nearform.jpg)][nearform] ## Contributing to milo -- Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet. -- Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it. -- Fork the project. -- Start a feature/bugfix branch. -- Commit and push until you are happy with your contribution. -- Make sure to add tests for it. This is important so I don't break it in a future version unintentionally. +See [AGENTS.md](./AGENTS.md) for development setup, local and Docker builds, tests, and contribution guidelines. ## Copyright @@ -402,12 +344,6 @@ Licensed under the ISC license, which can be found at https://choosealicense.com [nodeconf-talk]: https://youtube.com/watch?v=dcHbAeO_ccY [slides]: https://talks.paoloinsogna.dev/milo [isc]: https://choosealicense.com/licenses/isc -[procedural macro]: https://doc.rust-lang.org/reference/procedural-macros.html -[syn]: https://crates.io/crates/syn -[quote]: https://crates.io/crates/quote [match]: https://doc.rust-lang.org/rust-by-example/flow_control/match.html [match-slice]: https://doc.rust-lang.org/rust-by-example/flow_control/match/destructuring/destructure_slice.html -[cargo-make]: https://github.com/sagiegurari/cargo-make -[rustup]: https://rustup.rs/ [Clang]: https://clang.llvm.org/ - diff --git a/package.json b/package.json index 71a82b9..76ccd30 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ "private": true, "type": "module", "scripts": { - "format": "prettier -w \"parser/**/*.js\" \"benchmarks/**/*.js\" \"references/**/*.js\"", - "lint": "eslint --cache \"parser/**/*.js\" \"benchmarks/**/*.js\" \"references/**/*.js\"" + "test:wasm": "node --test parser/wasm/test/*.test.js", + "format": "prettier -w \"parser/wasm/**/*.js\" \"benchmarks/wasm/**/*.js\" \"references/wasm/**/*.js\" references/reference.js", + "lint": "eslint --cache \"parser/wasm/**/*.js\" \"benchmarks/wasm/**/*.js\" \"references/wasm/**/*.js\" references/reference.js" }, "dependencies": { "@cowtech/eslint-config": "^11.1.3", diff --git a/parser/Makefile.toml b/parser/Makefile.toml index f12cfab..bac0e53 100644 --- a/parser/Makefile.toml +++ b/parser/Makefile.toml @@ -11,10 +11,17 @@ dependencies = ["cpp", "wasm"] [tasks.test] - dependencies = ["build"] + dependencies = ["build", "test:rust", "test:wasm"] + +[tasks."test:rust"] command = "cargo" args = ["test"] +[tasks."test:wasm"] + dependencies = ["wasm:release"] + command = "node" + args = ["--test", "wasm/test/*.test.js"] + [tasks.cpp] dependencies = ["cpp:headers", "cpp:libs"] diff --git a/parser/src/matchers.rs b/parser/src/matchers.rs index d8ab9a9..0d1da37 100644 --- a/parser/src/matchers.rs +++ b/parser/src/matchers.rs @@ -318,8 +318,9 @@ pub fn find_header_line_end(ptr: *const u8, len: usize) -> HeaderLineScanResult let eq_7f = u8x16_eq(x, v_7f); // Header lines stop at CR; other control bytes are invalid except HTAB. - let ctrl = v128_andnot(eq_tab, lt_20); - let invalid = v128_andnot(eq_cr, v128_or(ctrl, eq_7f)); + // WASM andnot(a, b) computes a & !b, unlike the x86 intrinsic. + let ctrl = v128_andnot(lt_20, eq_tab); + let invalid = v128_andnot(v128_or(ctrl, eq_7f), eq_cr); let found = v128_or(eq_cr, invalid); if v128_any_true(found) { @@ -440,7 +441,8 @@ pub fn validate_token_value(ptr: *const u8, len: usize) -> bool { let eq_7f = u8x16_eq(x, v_7f); // Field values allow HTAB but reject the remaining C0 controls and DEL. - let ctrl = v128_andnot(eq_tab, lt_20); + // WASM andnot(a, b) computes a & !b, unlike the x86 intrinsic. + let ctrl = v128_andnot(lt_20, eq_tab); let invalid = v128_or(ctrl, eq_7f); if v128_any_true(invalid) { diff --git a/parser/tests/compliance.rs b/parser/tests/compliance.rs index e6da48b..fb1244a 100644 --- a/parser/tests/compliance.rs +++ b/parser/tests/compliance.rs @@ -555,17 +555,6 @@ fn compliance_chunk_extension_quoted_pair_control_rejected() { assert_error(&parser); } -// Bare LF is rejected in HTTP framing. -#[test] -fn compliance_bare_lf_rejected() { - let mut parser = response_parser(); - let message = "HTTP/1.1 200 OK\r\nHeader: value\nContent-Length: 0\r\n\r\n"; - - parse(&mut parser, message); - - assert_error(&parser); -} - // Bare CR is rejected in HTTP framing. #[test] fn compliance_bare_cr_rejected() { diff --git a/parser/tests/issue.rs b/parser/tests/issue.rs new file mode 100644 index 0000000..8d3d27d --- /dev/null +++ b/parser/tests/issue.rs @@ -0,0 +1,84 @@ +mod helpers; + +use milo_parser::{ERROR_NONE, Parser, STATE_ERROR}; + +use crate::helpers::{create_parser, parse}; + +fn response_parser() -> Parser { + let mut parser = create_parser(); + parser.autodetect = false; + parser.is_request = false; + parser +} + +fn field_messages(value: &[u8]) -> [Vec; 3] { + let fields: [(&[u8], &[u8]); 3] = [ + (b"HTTP/1.1 200 OK\r\nX-Long: ", b"\r\nContent-Length: 0\r\n\r\n"), + ( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX-Long: ", + b"\r\n\r\n", + ), + (b"HTTP/1.1 200 ", b"\r\nContent-Length: 0\r\n\r\n"), + ]; + fields.map(|(prefix, suffix)| [prefix, value, suffix].concat()) +} + +// Cover SIMD block boundaries and scalar tails in both field scanners. +#[test] +#[allow(non_snake_case)] +fn issue_22__field_values_reject_controls() { + let mut parser = response_parser(); + parser.active_callbacks = 0; + + for byte in (0u8..0x20).chain([0x7f]).filter(|byte| *byte != b'\t') { + for offset in [0, 7, 8, 15, 16, 20, 31, 32] { + for trailing in [0, 20] { + let mut value = vec![b'a'; offset]; + value.push(byte); + value.extend(vec![b'a'; trailing]); + for message in field_messages(&value) { + parser.reset(false); + parser.parse(message.as_ptr(), message.len()); + assert_eq!(parser.state, STATE_ERROR, "Accepted invalid field: {message:?}"); + } + } + } + } +} + +// HTAB and every obs-text byte remain valid, including across SIMD boundaries. +#[test] +#[allow(non_snake_case)] +fn issue_22__field_values_allow_tab_and_obs_text() { + let mut parser = response_parser(); + // Raw obs-text is not necessarily UTF-8, so bypass the text-decoding callbacks. + parser.active_callbacks = 0; + + for byte in [b'\t'].into_iter().chain(0x80..=0xff) { + for offset in [0, 7, 8, 15, 16, 20, 31, 32] { + for trailing in [0, 20] { + let mut value = vec![b'a'; offset]; + value.push(byte); + value.extend(vec![b'a'; trailing]); + for message in field_messages(&value) { + parser.reset(false); + parser.parse(message.as_ptr(), message.len()); + assert_ne!(parser.state, STATE_ERROR, "Rejected valid field: {message:?}"); + assert_eq!(parser.error_code, ERROR_NONE); + } + } + } + } +} + +// Bare LF is rejected in HTTP framing. +#[test] +#[allow(non_snake_case)] +fn issue_22__bare_lf_rejected() { + let mut parser = response_parser(); + let message = "HTTP/1.1 200 OK\r\nHeader: value\nContent-Length: 0\r\n\r\n"; + + parse(&mut parser, message); + + assert_eq!(parser.state, STATE_ERROR); +} diff --git a/parser/src/wasm/package.json b/parser/wasm/src/package.json similarity index 100% rename from parser/src/wasm/package.json rename to parser/wasm/src/package.json diff --git a/parser/src/wasm/template.js b/parser/wasm/src/template.js similarity index 100% rename from parser/src/wasm/template.js rename to parser/wasm/src/template.js diff --git a/parser/wasm/test/basic.test.js b/parser/wasm/test/basic.test.js new file mode 100644 index 0000000..eaf51ae --- /dev/null +++ b/parser/wasm/test/basic.test.js @@ -0,0 +1,466 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, http, setup } from './helpers.js' + +// Keep case names aligned with parser/tests/basic.rs for coverage comparisons. +it('basic_error_description_is_clamped_and_terminated', t => { + const { milo, parser } = createParser(t, setup) + milo.fail(parser, milo.ERROR_UNEXPECTED_CHARACTER, 'a'.repeat(300)) + + let memory = new Uint8Array(milo.memory.buffer) + assert.equal(memory[parser + milo.PARSER_FIELD_ERROR_DESCRIPTION_LEN], 254) + assert.equal(memory[parser + milo.PARSER_FIELD_ERROR_DESCRIPTION + 254], 0) + assert.equal(milo.getErrorDescription(parser), 'a'.repeat(254)) + + milo.reset(parser, false) + memory = new Uint8Array(milo.memory.buffer) + assert.equal(memory[parser + milo.PARSER_FIELD_ERROR_DESCRIPTION_LEN], 0) + assert.equal(memory[parser + milo.PARSER_FIELD_ERROR_DESCRIPTION], 0) +}) + +it('basic_disable_autodetect', t => { + const { milo, parser, parse } = createParser(t, setup) + const request = http(String.raw` + PUT /url HTTP/1.1\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + `) + const response = http(String.raw` + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + `) + + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, true) + parse(response) + assert.equal(milo.getState(parser), milo.STATE_ERROR) + milo.reset(parser, false) + + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + parse(request) + assert.equal(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_incomplete_string_1', t => { + const { milo, parser, parse } = createParser(t, setup) + for (const sample of ['GET / HTTP/1.1\r', '1.1\r\n', 'Head', 'Header:', 'Value', 'Value\r\n\r\n']) { + assert.equal(parse(sample), 0) + } + assert.equal(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_incomplete_string_2', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, true) + const message = 'GET / HTTP/1.1\r\nHost: foo\r\n\r\n' + + assert.equal(parse('GE'), 0) + assert.equal(parse(message), message.length) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_incomplete_string_automanaged_1', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldManageUnconsumed(parser, true) + const message = 'GET / HTTP/1.1\r\nHeader: Value\r\n\r\n' + const samples = [ + message.slice(0, 15), + message.slice(15, 16), + message.slice(16, 20), + message.slice(20, 24), + message.slice(24, 29), + message.slice(29) + ] + const consumed = [0, 16, 0, 0, 0, message.length - 16] + + for (let pass = 1; pass <= 2; pass++) { + for (let i = 0; i < samples.length; i++) { + assert.equal(parse(samples[i]), consumed[i]) + } + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) + assert.equal(milo.getParsed(parser), BigInt(message.length * pass)) + if (pass === 1) { + // Reset must preserve both the parsed counter and automatic buffering. + milo.reset(parser, true) + } + } +}) + +it('basic_incomplete_string_automanaged_2', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldManageUnconsumed(parser, true) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, true) + const message = 'GET / HTTP/1.1\r\nHost: foo\r\n\r\n' + + parse(message.slice(0, 2)) + parse(message.slice(2)) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) + assert.equal(milo.getParsed(parser), BigInt(message.length)) +}) + +it('basic_sample_multiple_requests', t => { + const { milo, parser, parse } = createParser(t, setup) + parse( + http(String.raw` + POST /chunked_w_unicorns_after_length HTTP/1.1\r\n + Transfer-Encoding: chunked\r\n + \r\n + 5;ilovew3;somuchlove=aretheseparametersfor\r\n + hello\r\n + 7;blahblah;blah\r\n + \s world\r\n + 0\r\n\r\n + \r\n + POST / HTTP/1.1\r\n + Host: www.example.com\r\n + Content-Type: application/x-www-form-urlencoded\r\n + Content-Length: 4\r\n + \r\n + q=42\r\n + \r\n + GET / HTTP/1.1\r\n\r\n + `) + ) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_connection_close', t => { + const { milo, parser, parse } = createParser(t, setup) + parse( + http(String.raw` + POST /chunked_w_unicorns_after_length HTTP/1.1\r\n + Connection: close\r\n + Transfer-Encoding: chunked\r\n + \r\n + 5;ilovew3;somuchlove=aretheseparametersfor\r\n + hello\r\n + 7;blahblah;blah\r\n + \s world\r\n + 0\r\n\r\n + `) + ) + assert.equal(milo.getState(parser), milo.STATE_FINISH) +}) + +it('basic_max_body_payload_content_length', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + milo.setMaxBodyPayload(parser, 3n) + const message = 'HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nabcdef' + const bodyStart = message.indexOf('\r\n\r\n') + 4 + + assert.equal(parse(message), bodyStart + 3) + assert.equal(milo.getRemainingContentLength(parser), 3n) + assert.equal(milo.isPaused(parser), false) + assert.equal(parse(message.slice(bodyStart + 3)), 3) + assert.equal(milo.getRemainingContentLength(parser), 0n) +}) + +it('basic_max_body_payload_chunked', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + milo.setMaxBodyPayload(parser, 3n) + const message = 'HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nabcdef\r\n0\r\n\r\n' + const chunkDataStart = message.indexOf('\r\n\r\n6\r\n') + 7 + + assert.equal(parse(message), chunkDataStart + 3) + assert.equal(milo.getRemainingChunkSize(parser), 3n) + assert.equal(milo.isPaused(parser), false) + const remaining = message.slice(chunkDataStart + 3) + assert.equal(parse(remaining), remaining.length) + assert.equal(milo.getRemainingChunkSize(parser), 0n) +}) + +it('basic_max_body_payload_no_length', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + milo.setMaxBodyPayload(parser, 3n) + const message = 'HTTP/1.1 200 OK\r\n\r\nabcdef' + const bodyStart = message.indexOf('\r\n\r\n') + 4 + + assert.equal(parse(message), bodyStart + 3) + assert.equal(milo.isPaused(parser), false) + assert.equal(parse(message.slice(bodyStart + 3)), 3) +}) + +it('basic_max_body_payload_zero_is_unlimited', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + milo.setMaxBodyPayload(parser, 0n) + const message = 'HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nabcdef' + + assert.equal(parse(message), message.length) + assert.equal(milo.getRemainingContentLength(parser), 0n) +}) + +it('basic_suspend_after_headers_content_length', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldSuspendAfterHeaders(parser, true) + const message = 'POST / HTTP/1.1\r\nContent-Length: 6\r\n\r\nabcdef' + const bodyStart = message.indexOf('\r\n\r\n') + 4 + + assert.equal(parse(message), bodyStart) + assert.equal(milo.getState(parser), milo.STATE_BODY_DECISION) + assert.equal(milo.isPaused(parser), false) + assert.equal(parse(message.slice(bodyStart)), message.length - bodyStart) + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +it('basic_suspend_after_headers_emits_headers_once', t => { + const { milo, parser, parse, headers } = createParser(t, setup) + milo.setActiveCallbacks(parser, milo.CALLBACK_ACTIVE_ON_HEADERS) + milo.setShouldSuspendAfterHeaders(parser, true) + const message = 'POST / HTTP/1.1\r\nContent-Length: 6\r\n\r\nabcdef' + const bodyStart = message.indexOf('\r\n\r\n') + 4 + + assert.equal(parse(message), bodyStart) + assert.equal(parse(message.slice(bodyStart)), message.length - bodyStart) + assert.equal(headers.length, 1) +}) + +it('basic_suspend_after_headers_zero_body_completes_on_empty_parse', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldSuspendAfterHeaders(parser, true) + const message = 'HTTP/1.1 204 No Content\r\n\r\n' + + assert.equal(parse(message), message.length) + assert.equal(milo.getState(parser), milo.STATE_BODY_DECISION) + assert.equal(parse(''), 0) + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +it('basic_complete_after_suspend_after_headers', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldSuspendAfterHeaders(parser, true) + const message = 'POST / HTTP/1.1\r\nContent-Length: 6\r\n\r\nabcdef' + const bodyStart = message.indexOf('\r\n\r\n') + 4 + + assert.equal(parse(message), bodyStart) + assert.equal(milo.getState(parser), milo.STATE_BODY_DECISION) + milo.complete(parser) + assert.equal(milo.getState(parser), milo.STATE_START) + assert.equal(milo.getErrorCode(parser), milo.ERROR_NONE) +}) + +it('basic_complete_rejects_invalid_state', t => { + const { milo, parser } = createParser(t, setup) + milo.complete(parser) + assert.equal(milo.getState(parser), milo.STATE_ERROR) + assert.equal(milo.getErrorCode(parser), milo.ERROR_UNEXPECTED_STATE) + assert.equal(milo.getErrorDescription(parser), 'Invalid state') +}) + +it('basic_event_buffer_full_stops_parsing', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + milo.setActiveCallbacks(parser, 0n) + milo.setActiveEvents(parser, milo.EVENT_ACTIVE_ON_HEADER_NAME | milo.EVENT_ACTIVE_ON_HEADER_VALUE) + let message = 'HTTP/1.1 200 OK\r\n' + for (let i = 0; i < 4000; i++) { + message += `Header${i}: value\r\n` + } + message += '\r\n' + + assert.ok(parse(message) < message.length) + assert.equal(milo.isPaused(parser), false) + assert.equal(milo.getErrorCode(parser), milo.ERROR_NONE) +}) + +it('basic_sample_multiple_responses', t => { + const { milo, parser, parse } = createParser(t, setup) + parse( + http(String.raw` + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + `) + ) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_trailers', t => { + const { milo, parser, parse } = createParser(t, setup) + parse( + http(String.raw` + POST /chunked_w_unicorns_after_length HTTP/1.1\r\n + Transfer-Encoding: chunked\r\n + Trailer: host,cache-control\r\n + \r\n + 5;ilovew3;somuchlove="arethesepara\"metersfor";another="1111\"2222\"3333"\r\n + hello\r\n + 7;blahblah;blah;somuchlove="arethesepara"\r\n + \s world\r\n + 0\r\n + Host: example.com\r\n + Cache-Control: private\r\n\r\n + `) + ) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_incomplete_body', t => { + const { milo, parser, parse } = createParser(t, setup) + for (const sample of ['POST / HTTP/1.1\r\nContent-Length:10\r\n\r\n12345', '67', '890\r\n']) { + assert.equal(parse(sample), sample.length) + } + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_incomplete_chunk', t => { + const { milo, parser, parse } = createParser(t, setup) + const samples = [ + 'POST / HTTP/1.1\r\nTransfer-Encoding:chunked\r\nTrailer: x-foo\r\n\r\na\r\n12345', + '67', + '890\r\n0\r\nx-foo:value\r\n\r\n' + ] + for (const sample of samples) { + assert.equal(parse(sample), sample.length) + } + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_connection_header', t => { + const { milo, parser, parse } = createParser(t, setup) + parse('PUT /url HTTP/1.1\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc') + assert.equal(milo.getState(parser), milo.STATE_FINISH) + milo.reset(parser, false) + parse('PUT /url HTTP/1.1\r\nContent-Length: 3\r\n\r\nabc') + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +it('basic_pause_and_resume', t => { + const { milo, parser, parse } = createParser(t, setup) + const sample1 = 'PUT /url HTTP/1.1\r\nContent-Length: 3\r\n' + const sample2 = '\r\nabc' + + assert.equal(milo.isPaused(parser), false) + assert.equal(parse(sample1), sample1.length) + assert.equal(milo.isPaused(parser), false) + milo.pause(parser) + assert.equal(milo.isPaused(parser), true) + assert.equal(parse(sample2), 0) + assert.equal(milo.isPaused(parser), true) + milo.resume(parser) + assert.equal(milo.isPaused(parser), false) + assert.equal(parse(sample2), sample2.length) + assert.equal(milo.isPaused(parser), false) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_restart', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + const response = http(String.raw` + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc\r\n + HTTP/1.1 200 OK\r\n + Header1: Value1\r\n + Header2: Value2\r\n + Content-Length: 3\r\n + \r\n + abc + `) + const request = 'PUT /url HTTP/1.1\r\nContent-Length: 3\r\nConnection: keep-alive\r\n\r\nabc' + + parse(response) + assert.equal(milo.getState(parser), milo.STATE_START) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, true) + milo.reset(parser, false) + parse(request) + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +it('basic_finish_logic', t => { + const { milo, parser, parse } = createParser(t, setup) + assert.equal(milo.getState(parser), milo.STATE_START) + milo.finish(parser) + assert.equal(milo.getState(parser), milo.STATE_FINISH) + milo.reset(parser, false) + + parse('PUT /url HTTP/1.1\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc') + assert.equal(milo.getState(parser), milo.STATE_FINISH) + milo.finish(parser) + assert.equal(milo.getState(parser), milo.STATE_FINISH) + milo.reset(parser, false) + + parse('PUT /url HTTP/1.1\r\nContent-Length: 3\r\n\r\nabc') + assert.equal(milo.getState(parser), milo.STATE_START) + milo.finish(parser) + assert.equal(milo.getState(parser), milo.STATE_FINISH) + milo.reset(parser, false) + + parse('PUT /url HTTP/1.1\r\n') + assert.equal(milo.getState(parser), milo.STATE_HEADER) + milo.finish(parser) + assert.equal(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_empty_fields', t => { + const { milo, parser, parse } = createParser(t, setup) + parse( + http(String.raw` + POST / HTTP/1.1\r\n + Transfer-Encoding: chunked\r\n + Content-Type: \r\n + Trailer: host\r\n + \r\n + 0\r\n + Host:\r\n\r\n + `) + ) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_space_after_header_name', t => { + const { milo, parser, parse } = createParser(t, setup) + parse('PUT /url HTTP/1.1\r\nContent-Length : 3\r\n\r\nabc\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_ERROR) +}) + +it('basic_response_204_has_no_body', t => { + const { milo, parser, parse } = createParser(t, setup) + milo.setShouldAutodetect(parser, false) + milo.setIsRequest(parser, false) + parse('HTTP/1.1 204 No content\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok') + assert.equal(milo.getState(parser), milo.STATE_START) +}) diff --git a/parser/wasm/test/benchmark.test.js b/parser/wasm/test/benchmark.test.js new file mode 100644 index 0000000..68267ce --- /dev/null +++ b/parser/wasm/test/benchmark.test.js @@ -0,0 +1,45 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, http, setup } from './helpers.js' + +it('benchmark_seanmonstar_httparse', t => { + const message = http(String.raw` + GET /wp-content/uploads/2010/03/hello-kitty-darth-vader-pink.jpg HTTP/1.1\r\n + Host: www.kittyhell.com\r\n + User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; ja-JP-mac; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 Pathtraq/0.9\r\n + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n + Accept-Language: ja,en-us;q=0.7,en;q=0.3\r\n + Accept-Encoding: gzip,deflate\r\n + Accept-Charset: Shift_JIS,utf-8;q=0.7,*;q=0.7\r\n + Keep-Alive: 115\r\n + Connection: keep-alive\r\n + Cookie: wp_ozh_wsa_visits=2; wp_ozh_wsa_visit_lasttime=xxxxxxxxxx; __utma=xxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.xxxxxxxxxx.x; __utmz=xxxxxxxxx.xxxxxxxxxx.x.x.utmccn=(referral)|utmcsr=reader.livedoor.com|utmcct=/reader/|utmcmd=referral\r\n\r\n + `) + const { milo, parser, parse } = createParser(t, setup) + + assert.equal(parse(message), message.length) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('benchmark_nodejs_http_parser', t => { + const message = http(String.raw` + POST /joyent/http-parser HTTP/1.1\r\n + Host: github.com\r\n + DNT: 1\r\n + Accept-Encoding: gzip, deflate, sdch\r\n + Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4\r\n + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) + AppleWebKit/537.36 (KHTML, like Gecko) + Chrome/39.0.2171.65 Safari/537.36\r\n + Accept: text/html,application/xhtml+xml,application/xml;q=0.9, + image/webp,*/*;q=0.8\r\n + Referer: https://github.com/joyent/http-parser\r\n + Connection: keep-alive\r\n + Transfer-Encoding: chunked\r\n + Cache-Control: max-age=0\r\n\r\nb\r\nhello world\r\n0\r\n\r\n + `) + const { milo, parser, parse } = createParser(t, setup) + + assert.equal(parse(message), message.length) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) diff --git a/parser/wasm/test/compliance.test.js b/parser/wasm/test/compliance.test.js new file mode 100644 index 0000000..a0cd1bf --- /dev/null +++ b/parser/wasm/test/compliance.test.js @@ -0,0 +1,398 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, setup } from './helpers.js' + +function requestParser (t) { + const context = createParser(t, setup) + context.milo.setShouldAutodetect(context.parser, false) + context.milo.setIsRequest(context.parser, true) + return context +} + +function responseParser (t) { + const context = createParser(t, setup) + context.milo.setShouldAutodetect(context.parser, false) + context.milo.setIsRequest(context.parser, false) + return context +} + +function assertOk (milo, parser) { + assert.notEqual(milo.getState(parser), milo.STATE_ERROR, milo.getErrorDescription(parser)) + assert.equal(milo.getErrorCode(parser), milo.ERROR_NONE) +} + +function assertError (milo, parser) { + assert.equal(milo.getState(parser), milo.STATE_ERROR) +} + +// RFC token syntax allows `|` in header names. +it('compliance_header_name_allows_pipe', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nX|Y: z\r\nContent-Length: 0\r\n\r\n') + assertOk(milo, parser) +}) + +// RFC token syntax rejects `,` in header names. +it('compliance_header_name_rejects_comma', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nX,Y: z\r\nContent-Length: 0\r\n\r\n') + assertError(milo, parser) +}) + +// RFC token syntax allows `|` in trailer names when chunked framing is valid. +it('compliance_trailer_name_allows_pipe', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX|Y: z\r\n\r\n') + assertOk(milo, parser) +}) + +// RFC token syntax rejects `,` in trailer names. +it('compliance_trailer_name_rejects_comma', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX,Y: z\r\n\r\n') + assertError(milo, parser) +}) + +// Unknown valid method tokens are accepted as extension methods. +it('compliance_unknown_method_token_is_accepted', t => { + const { milo, parser, parse } = requestParser(t) + parse('FOO|BAR / HTTP/1.1\r\n\r\n') + assertOk(milo, parser) + assert.equal(milo.getMethod(parser), milo.METHOD_OTHER) +}) + +// Invalid unknown method tokens are rejected. +it('compliance_unknown_method_token_rejects_comma', t => { + const { milo, parser, parse } = requestParser(t) + parse('BAD,METHOD / HTTP/1.1\r\n\r\n') + assertError(milo, parser) +}) + +// PRI is only accepted with HTTP/2.0 for switch-over tunneling. +it('compliance_pri_requires_http2', t => { + const { milo, parser, parse } = requestParser(t) + parse('PRI * HTTP/1.1\r\n\r\n') + assertError(milo, parser) +}) + +// PRI with HTTP/2.0 enters tunnel mode instead of parsing HTTP/1.1 headers. +it('compliance_pri_http2_enters_tunnel', t => { + const { milo, parser, parse } = requestParser(t) + parse('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) + +// HTTP/2.0 is rejected for normal requests. +it('compliance_http2_request_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('GET / HTTP/2.0\r\n\r\n') + assertError(milo, parser) +}) + +// HTTP/2.0 is rejected for responses. +it('compliance_http2_response_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/2.0 200 OK\r\n\r\n') + assertError(milo, parser) +}) + +// PRI with HTTP/2.0 must be followed by the exact HTTP/2 connection preface suffix. +it('compliance_pri_http2_invalid_preface_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('PRI * HTTP/2.0\r\ngarbage\r\n\r\n') + assertError(milo, parser) +}) + +// RTSP is not detected or accepted as an HTTP response protocol. +it('compliance_rtsp_response_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('RTSP/1.0 200 OK\r\n\r\n') + assertError(milo, parser) +}) + +// Request targets cannot contain fragments. +it('compliance_request_target_rejects_fragment', t => { + const { milo, parser, parse } = requestParser(t) + parse('GET /path#fragment HTTP/1.1\r\n\r\n') + assertError(milo, parser) +}) + +// Milo intentionally rejects bodies on GET requests. +it('compliance_get_body_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('GET / HTTP/1.1\r\nContent-Length: 1\r\n\r\nx') + assertError(milo, parser) +}) + +// Milo intentionally rejects bodies on HEAD requests. +it('compliance_head_body_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('HEAD / HTTP/1.1\r\nContent-Length: 1\r\n\r\nx') + assertError(milo, parser) +}) + +// Methods other than GET and HEAD can carry valid body framing. +it('compliance_post_body_accepted', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nContent-Length: 1\r\n\r\nx') + assertOk(milo, parser) +}) + +// 205 responses complete after headers like other no-body statuses. +it('compliance_205_without_body_completes', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 205 Reset Content\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +// 205 responses reject Content-Length as body framing. +it('compliance_205_content_length_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 205 Reset Content\r\nContent-Length: 0\r\n\r\n') + assertError(milo, parser) +}) + +// 205 responses reject Transfer-Encoding as body framing. +it('compliance_205_transfer_encoding_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 205 Reset Content\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// No-body responses except 304 reject Content-Length in strict Milo mode. +it('compliance_no_body_status_content_length_rejected', t => { + for (const status of ['100 Continue', '204 No Content', '205 Reset Content']) { + const { milo, parser, parse } = responseParser(t) + parse(`HTTP/1.1 ${status}\r\nContent-Length: 0\r\n\r\n`) + assertError(milo, parser) + } +}) + +// 304 allows Content-Length as metadata but still has no body. +it('compliance_304_content_length_accepted_without_body', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 304 Not Modified\r\nContent-Length: 10\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n') + assertOk(milo, parser) + assert.equal(milo.getState(parser), milo.STATE_START) +}) + +// No-body responses reject Transfer-Encoding in strict Milo mode. +it('compliance_no_body_status_transfer_encoding_rejected', t => { + for (const status of ['100 Continue', '204 No Content', '304 Not Modified']) { + const { milo, parser, parse } = responseParser(t) + parse(`HTTP/1.1 ${status}\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n`) + assertError(milo, parser) + } +}) + +// Trailer is invalid without chunked transfer coding. +it('compliance_trailer_without_chunked_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nTrailer: X\r\n\r\n') + assertError(milo, parser) +}) + +// Upgrade does not bypass Trailer validation. +it('compliance_upgrade_trailer_without_chunked_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: websocket\r\nTrailer: X\r\n\r\n') + assertError(milo, parser) +}) + +// Request upgrade may parse a chunked body and trailers before tunneling. +it('compliance_request_upgrade_chunked_trailers_before_tunnel', t => { + const { milo, parser, parse } = requestParser(t) + parse( + 'POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: websocket\r\nTransfer-Encoding: chunked\r\nTrailer: X\r\n\r\n0\r\nX: y\r\n\r\n' + ) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) + +// Response upgrade cannot use Trailer without valid chunked framing. +it('compliance_response_upgrade_trailer_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 101 Switching Protocols\r\nConnection: upgrade\r\nUpgrade: websocket\r\nTrailer: X\r\n\r\n') + assertError(milo, parser) +}) + +// Valid unknown Connection options are accepted and ignored. +it('compliance_connection_unknown_token_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nConnection: foo\r\nContent-Length: 0\r\n\r\n') + assertOk(milo, parser) +}) + +// Invalid unknown Connection options are rejected. +it('compliance_connection_unknown_token_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nConnection: foo@bar\r\nContent-Length: 0\r\n\r\n') + assertError(milo, parser) +}) + +// Empty Connection list items are rejected. +it('compliance_connection_empty_item_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nConnection: close,,upgrade\r\nContent-Length: 0\r\n\r\n') + assertError(milo, parser) +}) + +// Connection close finishes the parser and rejects subsequent data. +it('compliance_connection_close_rejects_later_data', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\n\r\nx') + assertError(milo, parser) +}) + +// Upgrade values are comma-separated protocol tokens without special known values. +it('compliance_upgrade_tokens_accepted', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: foo, HTTP/2.0\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) + +// Upgrade protocol values reject empty protocol names. +it('compliance_upgrade_empty_protocol_name_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: /2.0\r\n\r\n') + assertError(milo, parser) +}) + +// Upgrade protocol values reject empty protocol versions. +it('compliance_upgrade_empty_protocol_version_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: HTTP/\r\n\r\n') + assertError(milo, parser) +}) + +// Upgrade protocol values reject more than one protocol version separator. +it('compliance_upgrade_extra_protocol_separator_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: HTTP/2/extra\r\n\r\n') + assertError(milo, parser) +}) + +// Empty Upgrade values are rejected. +it('compliance_upgrade_empty_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: \r\n\r\n') + assertError(milo, parser) +}) + +// Invalid Upgrade tokens are rejected. +it('compliance_upgrade_invalid_token_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: foo@bar\r\n\r\n') + assertError(milo, parser) +}) + +// Empty Upgrade list items are rejected. +it('compliance_upgrade_empty_item_rejected', t => { + const { milo, parser, parse } = requestParser(t) + parse('POST / HTTP/1.1\r\nConnection: upgrade\r\nUpgrade: foo,,bar\r\n\r\n') + assertError(milo, parser) +}) + +// Unquoted chunk extension values must be RFC tokens. +it('compliance_chunk_extension_unquoted_token_value', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo=bar|baz\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Unquoted chunk extension values reject spaces. +it('compliance_chunk_extension_unquoted_space_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo=bar baz\r\nx\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// Unquoted chunk extension values reject non-token characters. +it('compliance_chunk_extension_unquoted_at_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo=bar@baz\r\nx\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// Quoted chunk extension values may contain spaces. +it('compliance_chunk_extension_quoted_space_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar baz"\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Quoted chunk extension values may contain quoted-pair escaped quotes. +it('compliance_chunk_extension_quoted_escaped_quote_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\\"baz"\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Quoted chunk extension values may contain quoted-pair escaped backslashes. +it('compliance_chunk_extension_quoted_escaped_backslash_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\\\\baz"\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Quoted chunk extension values may contain horizontal tabs. +it('compliance_chunk_extension_quoted_tab_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\tbaz"\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Quoted chunk extension values may contain obs-text. +it('compliance_chunk_extension_quoted_obs_text_accepted', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\u0080baz"\r\nx\r\n0\r\n\r\n') + assertOk(milo, parser) +}) + +// Quoted chunk extension values reject bare control characters other than HTAB. +it('compliance_chunk_extension_quoted_control_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\u0001baz"\r\nx\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// Quoted-pair in chunk extension values rejects escaped control characters. +it('compliance_chunk_extension_quoted_pair_control_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n1;foo="bar\\\u0001baz"\r\nx\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// Bare CR is rejected in HTTP framing. +it('compliance_bare_cr_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\rContent-Length: 0\r\r') + assertError(milo, parser) +}) + +// Obsolete folded headers are rejected. +it('compliance_obs_fold_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nHeader: value\r\n folded\r\nContent-Length: 0\r\n\r\n') + assertError(milo, parser) +}) + +// Chunked transfer coding must be final. +it('compliance_chunked_must_be_final', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked, gzip\r\n\r\n') + assertError(milo, parser) +}) + +// Content-Length cannot be combined with Transfer-Encoding. +it('compliance_content_length_transfer_encoding_conflict', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nContent-Length: 1\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n') + assertError(milo, parser) +}) + +// Connection close still finishes cleanly when no later data is received. +it('compliance_connection_close_finishes', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 0\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_FINISH) +}) diff --git a/parser/wasm/test/helpers.js b/parser/wasm/test/helpers.js new file mode 100644 index 0000000..97a8877 --- /dev/null +++ b/parser/wasm/test/helpers.js @@ -0,0 +1,38 @@ +// Exercise the generated packages, including their JavaScript bindings. +process.env.MILO_VARIANT ??= 'simd' +export const { setup } = await import(`../../../dist/wasm/release/package/src/${process.env.MILO_VARIANT}/index.js`) + +export function createParser (t, setup) { + const headers = [] + // Each test owns an instance, so its linear memory and input allocations are + // isolated and become collectible together when the test finishes. + const milo = setup({ + on_headers (parser, at, len) { + headers.push({ parser, at, len }) + } + }) + const parser = milo.create() + milo.setActiveCallbacks(parser, milo.CALLBACK_ACTIVE_ALL) + t.after(() => milo.destroy(parser)) + + function parse (input) { + const bytes = Buffer.from(input) + const ptr = milo.alloc(bytes.length) + // Allocation can grow memory and invalidate previously acquired views. + new Uint8Array(milo.memory.buffer, ptr, bytes.length).set(bytes) + return milo.parse(parser, ptr, bytes.length) + } + + return { milo, parser, parse, headers } +} + +// Match the HTTP fixture normalization in parser/tests/helpers/mod.rs. +export function http (input) { + return input + .trim() + .replace(/^\s+/gm, '') + .replaceAll('\n', '') + .replaceAll('\\r', '\r') + .replaceAll('\\n', '\n') + .replaceAll('\\s', ' ') +} diff --git a/parser/wasm/test/issue.test.js b/parser/wasm/test/issue.test.js new file mode 100644 index 0000000..0c1f25c --- /dev/null +++ b/parser/wasm/test/issue.test.js @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, setup } from './helpers.js' + +function responseParser (t) { + const context = createParser(t, setup) + context.milo.setShouldAutodetect(context.parser, false) + context.milo.setIsRequest(context.parser, false) + return context +} + +function fieldMessages (value) { + return [ + ['HTTP/1.1 200 OK\r\nX-Long: ', '\r\nContent-Length: 0\r\n\r\n'], + ['HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nX-Long: ', '\r\n\r\n'], + ['HTTP/1.1 200 ', '\r\nContent-Length: 0\r\n\r\n'] + ].map(([prefix, suffix]) => Buffer.concat([Buffer.from(prefix), value, Buffer.from(suffix)])) +} + +// Cover SIMD block boundaries and scalar tails in both field scanners. +it('issue_22__field_values_reject_controls', t => { + const { milo, parser, parse } = responseParser(t) + milo.setActiveCallbacks(parser, 0n) + const controls = [...Array.from({ length: 32 }, (_, byte) => byte), 0x7f].filter(byte => byte !== 9) + + for (const byte of controls) { + for (const offset of [0, 7, 8, 15, 16, 20, 31, 32]) { + for (const trailing of [0, 20]) { + const value = Buffer.concat([Buffer.alloc(offset, 'a'), Buffer.from([byte]), Buffer.alloc(trailing, 'a')]) + for (const message of fieldMessages(value)) { + milo.reset(parser, false) + parse(message) + assert.equal(milo.getState(parser), milo.STATE_ERROR, `Accepted invalid field: ${message.toString('hex')}`) + } + } + } + } +}) + +// HTAB and every obs-text byte remain valid, including across SIMD boundaries. +it('issue_22__field_values_allow_tab_and_obs_text', t => { + const { milo, parser, parse } = responseParser(t) + // Keep the callback configuration identical to the raw-byte Rust regression. + milo.setActiveCallbacks(parser, 0n) + const allowed = [9, ...Array.from({ length: 128 }, (_, byte) => byte + 0x80)] + + for (const byte of allowed) { + for (const offset of [0, 7, 8, 15, 16, 20, 31, 32]) { + for (const trailing of [0, 20]) { + const value = Buffer.concat([Buffer.alloc(offset, 'a'), Buffer.from([byte]), Buffer.alloc(trailing, 'a')]) + for (const message of fieldMessages(value)) { + milo.reset(parser, false) + parse(message) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR, `Rejected valid field: ${message.toString('hex')}`) + assert.equal(milo.getErrorCode(parser), milo.ERROR_NONE) + } + } + } + } +}) + +// Bare LF is rejected in HTTP framing. +it('issue_22__bare_lf_rejected', t => { + const { milo, parser, parse } = responseParser(t) + parse('HTTP/1.1 200 OK\r\nHeader: value\nContent-Length: 0\r\n\r\n') + assert.equal(milo.getState(parser), milo.STATE_ERROR) +}) diff --git a/parser/wasm/test/undici.test.js b/parser/wasm/test/undici.test.js new file mode 100644 index 0000000..c42a641 --- /dev/null +++ b/parser/wasm/test/undici.test.js @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, http, setup } from './helpers.js' + +it('undici', t => { + const message = http(String.raw` + HTTP/1.1 200 OK\r\n + Connection: keep-alive\r\n + Content-Length: 65535\r\n + Date: Sun, 05 Nov 2023 14:26:18 GMT\r\n + Keep-Alive: timeout=600\r\n\r\n + @ + `).replaceAll('@', '-'.repeat(65535)) + const { milo, parser, parse } = createParser(t, setup) + + parse(message) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) + assert.equal(milo.getParsed(parser), BigInt(Buffer.byteLength(message))) +}) + +it('undici_multiple', t => { + const message = http(String.raw` + HTTP/1.1 200 OK\r\n + Date: Mon, 08 Apr 2024 13:20:53 GMT\r\n + Connection: keep-alive\r\n + Keep-Alive: timeout=5\r\n + Transfer-Encoding: chunked\r\n + \r\n + 3e80\r\n + @\r\n + 3e80\r\n + @\r\n + 0\r\n\r\n + `).replaceAll('@', '-'.repeat(16000)) + const { milo, parser, parse } = createParser(t, setup) + + parse(message) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) +}) + +it('undici_multibyte', t => { + const message = http(String.raw` + HTTP/1.1 200 OK\r\n + Date: Tue, 09 Apr 2024 10:39:04 GMT\r\n + Connection: keep-alive\r\n + Keep-Alive: timeout=5\r\n + Content-Length: 300010\r\n + \r\n + {"asd":"@#"} + `) + .replaceAll('@', 'あ'.repeat(50000)) + .replaceAll('#', 'あ'.repeat(50000)) + const { milo, parser, parse } = createParser(t, setup) + const bytes = Buffer.from(message) + + // Split bytes, not JavaScript characters: a chunk may end inside a UTF-8 sequence. + for (let offset = 0; offset < bytes.length; offset += 65536) { + parse(bytes.subarray(offset, offset + 65536)) + assert.notEqual(milo.getState(parser), milo.STATE_ERROR) + } +}) diff --git a/parser/wasm/test/upgrade.test.js b/parser/wasm/test/upgrade.test.js new file mode 100644 index 0000000..deaba81 --- /dev/null +++ b/parser/wasm/test/upgrade.test.js @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict' +import { it } from 'node:test' +import { createParser, http, setup } from './helpers.js' + +it('upgrade_connect_request', t => { + const { milo, parser, parse } = createParser(t, setup) + const message1 = http(String.raw` + CONNECT example.com HTTP/1.1\r\n + Host: example.com\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + `) + const message2 = 'abc\r\n\r\n' + + assert.equal(parse(message1), 70) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) + assert.equal(parse(message2), 0) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) + +it('upgrade_connection_upgrade', t => { + const { milo, parser, parse } = createParser(t, setup) + const message1 = http(String.raw` + POST / HTTP/1.1\r\n + Host: example.com\r\n + Connection: upgrade\r\n + Upgrade: websocket\r\n + Content-Length: 3\r\n + \r\n + abc\r\n\r\n + `) + const message2 = 'abc\r\n\r\n' + + assert.equal(parse(message1), message1.length - 4) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) + assert.equal(parse(message2), 0) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) + +it('upgrade_http_101', t => { + const { milo, parser, parse } = createParser(t, setup) + const message1 = http(String.raw` + HTTP/1.1 101 Switching Protocols\r\n + hello: world\r\n + connection: upgrade\r\n + upgrade: websocket\r\n + \r\n + Body + `) + const message2 = 'abc\r\n\r\n' + + assert.equal(parse(message1), message1.length - 4) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) + assert.equal(parse(message2), 0) + assert.equal(milo.getState(parser), milo.STATE_TUNNEL) +}) diff --git a/scripts/src/bin/postbuild-wasm.rs b/scripts/src/bin/postbuild-wasm.rs index 92778a7..33a2482 100644 --- a/scripts/src/bin/postbuild-wasm.rs +++ b/scripts/src/bin/postbuild-wasm.rs @@ -216,7 +216,7 @@ fn generate_simple_callbacks(info: &BuildInfo) -> String { } fn generate_module(profile: &str, info: &BuildInfo, loader: &str, commonjs: bool) -> Result> { - let template = fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("../parser/src/wasm/template.js"))?; + let template = fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join("../parser/wasm/src/template.js"))?; let version = &info.version; let mut output = String::new(); @@ -351,7 +351,7 @@ fn main() -> Result<(), Box> { let profile = env::args().nth(1).ok_or("Usage: postbuild-wasm ")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); // Open the package.json and update the version. - let mut package_json: Value = serde_json::from_str(&fs::read_to_string(root.join("parser/src/wasm/package.json"))?)?; + let mut package_json: Value = serde_json::from_str(&fs::read_to_string(root.join("parser/wasm/src/package.json"))?)?; let profile_root = root.join("dist/wasm").join(&profile); let root_folder = profile_root.join("package"); let cjs_root_folder = profile_root.join("package-cjs"); diff --git a/scripts/src/bin/sync-versions.rs b/scripts/src/bin/sync-versions.rs index e59b7cf..85c171c 100644 --- a/scripts/src/bin/sync-versions.rs +++ b/scripts/src/bin/sync-versions.rs @@ -9,7 +9,7 @@ use std::{ use regex::{Captures, Regex}; fn main() -> Result<(), Box> { - let folders = ["macros", "parser", "references/rust", "parser/src/wasm"]; + let folders = ["macros", "parser", "references/rust", "parser/wasm/src"]; let version = env::args().nth(1).ok_or("Usage: sync-versions ")?; let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");