Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions conformance/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,11 @@ compression level, only the block type and the encodings it selects:
| `empty` | No data block: EOF only, `src_size = 0` |
| `all_256_values` | RAW block; all 256 byte values |
| `all_zeros_4k` | Offset-1 overlap run (long match at minimum distance) |
| `max_offset_128k` | Maximum back-reference distance; GLO `enc_off=0` |
| `text_64k_level1` | The GHI block type |
| `glo_rle_4k` | RLE literals, `enc_lit=1` |
| `glo_pivco_wide_l7` | PivCo literals, `enc_lit=2` |
| `glo_tok_huffman_l7` | Huffman-coded token section, `enc_tok=2` |
| `max_offset_128k` | Maximum back-reference distance; SPL `enc_off=0` |
| `text_64k_level1` | The PAK block type |
| `spl_rle_4k` | RLE literals, `enc_lit=1` |
| `spl_pivco_wide_l7` | PivCo literals, `enc_lit=2` |
| `spl_tok_huffman_l7` | Huffman-coded token section, `enc_tok=2` |
| `random_4k_checksum` | Checksums on their own, clear of the dictionary path |
| `multiblock_checksum`| Checksums past block 0, where the seed is the block index |
| `multiblock_mixed` | 16 data blocks; the 4 KB minimum chunk size |
Expand Down
94 changes: 47 additions & 47 deletions conformance/invalid_cases.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

/* Offset of the first block header; blocks follow the file header. */
#define BLK0 ZXC_FILE_HEADER_SIZE
/* Offset of the first block's payload (its GLO/GHI sub-header). */
/* Offset of the first block's payload (its SPL/PAK sub-header). */
#define PAY0 (BLK0 + ZXC_BLOCK_HEADER_SIZE)

/* Expected rejection reason, per vector.
Expand Down Expand Up @@ -55,9 +55,9 @@ static const invalid_expect_t INVALID_EXPECT[] = {
{"bad_version", .expected = ZXC_ERROR_BAD_VERSION},
{"corrupt_payload", ZXC_ERROR_BAD_CHECKSUM, .generated = 1},
{"dict_required", ZXC_ERROR_DICT_REQUIRED, .generated = 1},
{"ghi_forged_offset", ZXC_ERROR_BAD_OFFSET, .generated = 1},
{"glo_forged_enc_off", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"glo_insufficient_slack", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"pak_forged_offset", ZXC_ERROR_BAD_OFFSET, .generated = 1},
{"spl_forged_enc_off", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"spl_insufficient_slack", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"magic_then_zeros", .expected = ZXC_ERROR_BAD_VERSION},
{"too_short_4bytes", .expected = ZXC_ERROR_SRC_TOO_SMALL},
{"truncated_header_only", ZXC_ERROR_SRC_TOO_SMALL, .generated = 1},
Expand All @@ -73,8 +73,8 @@ static const invalid_expect_t INVALID_EXPECT[] = {
{"bad_block_header_checksum", ZXC_ERROR_BAD_HEADER, .generated = 1},
{"bad_footer_size", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"bad_footer_digest", ZXC_ERROR_BAD_CHECKSUM, .generated = 1},
{"glo_forged_offset", ZXC_ERROR_BAD_OFFSET, .generated = 1},
{"glo_output_overflow", ZXC_ERROR_OVERFLOW, .generated = 1},
{"spl_forged_offset", ZXC_ERROR_BAD_OFFSET, .generated = 1},
{"spl_output_overflow", ZXC_ERROR_OVERFLOW, .generated = 1},
{"varint_too_long", ZXC_ERROR_CORRUPT_DATA, .generated = 1},
{"dict_id_mismatch", ZXC_ERROR_DICT_MISMATCH, "dict_http.zxd", 0, .generated = 1},
};
Expand All @@ -99,7 +99,7 @@ static size_t find_eof_block(const uint8_t* d, size_t n, int has_checksum) {
return 0;
}

/* Compressible, deterministic, and long enough to make a GLO block. */
/* Compressible, deterministic, and long enough to make a SPL block. */
static size_t make_text(uint8_t** out) {
static const char unit[] = "the quick brown fox jumps over the lazy dog. ";
const size_t ulen = sizeof(unit) - 1, reps = 40, n = ulen * reps;
Expand Down Expand Up @@ -137,21 +137,21 @@ static size_t varint_len(uint8_t b0) {
return 4; /* out of spec: no legitimate value needs it */
}

/* Section offsets inside block 0's GLO payload, read off the wire. Returns 0
/* Section offsets inside block 0's SPL payload, read off the wire. Returns 0
* unless the block has the shape the patches assume, so a heuristic change
* makes the generator complain instead of shipping vectors that test nothing. */
typedef struct {
size_t tok, off, extras;
} glo_layout_t;
} spl_layout_t;

static int glo_layout(const uint8_t* d, size_t n, glo_layout_t* L) {
if (d[BLK0] != ZXC_BLOCK_GLO) return 0;
static int spl_layout(const uint8_t* d, size_t n, spl_layout_t* L) {
if (d[BLK0] != ZXC_BLOCK_SPL) return 0;
const uint32_t comp = zxc_le32(d + BLK0 + 3);
const uint32_t n_seq = zxc_le32(d + PAY0);
const uint32_t n_lit = zxc_le32(d + PAY0 + 4);
const uint8_t enc_lit = d[PAY0 + 8], enc_tok = d[PAY0 + 9], enc_off = d[PAY0 + 11];
if (enc_lit != 0 || enc_tok != 0 || n_seq == 0) return 0; /* RAW sections only */
L->tok = PAY0 + ZXC_GLO_HEADER_BINARY_SIZE + n_lit;
L->tok = PAY0 + ZXC_SPL_HEADER_BINARY_SIZE + n_lit;
L->off = L->tok + n_seq;
L->extras = L->off + (size_t)n_seq * (enc_off ? 1U : 2U);
if (L->extras + 8 > PAY0 + comp || L->extras + 8 > n) return 0;
Expand All @@ -164,8 +164,8 @@ static int glo_layout(const uint8_t* d, size_t n, glo_layout_t* L) {

/* The four well-formed archives every vector is patched from. Built once. */
typedef struct {
uint8_t *plain, *chk, *ghi, *seek;
size_t n_plain, n_chk, n_ghi, n_seek;
uint8_t *plain, *chk, *pak, *seek;
size_t n_plain, n_chk, n_pak, n_seek;
int state; /* 0 unbuilt, 1 ready, -1 failed (do not retry, do not leak) */
} invalid_bases_t;

Expand All @@ -190,29 +190,29 @@ static int invalid_bases(invalid_bases_t* b) {
zxc_compress_opts_t chk = plain;
chk.checksum_enabled = 1;

zxc_compress_opts_t ghi = plain;
ghi.level = 1; /* levels 1-2 emit GHI blocks */
zxc_compress_opts_t pak = plain;
pak.level = 1; /* levels 1-2 emit PAK blocks */

zxc_compress_opts_t seek = plain;
seek.seekable = 1; /* appends the SEK block sek_forged_entry patches */

b->n_plain = build_base(text, text_n, &plain, &b->plain);
b->n_chk = build_base(text, text_n, &chk, &b->chk);
b->n_ghi = build_base(text, text_n, &ghi, &b->ghi);
b->n_pak = build_base(text, text_n, &pak, &b->pak);
b->n_seek = build_base(text, text_n, &seek, &b->seek);
free(text);

if (!b->n_plain || !b->n_chk || !b->n_ghi || !b->n_seek) {
if (!b->n_plain || !b->n_chk || !b->n_pak || !b->n_seek) {
invalid_bases_free(b);
b->state = -1;
return 0;
}

/* Every GLO patch below addresses a fixed offset inside block 0's sub-header.
/* Every SPL patch below addresses a fixed offset inside block 0's sub-header.
* If a heuristic change ever made that block RAW, the patches would land on
* literal bytes and the vectors would quietly stop testing anything. */
if (b->plain[BLK0] != ZXC_BLOCK_GLO || b->chk[BLK0] != ZXC_BLOCK_GLO) {
fprintf(stderr, " block 0 is type %u/%u, expected GLO - vectors would test nothing\n",
if (b->plain[BLK0] != ZXC_BLOCK_SPL || b->chk[BLK0] != ZXC_BLOCK_SPL) {
fprintf(stderr, " block 0 is type %u/%u, expected SPL - vectors would test nothing\n",
b->plain[BLK0], b->chk[BLK0]);
invalid_bases_free(b);
b->state = -1;
Expand All @@ -235,9 +235,9 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
!strcmp(name, "bad_footer_digest")) {
n = b->n_chk;
src = b->chk;
} else if (!strcmp(name, "ghi_forged_offset")) {
n = b->n_ghi;
src = b->ghi;
} else if (!strcmp(name, "pak_forged_offset")) {
n = b->n_pak;
src = b->pak;
} else if (!strcmp(name, "sek_forged_entry") || !strcmp(name, "sek_table_no_flag") ||
!strcmp(name, "sek_reserved_set") || !strcmp(name, "sek_flags_set")) {
n = b->n_seek;
Expand Down Expand Up @@ -310,21 +310,21 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
/* --- Payload defects (nothing covers the sub-header without checksums) */
} else if (!strcmp(name, "bad_enc_lit")) {
d[PAY0 + 8] = 9; /* enc_lit outside {0,1,2,3} */
} else if (!strcmp(name, "glo_forged_enc_off")) {
} else if (!strcmp(name, "spl_forged_enc_off")) {
d[PAY0 + 11] = 7; /* enc_off outside {0,1} */
} else if (!strcmp(name, "glo_insufficient_slack")) {
} else if (!strcmp(name, "spl_insufficient_slack")) {
/* Leave fewer than ZXC_BLOCK_LIT_SLACK bytes behind the literal section
* by claiming more literals than the payload can carry behind them. */
const uint32_t comp = zxc_le32(d + BLK0 + 3);
const uint32_t want = comp - ZXC_GLO_HEADER_BINARY_SIZE - (ZXC_BLOCK_LIT_SLACK - 1);
const uint32_t want = comp - ZXC_SPL_HEADER_BINARY_SIZE - (ZXC_BLOCK_LIT_SLACK - 1);
zxc_store_le32(d + PAY0 + 4, want);
} else if (!strcmp(name, "ghi_forged_offset")) {
} else if (!strcmp(name, "pak_forged_offset")) {
/* Force the 16-bit offset far behind the output start. seq0 is derived
* from the wire, so guard the shape: a patch landing in the extras
* padding would ship a vector that decodes cleanly. */
const size_t seq0 = PAY0 + ZXC_GHI_HEADER_BINARY_SIZE + zxc_le32(d + PAY0 + 4);
if (d[BLK0] != ZXC_BLOCK_GHI || zxc_le32(d + PAY0) == 0 || seq0 + 4 > len) {
fprintf(stderr, " cannot forge a GHI offset (type %u, n_seq %u, seq0 %zu of %zu)\n",
const size_t seq0 = PAY0 + ZXC_PAK_HEADER_BINARY_SIZE + zxc_le32(d + PAY0 + 4);
if (d[BLK0] != ZXC_BLOCK_PAK || zxc_le32(d + PAY0) == 0 || seq0 + 4 > len) {
fprintf(stderr, " cannot forge a PAK offset (type %u, n_seq %u, seq0 %zu of %zu)\n",
d[BLK0], zxc_le32(d + PAY0), seq0, len);
ok = 0;
} else {
Expand Down Expand Up @@ -380,11 +380,11 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
}
} else if (!strcmp(name, "corrupt_payload")) {
/* A raw literal, past tok_comp when present (Sec 5.2): only the checksum catches it. */
const size_t lit = PAY0 + ZXC_GLO_HEADER_BINARY_SIZE +
const size_t lit = PAY0 + ZXC_SPL_HEADER_BINARY_SIZE +
(d[PAY0 + 9] == ZXC_SECTION_ENCODING_HUFFMAN ? 4U : 0U);
if (d[BLK0] != ZXC_BLOCK_GLO || d[PAY0 + 8] != ZXC_SECTION_ENCODING_RAW ||
if (d[BLK0] != ZXC_BLOCK_SPL || d[PAY0 + 8] != ZXC_SECTION_ENCODING_RAW ||
zxc_le32(d + PAY0 + 4) <= 4 || lit + 4 >= len) {
fprintf(stderr, " block 0 is not GLO with raw literals\n");
fprintf(stderr, " block 0 is not SPL with raw literals\n");
ok = 0;
} else {
d[lit + 4] ^= 0xFFU; /* a raw literal byte */
Expand All @@ -410,22 +410,22 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
} else {
d[len - sizes - ZXC_FILE_DIGEST_SIZE] ^= 0xFFU; /* checksummed base */
}
} else if (!strcmp(name, "glo_forged_offset")) {
/* GHI has its own vector. The first sequence has only its literal run
} else if (!strcmp(name, "spl_forged_offset")) {
/* PAK has its own vector. The first sequence has only its literal run
* behind it, so any large offset reaches before the output start. */
glo_layout_t L;
if (!glo_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section GLO - cannot forge an offset\n");
spl_layout_t L;
if (!spl_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section SPL - cannot forge an offset\n");
ok = 0;
} else {
d[L.off] = 0xFFU;
}
} else if (!strcmp(name, "glo_output_overflow")) {
} else if (!strcmp(name, "spl_output_overflow")) {
/* Saturated LL reads a varint from extras; this legal 3-byte one
* (Sec 6's own example, 100003) asks for more than the chunk holds. */
glo_layout_t L;
if (!glo_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section GLO - cannot forge extras\n");
spl_layout_t L;
if (!spl_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section SPL - cannot forge extras\n");
ok = 0;
} else {
d[L.extras] = 0xC3;
Expand All @@ -436,9 +436,9 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
/* A first byte >= 0xE0 announces four bytes, out of spec per Sec 6.
* Patch the ML varint: LL and the offset stay valid, so the read gets
* this far instead of failing earlier. */
glo_layout_t L;
if (!glo_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section GLO - cannot forge extras\n");
spl_layout_t L;
if (!spl_layout(d, len, &L)) {
fprintf(stderr, " block 0 is not a RAW-section SPL - cannot forge extras\n");
ok = 0;
} else {
const size_t ml = L.extras + varint_len(d[L.extras]);
Expand All @@ -464,7 +464,7 @@ static int build_invalid(invalid_bases_t* b, const char* name, uint8_t** out, si
static void invalid_bases_free(invalid_bases_t* b) {
free(b->plain);
free(b->chk);
free(b->ghi);
free(b->pak);
free(b->seek);
memset(b, 0, sizeof *b);
}
Expand Down
22 changes: 11 additions & 11 deletions conformance/v9/vectors.sha256
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ c8834372350dfbe361b1456103c9126ced155082a1a91150c520404219f4808a conformance/v9
5c414eb968d80bdabbd12ab4eaeb60551338bbfb22476f0fc6c560ac53034a7a conformance/v9/invalid/dict_required.zxc
116b1287c6c052bc74601d5b85ffff7a1101ff6dbab121522f1e01a9c9ef0586 conformance/v9/invalid/eof_flags_set.zxc
84744bb95b908f11908dd42b0ad63f2810e9073eb051b07e76abd1d6e2871e68 conformance/v9/invalid/eof_reserved_set.zxc
96fab6be8b8f0835b5eb41451bd08369ea73c11d49fa680d268a9984e1fbd8d3 conformance/v9/invalid/ghi_forged_offset.zxc
d09856f3bc0e794fd6844c2b0972ed898aba9c9e18b1909f46b750be927b628a conformance/v9/invalid/glo_forged_enc_off.zxc
17507a25ce12fa963ed5eebe1190880b524da6b65b5fb9196388218c9c7a95de conformance/v9/invalid/glo_forged_offset.zxc
7b19a262cdceb548027f69f2e93a2462597a39a6cd9bdd683f8e5c55c00edb3c conformance/v9/invalid/glo_insufficient_slack.zxc
4ca62090cd8c42977745975b4ab88a29a8145d3e600e49ceb45beebea83bd1e3 conformance/v9/invalid/glo_output_overflow.zxc
96fab6be8b8f0835b5eb41451bd08369ea73c11d49fa680d268a9984e1fbd8d3 conformance/v9/invalid/pak_forged_offset.zxc
d09856f3bc0e794fd6844c2b0972ed898aba9c9e18b1909f46b750be927b628a conformance/v9/invalid/spl_forged_enc_off.zxc
17507a25ce12fa963ed5eebe1190880b524da6b65b5fb9196388218c9c7a95de conformance/v9/invalid/spl_forged_offset.zxc
7b19a262cdceb548027f69f2e93a2462597a39a6cd9bdd683f8e5c55c00edb3c conformance/v9/invalid/spl_insufficient_slack.zxc
4ca62090cd8c42977745975b4ab88a29a8145d3e600e49ceb45beebea83bd1e3 conformance/v9/invalid/spl_output_overflow.zxc
edfe109dbdb8c4d65c99b600719c44278ff71dc2e05b7b2e4f84bece527deca7 conformance/v9/invalid/magic_then_zeros.zxc
f1c1086e945378b2b573e6681687b7f1393d6144918e8ff7d92dc35434f58d31 conformance/v9/invalid/sek_flag_no_table.zxc
12e3f8063a71baa1c599dbe11bbcebcfd1ba4b38902e499fdbcacd955ad1d49a conformance/v9/invalid/sek_flags_set.zxc
Expand All @@ -46,12 +46,12 @@ caf29ade7741a7e6e7969c558b1263f0acb107fb88aa2298bbdeb62f33cdc447 conformance/v9
ecff3040b396cfe41c3baeb3e33a3d176ea53fdd0098d3df99b848599b1fa0a6 conformance/v9/valid/dict_text.zxd
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 conformance/v9/valid/empty.expected
38e20a5f08bba007ed6d0bb0db7c30d018c29f3503e3d81da269a9cab1d9278f conformance/v9/valid/empty.zxc
f957032932bb903c799933b9461b5d3206d3672276dbfbb79a5f9754310f1d8f conformance/v9/valid/glo_pivco_wide_l7.expected
a81ced363bc92acaa689cdd3ca277598ed384d99a6d84829aed377792cf909b4 conformance/v9/valid/glo_pivco_wide_l7.zxc
c252fc1436210917dbe39067fd7e1837b2072de5fd07610d86e79e62a7c323cc conformance/v9/valid/glo_rle_4k.expected
4c5acd56de0dfea537b2148d3bddd4e408d34bb964a29fe10a4f9e16e2acb0de conformance/v9/valid/glo_rle_4k.zxc
aaa848621d3abe0f8a5bca8a7dac22a39811625b600656a2cd578fc8bbe53870 conformance/v9/valid/glo_tok_huffman_l7.expected
207cba0fa935c7c42514881f0fb266f2062e5ded994f86af673cc426de2ba165 conformance/v9/valid/glo_tok_huffman_l7.zxc
f957032932bb903c799933b9461b5d3206d3672276dbfbb79a5f9754310f1d8f conformance/v9/valid/spl_pivco_wide_l7.expected
a81ced363bc92acaa689cdd3ca277598ed384d99a6d84829aed377792cf909b4 conformance/v9/valid/spl_pivco_wide_l7.zxc
c252fc1436210917dbe39067fd7e1837b2072de5fd07610d86e79e62a7c323cc conformance/v9/valid/spl_rle_4k.expected
4c5acd56de0dfea537b2148d3bddd4e408d34bb964a29fe10a4f9e16e2acb0de conformance/v9/valid/spl_rle_4k.zxc
aaa848621d3abe0f8a5bca8a7dac22a39811625b600656a2cd578fc8bbe53870 conformance/v9/valid/spl_tok_huffman_l7.expected
207cba0fa935c7c42514881f0fb266f2062e5ded994f86af673cc426de2ba165 conformance/v9/valid/spl_tok_huffman_l7.zxc
bfd64e444fc3da30fdc9c9e0d27ade38cb87f2efbe7b2f9067039ca79e681566 conformance/v9/valid/max_offset_128k.expected
7182e93c351d9079115b584ad0707d7d3521cccbda1a3e60b830063b6bee654b conformance/v9/valid/max_offset_128k.zxc
7daca2095d0438260fa849183dfc67faa459fdf4936e1bc91eec6b281b27e4c2 conformance/v9/valid/multiblock_checksum.expected
Expand Down
8 changes: 4 additions & 4 deletions conformance/valid_cases.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,14 @@ static const valid_case_t VALID_CASES[] = {
{"all_zeros_4k", NULL, {.level = 3, .block_size = VC_KB(512)}},
{"max_offset_128k", NULL, {.level = 3, .block_size = VC_KB(512)}},

/* Block types and literal encodings: the only GHI, the only RLE literals
/* Block types and literal encodings: the only PAK, the only RLE literals
* (enc_lit = 1), the only PivCo literals (enc_lit = 2). */
{"text_64k_level1", NULL, {.level = 1, .block_size = VC_KB(512)}},
{"glo_rle_4k", NULL, {.level = 3, .block_size = VC_KB(512)}},
{"glo_pivco_wide_l7", NULL, {.level = 7, .block_size = VC_KB(512)}},
{"spl_rle_4k", NULL, {.level = 3, .block_size = VC_KB(512)}},
{"spl_pivco_wide_l7", NULL, {.level = 7, .block_size = VC_KB(512)}},

/* Token section: the only enc_tok = 2. */
{"glo_tok_huffman_l7", NULL, {.level = 7, .block_size = VC_KB(512)}},
{"spl_tok_huffman_l7", NULL, {.level = 7, .block_size = VC_KB(512)}},

/* Checksums on their own, so a failure here is not confounded with the
* dictionary vectors below (the only other place they are exercised).
Expand Down
2 changes: 1 addition & 1 deletion docs/API.md
Original file line number Diff line number Diff line change
Expand Up @@ -727,7 +727,7 @@ for integrations whose output buffer cannot be oversized (page-aligned
decoding into mapped pages, fixed-size slots in a columnar layout, etc.).

Output is **bit-identical** to `zxc_decompress_block()`. RAW blocks
transparently forward to the fast path; only GLO/GHI blocks use the
transparently forward to the fast path; only SPL/PAK blocks use the
strict-tail decoder, which is slightly slower than the wild-copy fast path
(see the performance table in `EXAMPLES.md`).

Expand Down
Loading
Loading