Skip to content

Commit 653ec51

Browse files
committed
Widen the tile-buffer N step for tensor ops so the engine no longer rebases B every 15 steps
1 parent 0793d39 commit 653ec51

8 files changed

Lines changed: 55 additions & 58 deletions

File tree

‎docs/designs/tensor_memory.md‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -193,16 +193,16 @@ Cycles: rtlsim, SimX in parentheses, with the configs of
193193

194194
| | WGMMA only | TMEM | Difference |
195195
|---|---|---|---|
196-
| LUT | 100,302 | 110,914 | +10,612 (+10.6%) |
197-
| FF | 60,960 | 63,351 | +2,391 |
196+
| LUT | 100,302 | 110,959 | +10,657 (+10.6%) |
197+
| FF | 60,960 | 63,355 | +2,395 |
198198
| Block RAM | 0 | 32 | the TMEM storage |
199199

200200
| Where the LUTs go | LUT |
201201
|---|---|
202-
| `VX_tcu_tmem` | 7,569 |
203-
| A buffers: the A-from-TMEM fill (4 blocks) | 1,936 |
204-
| B buffer: dense K-major prefetch and row merging | 565 |
205-
| Tile-buffer LMEM port: the `tc.cp` master | 108 |
202+
| `VX_tcu_tmem` | 7,516 |
203+
| A buffers: the A-from-TMEM fill (4 blocks) | 1,960 |
204+
| B buffer: dense K-major prefetch, row merging, wide N step | 628 |
205+
| Tile-buffer LMEM port: the `tc.cp` master | 119 |
206206
| Blocks: engine steps beside warp ops | 988 |
207207
| Warp WGMMA logic left out (§7) | −553 |
208208

@@ -217,7 +217,7 @@ Cycles: rtlsim, SimX in parentheses, with the configs of
217217
| Allocator | 341 |
218218
| Row-hazard check | 176 |
219219
| Op queue | ~100 |
220-
| Engine walk, step register, write-back queues, port control | ~1,600 |
220+
| Engine walk, step register, write-back queues, port control | ~1,550 |
221221

222222
**Builds without TMEM** match the base: every non-TMEM perf gate keeps its
223223
cycle count and flip-flops are identical. Beacon LUTs are equal for WMMA
@@ -228,10 +228,10 @@ buffer's TMEM-gated code maps slightly differently.
228228

229229
| Kernel | Cycles |
230230
|---|---|
231-
| GEMM 128×128×128 | 46,744 (48,153) |
232-
| GEMM 128×128×512 | 183,812 (185,772) |
231+
| GEMM 128×128×128 | 46,724 (48,132) |
232+
| GEMM 128×128×512 | 183,529 (185,767) |
233233
| Sparse GEMM 64×64×64, metadata by `tc.cp` | 8,191 (8,631) |
234-
| FlashAttention-4, 4 tiles (`-l64 -n32 -d64 -b4`) | 59,434 (59,314) |
234+
| FlashAttention-4, 4 tiles (`-l64 -n32 -d64 -b4`) | 59,649 (59,205) |
235235

236236
## 9. Not covered
237237

‎hw/rtl/tcu/VX_tcu_abuf.sv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
6464
input wire req_valid,
6565
input wire req_setup,
6666
input wire [3:0] req_step_m,
67-
input wire [3:0] req_step_n,
67+
input wire [TCU_TBUF_N_W-1:0] req_step_n,
6868
input wire [3:0] req_step_k,
6969
input wire [`VX_CFG_XLEN-1:0] req_desc_a,
7070
input wire req_a_is_smem,

‎hw/rtl/tcu/VX_tcu_bbuf.sv‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
5959
input wire req_is_sparse,
6060
input wire [3:0] req_step_m,
6161
input wire [3:0] req_step_k,
62-
input wire [3:0] req_step_n,
62+
input wire [TCU_TBUF_N_W-1:0] req_step_n,
6363
input wire [1:0] req_cd_nregs,
6464
input wire [NCTA_WIDTH-1:0] req_cta_id,
6565
input wire [`VX_CFG_XLEN-1:0] req_desc_b,
@@ -116,8 +116,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
116116
localparam LG_B_BLOCK_WORDS = $clog2(B_BLOCK_WORDS);
117117
localparam KM_CTR_W = $clog2(TCU_TC_N + 1);
118118
// 32-bit-word offset width for K-major address arithmetic.
119-
// step_n × TCU_TC_N + j has ≤ 4+4 bits; × ldm_words adds LDM_W.
120-
localparam KM_OFF_W = LDM_W + 4 + 4;
119+
// step_n × TCU_TC_N + j has ≤ N_W+4 bits; × ldm_words adds LDM_W.
120+
localparam KM_OFF_W = LDM_W + TCU_TBUF_N_W + 4;
121121
// Tensor ops reuse slots across descriptors and prefetch the next N block.
122122
localparam bit TMEM_EN = (`VX_CFG_TCU_TMEM_ENABLED != 0);
123123

@@ -126,14 +126,23 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
126126
// K_STEPS=2 always; N_STEPS=4/8/16 for cd_nregs=0/1/2 (NRC=8/16/32).
127127
// -----------------------------------------------------------------------
128128

129-
logic [4:0] block_index;
129+
// A tensor op is one K step (step_k = 0) over up to 2^N_W N blocks, so
130+
// its block index is step_n itself.
131+
localparam BI_W = 1 + TCU_TBUF_N_W;
132+
logic [BI_W-1:0] block_index;
130133
always_comb begin
131-
case (req_cd_nregs)
132-
2'd0: block_index = {2'b0, req_step_k[0], req_step_n[1:0]}; // N_STEPS=4
133-
2'd1: block_index = {1'b0, req_step_k[0], req_step_n[2:0]}; // N_STEPS=8
134-
default: block_index = {req_step_k[0], req_step_n[3:0]}; // N_STEPS=16
135-
endcase
134+
if (TMEM_EN) begin
135+
block_index = BI_W'(req_step_n);
136+
end else begin
137+
case (req_cd_nregs)
138+
2'd0: block_index = BI_W'({req_step_k[0], req_step_n[1:0]}); // N_STEPS=4
139+
2'd1: block_index = BI_W'({req_step_k[0], req_step_n[2:0]}); // N_STEPS=8
140+
default: block_index = BI_W'({req_step_k[0], req_step_n[3:0]}); // N_STEPS=16
141+
endcase
142+
end
136143
end
144+
`RUNTIME_ASSERT (~(TMEM_EN && req_valid && (req_step_k != 4'd0)),
145+
("%s tensor-op tile-buffer request with step_k %0d", INSTANCE_ID, req_step_k))
137146
if (4 > 1) begin : g_step_k_upper_unused
138147
`UNUSED_VAR (req_step_k[3:1])
139148
end
@@ -147,19 +156,17 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
147156
// Sparse blocks use the flat producer layout and can span one or two
148157
// logical 32-bit bank rows.
149158
localparam TOTAL_SHIFT = LG_B_SUB_BLOCKS + LG_XLEN_RATIO;
150-
wire [4:0] dense_offset = (TOTAL_SHIFT == 0)
151-
? block_index
152-
: 5'(block_index >> TOTAL_SHIFT);
159+
wire [BANK_ADDR_WIDTH-1:0] dense_offset = BANK_ADDR_WIDTH'(block_index >> TOTAL_SHIFT);
153160

154161
// Dense within-physical-bank-row selector (XLEN>32 only). Picks which
155162
// of the XLEN_RATIO logical 32-bit bank-rows to copy into slot A.
156163
localparam SUB_HALF_W = (LG_XLEN_RATIO == 0) ? 1 : LG_XLEN_RATIO;
157164
wire [SUB_HALF_W-1:0] dense_sub_half =
158165
(LG_XLEN_RATIO == 0)
159166
? '0
160-
: SUB_HALF_W'(({27'b0, block_index} >> LG_B_SUB_BLOCKS) & ((1 << LG_XLEN_RATIO) - 1));
167+
: SUB_HALF_W'((32'(block_index) >> LG_B_SUB_BLOCKS) & ((1 << LG_XLEN_RATIO) - 1));
161168

162-
localparam SP_LOGROW_W = 6;
169+
localparam SP_LOGROW_W = BI_W + 1;
163170
wire [SP_LOGROW_W-1:0] sp_logrow_a =
164171
SP_LOGROW_W'(block_index) * SP_LOGROW_W'(LOGICAL_ROWS_PER_BLK);
165172
wire [SP_LOGROW_W-1:0] sp_logrow_b = sp_logrow_a + SP_LOGROW_W'(1);
@@ -214,7 +221,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
214221
logic slot_row_major_r;
215222
logic [LDM_W-1:0] slot_ldm_words_r;
216223
logic [3:0] slot_step_k_r;
217-
logic [3:0] slot_step_n_r;
224+
logic [TCU_TBUF_N_W-1:0] slot_step_n_r;
218225
logic [NCTA_WIDTH-1:0] slot_cta_id_r;
219226
// K-major descriptors can point to storage rewritten between WGMMA instructions.
220227
logic refetched_for_first_uop_r;
@@ -231,7 +238,7 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
231238

232239
// Per-mode fetch addresses.
233240
wire [BANK_ADDR_WIDTH-1:0] fetch_addr_dense =
234-
desc_b_row_base + BANK_ADDR_WIDTH'(dense_offset);
241+
desc_b_row_base + dense_offset;
235242
wire [BANK_ADDR_WIDTH-1:0] fetch_addr_a_sparse =
236243
desc_b_row_base + BANK_ADDR_WIDTH'(sp_logrow_a >> LG_XLEN_RATIO);
237244
wire [BANK_ADDR_WIDTH-1:0] fetch_addr_b_sparse =
@@ -279,7 +286,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
279286
// slot B (unused by dense B) while slot A fetches the next N block.
280287
wire req_dense_km = req_wants_kmajor && !req_is_sparse;
281288
logic w1_valid_r;
282-
logic [3:0] w1_step_k_r, w1_step_n_r;
289+
logic [3:0] w1_step_k_r;
290+
logic [TCU_TBUF_N_W-1:0] w1_step_n_r;
283291
logic [NCTA_WIDTH-1:0] w1_cta_id_r;
284292
logic [BANK_ADDR_WIDTH-1:0] w1_row_base_r;
285293
logic [BANK_ROW_WORDS_LOG2-1:0] w1_inrow_r;
@@ -299,8 +307,8 @@ module VX_tcu_bbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
299307
wire need_fetch = req_valid && !bank_row_resident;
300308
wire alloc_en = need_fetch && !slot_fetching_r;
301309
wire pf_en = TMEM_EN && req_valid && req_dense_km && req_pf_ok && bank_row_resident_kmajor
302-
&& !slot_fetching_r && !req_setup && (req_step_n != 4'd15);
303-
wire [3:0] fetch_step_n = pf_en ? (req_step_n + 4'd1) : req_step_n;
310+
&& !slot_fetching_r && !req_setup && (req_step_n != '1);
311+
wire [TCU_TBUF_N_W-1:0] fetch_step_n = pf_en ? (req_step_n + TCU_TBUF_N_W'(1)) : req_step_n;
304312
wire rd_w1 = bank_row_resident_w1 && !bank_row_resident_kmajor;
305313

306314
assign bbuf_ready = !req_valid || bank_row_resident;

‎hw/rtl/tcu/VX_tcu_pkg.sv‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,10 @@ package VX_tcu_pkg;
468468

469469
`DECL_EXECUTE_T (tcu, `VX_CFG_NUM_TCU_LANES);
470470

471+
// N step width of a tile-buffer request: a tensor op walks up to 128
472+
// fragments; a WGMMA at most 16 N steps.
473+
localparam TCU_TBUF_N_W = `VX_CFG_TCU_TMEM_ENABLED ? 8 : 4;
474+
471475
// tcu_tbuf_req_t — per-block uop request packed for VX_tcu_tbuf.
472476
// `valid` is pre-gated to WGMMA and masked by `cta_conflict`.
473477
typedef struct packed {
@@ -476,7 +480,7 @@ package VX_tcu_pkg;
476480
logic [NW_WIDTH-1:0] wid;
477481
logic [NCTA_WIDTH-1:0] cta_id;
478482
logic [3:0] step_m;
479-
logic [3:0] step_n;
483+
logic [TCU_TBUF_N_W-1:0] step_n;
480484
logic [3:0] step_k;
481485
logic [1:0] cd_nregs;
482486
logic [`VX_CFG_XLEN-1:0] desc_a;

‎hw/rtl/tcu/VX_tcu_tbuf.sv‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
162162
wire bbuf_req_is_sparse;
163163
wire [3:0] bbuf_req_step_m;
164164
wire [3:0] bbuf_req_step_k;
165-
wire [3:0] bbuf_req_step_n;
165+
wire [TCU_TBUF_N_W-1:0] bbuf_req_step_n;
166166
wire [1:0] bbuf_req_cd_nregs;
167167
wire [NCTA_WIDTH-1:0] bbuf_req_cta_id;
168168
wire [`VX_CFG_XLEN-1:0] bbuf_req_desc_b;
@@ -211,7 +211,7 @@ module VX_tcu_tbuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
211211
logic sel_is_sparse;
212212
logic [3:0] sel_step_m;
213213
logic [3:0] sel_step_k;
214-
logic [3:0] sel_step_n;
214+
logic [TCU_TBUF_N_W-1:0] sel_step_n;
215215
logic [1:0] sel_cd_nregs;
216216
logic [NCTA_WIDTH-1:0] sel_cta_id;
217217
logic [`VX_CFG_XLEN-1:0] sel_desc_b;

‎hw/rtl/tcu/VX_tcu_tmem.sv‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,6 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
119119
localparam WQ_DEPTH = 16; // steps in flight
120120
localparam SCNTW = 16; // step counters
121121
localparam A_READS = TCU_WG_FEDP_K / TC_N;
122-
localparam KT_BYTES = TCU_WG_FEDP_K * 4;
123-
localparam B_BLK_BYTES = TC_N * KT_BYTES; // a block-major B (k, n) block
124122
localparam CP_ROW_WORDS = `VX_CFG_LMEM_NUM_BANKS * (`VX_CFG_XLEN / 32);
125123
localparam CP_FRAGS = CP_ROW_WORDS / NT; // fragment rows per LMEM bank row
126124
localparam CP_ADDRW = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(CP_ROW_WORDS * 4);
@@ -636,14 +634,12 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
636634
logic run_go_r; // the descriptors are set; prologue and steps follow
637635
logic run_m_r;
638636
logic [7:0] run_n_r;
639-
logic [3:0] run_sn_r;
640-
logic [15:0] run_choff_r;
641637
logic [3:0] pro_idx_r;
642638

643639
// step register E
644640
logic e_valid_r;
645641
logic e_m_r;
646-
logic [3:0] e_sn_r;
642+
logic [7:0] e_sn_r;
647643
logic e_first_r;
648644
logic e_pf_ok_r;
649645
logic e_zero_r;
@@ -924,7 +920,7 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
924920
jd.op_args.tcu.fmt_s = e_fmt_s_r;
925921
jd.op_args.tcu.fmt_d = (e_fmt_s_r >= 5'(TCU_I32_ID)) ? 5'(TCU_I32_ID) : 5'(TCU_FP32_ID);
926922
jd.op_args.tcu.step_m = 4'(e_m_r);
927-
jd.op_args.tcu.step_n = e_sn_r;
923+
jd.op_args.tcu.step_n = 4'(e_sn_r); // the core takes its low bits
928924
jd.op_args.tcu.is_first_uop = e_first_r;
929925
for (int t = 0; t < NT; ++t) begin
930926
jd.rs3_data[t] = `VX_CFG_XLEN'(e_c[b][t]);
@@ -936,7 +932,7 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
936932
tr.wid = e_wid_r;
937933
tr.cta_id = e_cta_r;
938934
tr.step_m = 4'(e_m_r);
939-
tr.step_n = e_sn_r;
935+
tr.step_n = TCU_TBUF_N_W'(e_sn_r);
940936
tr.cd_nregs = 2'd2;
941937
tr.desc_a = `VX_CFG_XLEN'(smem_desc(e_a_off_r[b], e_lda_r));
942938
tr.desc_b = `VX_CFG_XLEN'(smem_desc(e_b_off_r, e_ldb_r));
@@ -1165,7 +1161,7 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
11651161
if (load) begin
11661162
e_valid_r <= 1'b1;
11671163
e_m_r <= run_m_r;
1168-
e_sn_r <= run_sn_r;
1164+
e_sn_r <= run_n_r;
11691165
e_g_r <= load_g;
11701166
e_uuid_r <= rj.uuid;
11711167
e_wid_r <= rj.wid;
@@ -1178,30 +1174,22 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
11781174
for (int b = 0; b < BLOCK_SIZE; ++b) begin
11791175
e_a_off_r[b] <= r_a_off + 18'(32'(b) * XM * 32'(r_lda));
11801176
end
1181-
e_b_off_r <= r_b_off + 18'(run_choff_r);
1177+
e_b_off_r <= r_b_off;
11821178
e_first_r <= (run_m_r == 1'b0) && (run_n_r == 8'd0);
1183-
e_pf_ok_r <= (run_n_r != r_n_last) && (run_sn_r != 4'd15);
1179+
e_pf_ok_r <= (run_n_r != r_n_last);
11841180
e_zero_r <= r_acc_init;
11851181
end else if (job_fire) begin
11861182
e_valid_r <= 1'b0;
11871183
end
11881184

1189-
// walk m (inner), then n. Step n of the bbuf/abuf/FEDP is 4 bits and
1190-
// 0 marks the op's first step, so later steps cycle through 1..15
1191-
// with the remainder folded into the B descriptor.
1185+
// walk m (inner), then n
11921186
if (load) begin
11931187
if (~load_m_last) begin
11941188
run_m_r <= 1'b1;
11951189
end else begin
11961190
run_m_r <= 1'b0;
11971191
if (~load_last) begin
11981192
run_n_r <= run_n_r + 8'd1;
1199-
if (run_sn_r == 4'd15) begin
1200-
run_sn_r <= 4'd1;
1201-
run_choff_r <= run_choff_r + ((r_ldb != 0) ? (16'd15 * 16'(TC_N) * r_ldb) : 16'(15 * B_BLK_BYTES));
1202-
end else begin
1203-
run_sn_r <= run_sn_r + 4'd1;
1204-
end
12051193
end
12061194
end
12071195
end
@@ -1231,8 +1219,6 @@ module VX_tcu_tmem import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
12311219
run_op_r <= head;
12321220
run_m_r <= 1'b0;
12331221
run_n_r <= 8'd0;
1234-
run_sn_r <= 4'd0;
1235-
run_choff_r <= 16'd0;
12361222
pro_idx_r <= 4'd0;
12371223
cp_req_r <= '0;
12381224
cp_rsp_r <= '0;

‎hw/rtl/tcu/VX_tcu_wgmma.sv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #(
163163
assign req[bi].cta_id = exec_data[bi].header.cta_id;
164164
assign req[bi].step_m = exec_data[bi].op_args.tcu.step_m;
165165
assign req[bi].step_k = exec_data[bi].op_args.tcu.step_k;
166-
assign req[bi].step_n = exec_data[bi].op_args.tcu.step_n;
166+
assign req[bi].step_n = TCU_TBUF_N_W'(exec_data[bi].op_args.tcu.step_n);
167167
assign req[bi].cd_nregs = exec_data[bi].op_args.tcu.cd_nregs;
168168
wire use_live_desc = is_first_uop_b_w[bi]
169169
&& !needs_setup_b_w[bi];

‎sim/simx/tcu/tcu_unit.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1657,10 +1657,9 @@ class TcuUnit::Impl {
16571657
}
16581658

16591659
// Whether the step engine prefetches block n + 1: dense K-major B, within
1660-
// the op and the RTL's 16-block descriptor chunk (step_n 15 wraps).
1660+
// the op.
16611661
static bool job_b_prefetch(const tj_t& job) {
1662-
return !job.sparse && job.ldb != 0 && job.n + 1 < job.n_frag
1663-
&& !(job.n != 0 && job.n % 15 == 0);
1662+
return !job.sparse && job.ldb != 0 && job.n + 1 < job.n_frag;
16641663
}
16651664

16661665
// A dense K-major block's row reads stay in flight together.

0 commit comments

Comments
 (0)