Repository navigation
Expand file tree
/
Copy pathom_core.cpp
More file actions
699 lines (613 loc) · 26.1 KB
/
Copy pathom_core.cpp
File metadata and controls
699 lines (613 loc) · 26.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
// Copyright © 2019-2023
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "om_core.h"
#include <array>
#include <cstring>
#include <unordered_map>
#include <vector>
#include "gfx_ff_model.h"
#include "cluster.h"
#include "mem_block_pool.h"
#include "constants.h"
#include "debug.h"
using namespace vortex;
namespace {
// Ocache configuration constants (default config).
constexpr uint32_t kOcacheNumReqs = VX_CFG_OCACHE_NUM_BANKS;
constexpr uint32_t kOcacheMemPorts = 1;
constexpr uint32_t kOcacheLineSize = VX_CFG_MEM_BLOCK_SIZE;
constexpr uint64_t kOcacheLineMask = ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1);
// Inflight OmReq slot count.
constexpr uint32_t kInflight = VX_CFG_OM_MEM_QUEUE_SIZE;
// Cores per cluster.
constexpr uint32_t kCoresPerCluster = NUM_SOCKETS * VX_CFG_SOCKET_SIZE;
} // namespace
// ════════════════════════════════════════════════════════════════════
// OmCore::Impl
// ════════════════════════════════════════════════════════════════════
class OmCore::Impl {
public:
// Per-request lifecycle: mem_read → ds/blend compute → mem_write.
enum class State : uint8_t {
ADDR, // compute per-lane addresses + read/write enables
READ_ISSUE, // issue MemReq{READ} for needed (lane, port)
READ_GATHER, // wait for MemRsps; capture dst_color / dst_depthstencil
COMPUTE, // run DS test + blend; decide what to write
WRITE_ISSUE, // issue MemReq{WRITE} per lane port that needs it
DONE, // free slot
};
// Per-thread lane state.
struct LaneState {
bool active = false;
uint32_t pos_x = 0;
uint32_t pos_y = 0;
bool face = false;
uint32_t src_color = 0;
uint32_t src_depth = 0;
// Which records this fragment actually carries (the export mask). A record
// it does not carry has no source value, so it can be neither tested nor
// written.
bool has_color = false;
bool has_depth = false;
uint64_t zbuf_addr_byte = 0;
uint64_t cbuf_addr_byte = 0;
bool need_z_read = false;
bool need_c_read = false;
bool z_arrived = false;
bool c_arrived = false;
uint32_t dst_depthstencil = 0;
uint32_t dst_color = 0;
bool ds_pass = false;
uint32_t merged_depthstencil = 0;
uint32_t blended_color = 0;
bool need_z_write = false;
bool need_c_write = false;
uint32_t z_write_value = 0;
uint32_t c_write_value = 0;
bool z_write_issued = false;
bool c_write_issued = false;
bool z_read_issued = false;
bool c_read_issued = false;
// Write-completion acks (ocache write responses). A slot holds its
// same-pixel R-M-W interlock until its writes COMMIT, not merely issue, so a
// later same-pixel fragment's read can't bypass an in-flight write (the
// cross-draw depth-ordering hazard).
bool z_write_acked = false;
bool c_write_acked = false;
};
struct Slot {
bool in_use = false;
State state = State::ADDR;
OmReq req;
std::array<LaneState, VX_CFG_NUM_THREADS> lanes = {};
uint64_t issue_cycle = 0;
};
// Pending memory request bookkeeping. Each tag identifies a unique
// (slot, lane, port) triple so the response can be deposited in the right
// lane buffer. Port: 0 = zbuf (depth+stencil), 1 = cbuf (color).
struct PendingFill {
uint32_t slot;
uint8_t lane;
uint8_t port; // 0=zbuf, 1=cbuf
uint32_t byte_off; // offset within the 64-byte cache line
bool is_write = false; // write ack (commit) vs read fill
};
explicit Impl(OmCore* simobject)
: simobject_(simobject)
, slots_(kInflight)
, cycle_(0)
{}
void reset() {
cycle_ = 0;
// perf_stats_ survives: the MPM counters run from device reset.
for (auto& s : slots_) {
s.in_use = false;
s.state = State::ADDR;
for (auto& l : s.lanes) l = LaneState{};
}
pending_mem_.clear();
next_mem_tag_ = 0;
rr_req_ = 0;
}
int dcr_write(uint32_t addr, uint32_t value) {
dcrs_.write(addr, value);
depth_stencil_.configure(dcrs_);
for (uint32_t rt = 0; rt < VX_OM_MAX_RT; ++rt) {
blender_[rt].configure(dcrs_, rt);
}
recompute_state();
return 0;
}
const OmCore::PerfStats& perf_stats() const { return perf_stats_; }
void tick() {
++cycle_;
// 1) Drain ocache responses.
drain_mem_rsp();
// 2) Per-slot state advance.
for (auto& s : slots_) {
if (!s.in_use) continue;
switch (s.state) {
case State::ADDR: advance_addr(s); break;
case State::READ_ISSUE: advance_read_issue(s); break;
case State::READ_GATHER: advance_read_gather(s); break;
case State::COMPUTE: advance_compute(s); break;
case State::WRITE_ISSUE: advance_write_issue(s); break;
case State::DONE: free_slot(s); break;
}
}
// 3) Drain incoming OmReqs into free slots (round-robin per-core).
drain_req_in();
// perf
bool stalled = false;
for (auto& s : slots_) {
if (s.in_use && (s.state == State::READ_GATHER ||
s.state == State::READ_ISSUE ||
s.state == State::WRITE_ISSUE)) {
stalled = true;
break;
}
}
if (stalled) ++perf_stats_.stall_cycles;
perf_stats_.mem_latency += pending_mem_.size();
}
private:
// ── DCR-derived state (cached, recomputed on every dcr_write) ───────
void recompute_state() {
zbuf_baseaddr_ = uint64_t(dcrs_.read(VX_DCR_OM_ZBUF_ADDR)) << 6;
zbuf_pitch_ = dcrs_.read(VX_DCR_OM_ZBUF_PITCH);
depth_writemask_ = dcrs_.read(VX_DCR_OM_DEPTH_WRITEMASK) & 0x1;
stencil_front_writemask_ = dcrs_.read(VX_DCR_OM_STENCIL_WRITEMASK) & 0xffff;
stencil_back_writemask_ = dcrs_.read(VX_DCR_OM_STENCIL_WRITEMASK) >> 16;
for (uint32_t rt = 0; rt < VX_OM_MAX_RT; ++rt) {
cbuf_baseaddr_[rt] = uint64_t(dcrs_.read(rt, VX_DCR_OM_CBUF_ADDR)) << 6;
cbuf_pitch_[rt] = dcrs_.read(rt, VX_DCR_OM_CBUF_PITCH);
uint32_t cbuf_writemask = dcrs_.read(rt, VX_DCR_OM_CBUF_WRITEMASK) & 0xf;
cbuf_writemask_[rt] = (((cbuf_writemask >> 0) & 0x1) * 0x000000ff)
| (((cbuf_writemask >> 1) & 0x1) * 0x0000ff00)
| (((cbuf_writemask >> 2) & 0x1) * 0x00ff0000)
| (((cbuf_writemask >> 3) & 0x1) * 0xff000000);
color_read_[rt] = (cbuf_writemask != 0xf);
color_write_[rt] = (cbuf_writemask != 0x0);
}
}
// Same-pixel R-M-W interlock: a real ROP serialises fragments that touch the
// same (x,y) so the depth/colour read-modify-write of one fragment is visible
// to the next. OM runs up to kInflight slots concurrently, so two same-pixel
// fragments could otherwise both READ stale depth before either WRITES, and
// the last write would win by slot scheduling rather than submit order.
//
static uint32_t pixel_hash(uint32_t x, uint32_t y) {
return ((y & 0x7) << 3) | (x & 0x7);
}
// The compare is on the pixel's 8x8-tile hash bucket, not the exact
// coordinate: two pixels alias only across tiles and an alias costs a
// stall, never a wrong result.
// Returns true if `cand` covers a bucket an in-flight slot still owns.
bool collides_with_inflight(const OmReq& cand) const {
for (uint32_t s = 0; s < slots_.size(); ++s) {
if (!slots_[s].in_use) continue;
const OmReq& other = slots_[s].req;
for (uint32_t a = 0; a < VX_CFG_NUM_THREADS; ++a) {
if (!(cand.tmask_bits & (1u << a))) continue;
for (uint32_t b = 0; b < VX_CFG_NUM_THREADS; ++b) {
if (!(other.tmask_bits & (1u << b))) continue;
if (pixel_hash(cand.pos_x[a], cand.pos_y[a])
== pixel_hash(other.pos_x[b], other.pos_y[b])) {
return true;
}
}
}
}
return false;
}
// ── Stage: ACCEPT (drain per-core inputs into free slots) ───────────
// vx_om_export arrives as an aperture ADDRESS, not a position: recovering
// (x, y, face) needs the aperture DCRs, which are cluster state. This is the
// SimX counterpart of VX_om_ingress -- the core-side unit deliberately does not
// have the OM's DCRs, so the decode belongs here.
//
// The encoding is shift-only (the pitch is padded to a power of two), so this
// is bit-slicing, not division:
// offset = ((((rt << 1) | face) << ybits | y) << xbits | x) << record_shift
void decode_aperture(OmReq& req) const {
uint32_t xbits = dcrs_.read(VX_DCR_OM_APERTURE_XBITS);
uint32_t ybits = dcrs_.read(VX_DCR_OM_APERTURE_YBITS);
uint32_t shift = dcrs_.read(VX_DCR_OM_APERTURE_RECORD_SHIFT);
bool rt_seen = false;
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
if (!(req.tmask_bits & (uint64_t(1) << t))) continue;
uint64_t off = req.addr[t] - uint64_t(VX_MEM_OM_BASE_ADDR);
uint64_t rec = off >> shift;
req.pos_x[t] = uint32_t(rec & ((1ull << xbits) - 1));
req.pos_y[t] = uint32_t((rec >> xbits) & ((1ull << ybits) - 1));
req.face[t] = uint8_t((rec >> (xbits + ybits)) & 0x1);
// The index is above face, so a single-attachment export decodes to zero
// without its producer knowing the field exists. It names the attachment
// of the whole record, so every lane of one export must agree.
uint32_t rt = uint32_t((rec >> (xbits + ybits + 1)) & (VX_OM_MAX_RT - 1));
assert((!rt_seen || rt == req.rt) && "OM: one export spans two colour attachments");
req.rt = rt;
rt_seen = true;
}
__unused(rt_seen);
// Several attachments share one depth attachment, so a fragment covering
// them exports once per attachment and each export runs the depth/stencil
// test. The repeats only agree if the test writes nothing back.
assert((req.rt == 0
|| !(dcrs_.read(VX_DCR_OM_DEPTH_WRITEMASK) & 0x1))
&& "OM: colour attachment exported while the depth write is enabled");
assert((req.rt == 0 || dcrs_.read(VX_DCR_OM_STENCIL_WRITEMASK) == 0)
&& "OM: colour attachment exported while the stencil write is enabled");
// A one-word record holds colour or depth, never both. Which one is stated
// twice and by two different producers: the host writes it as a DCR, the
// shader encodes it in the export. This is the only place that holds both,
// and a disagreement writes the wrong buffer with no other symptom.
if (shift != 3) {
uint32_t depth_only = dcrs_.read(VX_DCR_OM_APERTURE_DEPTH_ONLY);
assert((req.export_mask & 0x3) == (depth_only ? 0x2u : 0x1u)
&& "OM: aperture record shape disagrees with the export mask");
__unused(depth_only);
}
req.from_aperture = false; // decoded; downstream sees an ordinary request
}
void drain_req_in() {
auto& chs = simobject_->om_req_in;
if (chs.empty()) return;
for (uint32_t i = 0; i < chs.size(); ++i) {
uint32_t cid = (rr_req_ + i) % chs.size();
auto& ch = chs.at(cid);
if (ch.empty()) continue;
OmReq req = ch.peek();
if (req.from_aperture) {
decode_aperture(req);
}
// Hold a same-pixel fragment until the in-flight owner retires (ROP
// ordering); other pixels on other channels still make progress.
// NOTE: this must run on the DECODED request -- it compares positions.
if (collides_with_inflight(req)) continue;
uint32_t free_slot = UINT32_MAX;
for (uint32_t s = 0; s < slots_.size(); ++s) {
if (!slots_[s].in_use) { free_slot = s; break; }
}
if (free_slot == UINT32_MAX) break; // no slot
auto& slot = slots_[free_slot];
slot.in_use = true;
slot.state = State::ADDR;
slot.req = req;
slot.issue_cycle = cycle_;
for (auto& l : slot.lanes) l = LaneState{};
ch.pop();
rr_req_ = (cid + 1) % chs.size();
DT(4, simobject_->name() << " accept: uuid=" << slot.req.uuid
<< ", slot=" << free_slot);
break;
}
}
// ── Stage: ADDR — compute per-lane addresses + read/write enables ───
void advance_addr(Slot& s) {
bool depth_enabled = depth_stencil_.depth_enabled();
bool blend_enabled = blender_[s.req.rt].enabled();
// funct7[1:0] = {has_depth, has_colour}. A shader may export colour only
// (the common case once early-Z owns both the depth test and the depth
// write), depth only (z-prepass), or both. A record the fragment does not
// carry cannot be tested or written: there is no source value for it.
bool has_color = (s.req.export_mask & 0x1) != 0;
bool has_depth = (s.req.export_mask & 0x2) != 0;
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
if (!(s.req.tmask_bits & (uint64_t(1) << t))) {
s.lanes[t].active = false;
continue;
}
LaneState& l = s.lanes[t];
l.active = true;
l.has_color = has_color;
l.has_depth = has_depth;
l.pos_x = s.req.pos_x[t];
l.pos_y = s.req.pos_y[t];
l.face = s.req.face[t] != 0;
l.src_color = s.req.color[t];
l.src_depth = s.req.depth[t];
l.zbuf_addr_byte = zbuf_baseaddr_ + uint64_t(l.pos_y) * zbuf_pitch_ + l.pos_x * 4;
l.cbuf_addr_byte = cbuf_baseaddr_[s.req.rt] + uint64_t(l.pos_y) * cbuf_pitch_[s.req.rt] + l.pos_x * 4;
bool stencil_enabled = depth_stencil_.stencil_enabled(l.face);
l.need_z_read = has_depth && (depth_enabled || stencil_enabled);
l.need_c_read = has_color && color_write_[s.req.rt] && (color_read_[s.req.rt] || blend_enabled);
}
s.state = State::READ_ISSUE;
}
// ── Stage: READ_ISSUE — issue MemReq{READ} for needed lane ports ────
void advance_read_issue(Slot& s) {
bool any_read_needed = false;
bool all_issued = true;
// One transaction per ocache request port per cycle. All issuance targets
// ocache_req_out.at(0) (a single req port), so the per-tick cap is 1. If OM
// ever gains multiple ocache request channels, fan out across the distinct
// channels — do NOT loop >1 request into one channel (that fabricates BW).
uint32_t budget = 1;
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS && budget > 0; ++t) {
LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_read && !l.z_read_issued) {
any_read_needed = true;
if (try_issue_read(s, t, /*port=*/0, l.zbuf_addr_byte)) {
l.z_read_issued = true;
if (--budget == 0) break;
} else {
all_issued = false;
break;
}
}
if (l.need_c_read && !l.c_read_issued) {
any_read_needed = true;
if (try_issue_read(s, t, /*port=*/1, l.cbuf_addr_byte)) {
l.c_read_issued = true;
if (--budget == 0) break;
} else {
all_issued = false;
break;
}
}
}
// Detect "is anything still un-issued?"
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_read && !l.z_read_issued) { all_issued = false; }
if (l.need_c_read && !l.c_read_issued) { all_issued = false; }
}
if (!any_read_needed) {
// No reads needed — color-write-only bypass path; jump to COMPUTE.
s.state = State::COMPUTE;
} else if (all_issued) {
s.state = State::READ_GATHER;
}
}
// ── Try to issue a read MemReq for (slot, lane, port). ──────────────
// Returns true if sent, false if backpressured.
bool try_issue_read(Slot& s, uint32_t lane, uint8_t port, uint64_t byte_addr) {
auto& req_ch = simobject_->ocache_req_out.at(0);
if (req_ch.full()) return false;
uint64_t cl_addr = byte_addr & kOcacheLineMask;
uint32_t off = uint32_t(byte_addr - cl_addr);
MemReq mreq;
mreq.addr = cl_addr;
mreq.op = MemOp::LD;
mreq.tag = next_mem_tag_++;
mreq.hart_id = s.req.tag;
mreq.uuid = s.req.uuid;
PendingFill pf;
pf.slot = (uint32_t)(&s - &slots_[0]);
pf.lane = uint8_t(lane);
pf.port = port;
pf.byte_off = off;
pending_mem_[mreq.tag] = pf;
req_ch.send(mreq);
++perf_stats_.mem_reads;
return true;
}
// ── Drain ocache responses → fill per-lane buffers ──────────────────
void drain_mem_rsp() {
// One response per port per cycle (each channel in ocache_rsp_in is a port).
for (auto& ch : simobject_->ocache_rsp_in) {
if (ch.empty()) continue;
auto& rsp = ch.peek();
auto it = pending_mem_.find(uint32_t(rsp.tag));
if (it == pending_mem_.end()) {
ch.pop();
continue;
}
const PendingFill pf = it->second;
pending_mem_.erase(it);
Slot& s = slots_[pf.slot];
LaneState& l = s.lanes[pf.lane];
if (pf.is_write) {
// Write ack (commit). Releases the slot's same-pixel interlock once all
// its writes land — see advance_write_issue.
if (pf.port == 0) l.z_write_acked = true;
else l.c_write_acked = true;
ch.pop();
continue;
}
uint32_t v = 0;
if (rsp.data) {
std::memcpy(&v, rsp.data->data() + pf.byte_off, 4);
}
if (pf.port == 0) {
l.dst_depthstencil = v;
l.z_arrived = true;
} else {
l.dst_color = v;
l.c_arrived = true;
}
ch.pop();
}
}
// ── Stage: READ_GATHER — wait for all needed responses ──────────────
void advance_read_gather(Slot& s) {
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
const LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_read && !l.z_arrived) return;
if (l.need_c_read && !l.c_arrived) return;
}
s.state = State::COMPUTE;
}
// ── Stage: COMPUTE — run DS test + blend, decide writes ─────────────
void advance_compute(Slot& s) {
bool depth_enabled = depth_stencil_.depth_enabled();
bool blend_enabled = blender_[s.req.rt].enabled();
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
LaneState& l = s.lanes[t];
if (!l.active) continue;
bool stencil_enabled = l.has_depth && depth_stencil_.stencil_enabled(l.face);
bool ds_active = l.has_depth && depth_enabled;
ds_active = ds_active || stencil_enabled;
uint32_t merged = 0;
l.ds_pass = !ds_active
|| depth_stencil_.test(l.face, l.src_depth, l.dst_depthstencil, &merged);
l.merged_depthstencil = merged;
l.blended_color = (blend_enabled && l.ds_pass)
? blender_[s.req.rt].blend(l.src_color, l.dst_color)
: l.src_color;
// Decide writes.
uint32_t stencil_writemask = l.face ? stencil_back_writemask_ : stencil_front_writemask_;
uint32_t ds_writemask =
((l.has_depth && depth_enabled && l.ds_pass && depth_writemask_) ? OM_DEPTH_MASK : 0u)
| (stencil_enabled ? (uint32_t(stencil_writemask) << VX_OM_DEPTH_BITS) : 0u);
l.need_z_write = (ds_writemask != 0);
if (l.need_z_write) {
l.z_write_value = (l.dst_depthstencil & ~ds_writemask)
| (l.merged_depthstencil & ds_writemask);
}
l.need_c_write = l.has_color && color_write_[s.req.rt] && l.ds_pass;
if (l.need_c_write) {
// If color_read_ is false (writemask == 0xf), dst_color is unread —
// we'll still write the full word; the merge is a no-op.
l.c_write_value = (l.dst_color & ~cbuf_writemask_[s.req.rt])
| (l.blended_color & cbuf_writemask_[s.req.rt]);
}
}
s.state = State::WRITE_ISSUE;
}
// ── Stage: WRITE_ISSUE — emit per-lane writes (rate-limited) ────────
void advance_write_issue(Slot& s) {
bool all_done = true;
// One write per ocache request port per cycle. If OM ever gains multiple
// ocache request channels, fan out across distinct channels — do NOT loop
// >1 request into one channel (that fabricates BW).
uint32_t budget = 1;
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS && budget > 0; ++t) {
LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_write && !l.z_write_issued) {
if (try_issue_write(s, t, /*port=*/0, l.zbuf_addr_byte, l.z_write_value)) {
l.z_write_issued = true;
if (--budget == 0) break;
} else {
all_done = false;
break;
}
}
if (l.need_c_write && !l.c_write_issued) {
if (try_issue_write(s, t, /*port=*/1, l.cbuf_addr_byte, l.c_write_value)) {
l.c_write_issued = true;
if (--budget == 0) break;
} else {
all_done = false;
break;
}
}
}
// Re-scan for un-issued writes.
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
const LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_write && !l.z_write_issued) all_done = false;
if (l.need_c_write && !l.c_write_issued) all_done = false;
}
if (!all_done)
return; // still issuing writes
// All writes issued — hold the slot (and its same-pixel interlock) until
// every write COMMITS (ocache write ack), so a later same-pixel read can't
// bypass an in-flight write. Reads bypass this wait (no writes pending).
for (uint32_t t = 0; t < VX_CFG_NUM_THREADS; ++t) {
const LaneState& l = s.lanes[t];
if (!l.active) continue;
if (l.need_z_write && !l.z_write_acked) return;
if (l.need_c_write && !l.c_write_acked) return;
}
s.state = State::DONE;
}
// ── Issue a 4-byte write at byte_addr. Software RMW already merged. ─
bool try_issue_write(Slot& s, uint32_t lane, uint8_t port,
uint64_t byte_addr, uint32_t value) {
auto& req_ch = simobject_->ocache_req_out.at(0);
if (req_ch.full()) return false;
uint64_t cl_addr = byte_addr & kOcacheLineMask;
uint32_t off = uint32_t(byte_addr - cl_addr);
MemReq mreq;
mreq.addr = cl_addr;
mreq.op = MemOp::ST;
mreq.tag = next_mem_tag_++;
mreq.hart_id = s.req.tag;
mreq.uuid = s.req.uuid;
mreq.byteen = uint64_t(0xfu) << off;
auto blk = make_mem_block();
std::memcpy(blk->data() + off, &value, 4);
mreq.data = blk;
// Track the write so its ocache commit ack releases the same-pixel interlock.
PendingFill pf;
pf.slot = (uint32_t)(&s - &slots_[0]);
pf.lane = uint8_t(lane);
pf.port = port;
pf.byte_off = off;
pf.is_write = true;
pending_mem_[mreq.tag] = pf;
req_ch.send(mreq);
++perf_stats_.mem_writes;
return true;
}
void free_slot(Slot& s) {
DT(4, simobject_->name() << " complete: uuid=" << s.req.uuid
<< ", latency=" << (cycle_ - s.issue_cycle));
s.in_use = false;
s.state = State::ADDR;
for (auto& l : s.lanes) l = LaneState{};
}
// ── Members ──────────────────────────────────────────────────────────
OmCore* simobject_;
OMDCRS dcrs_;
DepthTencil depth_stencil_;
// One blender per colour attachment; the depth/stencil test is shared.
Blender blender_[VX_OM_MAX_RT];
// DCR-derived cache (recomputed on every dcr_write).
uint64_t cbuf_baseaddr_[VX_OM_MAX_RT] = {};
uint32_t cbuf_pitch_[VX_OM_MAX_RT] = {};
uint32_t cbuf_writemask_[VX_OM_MAX_RT] = {};
bool color_read_[VX_OM_MAX_RT] = {};
bool color_write_[VX_OM_MAX_RT] = {};
uint64_t zbuf_baseaddr_ = 0;
uint32_t zbuf_pitch_ = 0;
bool depth_writemask_ = false;
uint32_t stencil_front_writemask_ = 0;
uint32_t stencil_back_writemask_ = 0;
std::vector<Slot> slots_;
std::unordered_map<uint32_t, PendingFill> pending_mem_;
uint32_t next_mem_tag_ = 0;
uint32_t rr_req_ = 0;
uint64_t cycle_;
OmCore::PerfStats perf_stats_;
};
// ════════════════════════════════════════════════════════════════════
// OmCore — wrappers
// ════════════════════════════════════════════════════════════════════
OmCore::OmCore(const SimContext& ctx, const char* name, Cluster* cluster)
: SimObject<OmCore>(ctx, name)
, om_req_in(kCoresPerCluster, this)
, ocache_req_out(kOcacheNumReqs, this)
, ocache_rsp_in(kOcacheNumReqs, this)
{
__unused(cluster);
impl_ = new Impl(this);
}
OmCore::~OmCore() {
delete impl_;
}
void OmCore::on_reset() { impl_->reset(); }
void OmCore::on_tick() { impl_->tick(); }
int OmCore::dcr_write(uint32_t addr, uint32_t value) {
return impl_->dcr_write(addr, value);
}
const OmCore::PerfStats& OmCore::perf_stats() const {
return impl_->perf_stats();
}