diff options
| author | Mukul Joshi <mukul.joshi@amd.com> | 2025-09-12 17:48:12 -0400 |
|---|---|---|
| committer | Alex Deucher <alexander.deucher@amd.com> | 2026-01-05 16:59:56 -0500 |
| commit | 258cc2b687bf67f7872707061158aefddbc449e1 (patch) | |
| tree | a4eda93e938439a05ac5c6da74ad0b89b21ad34b | |
| parent | 5a8c343d2e87a4d623934e73a0c3a200e2a47a42 (diff) | |
drm/amdkfd: Add back CWSR trap handler for GFX 12.1
CWSR Trap handler for GFX 12.1 was missed when merging changes
from 6.14 NPI branch to 6.16 NPI branch. This change adds back
the CWSR trap handler for GFX 12.1.
Signed-off-by: Mukul Joshi <mukul.joshi@amd.com>
Reviewed-by: Alex Sierra <alex.sierra@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h | 540 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm | 746 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 7 |
3 files changed, 997 insertions, 296 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index f98c735b2905..5c6d533a5956 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -4664,3 +4664,543 @@ static const uint32_t cwsr_trap_gfx9_5_0_hex[] = { 0xbf8a0000, 0xbe801f6c, 0xbf9b0000, 0x00000000, }; + +static const uint32_t cwsr_trap_gfx12_1_0_hex[] = { + 0xbfa00001, 0xbfa002a2, + 0xb0804009, 0xb8f8f804, + 0x9178ff78, 0x00008c00, + 0xb8fbf811, 0x8b6eff78, + 0x00004000, 0xbfa10008, + 0x8b6eff7b, 0x00000080, + 0xbfa20018, 0x8b6ea07b, + 0xbfa2004f, 0xbf830010, + 0xb8fbf811, 0xbfa0fffb, + 0x8b6eff7b, 0x00000bd0, + 0xbfa20010, 0xb8eef812, + 0x8b6f8f7b, 0xbfa10002, + 0x8c6eff6e, 0x00000080, + 0xb8eff813, 0x8b6e6e6f, + 0xbfa20008, 0x8b6eff6d, + 0xf0000000, 0xbfa20005, + 0x8b6fff6f, 0x00000200, + 0xbfa20002, 0x8b6ea07b, + 0xbfa20039, 0x9177ff77, + 0x007fc000, 0xb8fa04a1, + 0x847a967a, 0x8c777a77, + 0xb8fa0421, 0x847a957a, + 0x8c777a77, 0xb8fa3021, + 0x847a8e7a, 0x8c777a77, + 0xb980f821, 0x00000000, + 0xbefa4d82, 0xbf8a0000, + 0x84fa887a, 0xbf0d987b, + 0xbfa10002, 0x8c7bff7b, + 0xfe000000, 0xf4601bbd, + 0xf8000010, 0xbf8a0000, + 0x846e976e, 0x9177ff77, + 0x00800000, 0x8c776e77, + 0xf4603bbd, 0xf8000000, + 0xbf8a0000, 0xf4603ebd, + 0xf8000008, 0xbf8a0000, + 0x8bee6e6e, 0xbfa10001, + 0xbe80486e, 0x8b6eff6d, + 0xf0000000, 0xbfa20009, + 0xb8eef811, 0x8b6eff6e, + 0x00000080, 0xbfa20007, + 0x8c78ff78, 0x00004000, + 0x80ec886c, 0x82ed806d, + 0xbfa00002, 0x806c846c, + 0x826d806d, 0x8b6dff6d, + 0x01ffffff, 0x8bfe7e7e, + 0x8bea6a6a, 0x85788978, + 0xb9783244, 0xbe804a6c, + 0xb8faf802, 0xbf0d987a, + 0xbfa10001, 0xbfb00000, + 0x8b6dff6d, 0x01ffffff, + 0xbefa0080, 0xb97a0151, + 0xbeee007e, 0xbeef007f, + 0xbefe0180, 0xbefe4d84, + 0xbf8a0000, 0x8b7aff7f, + 0x04000000, 0x847a857a, + 0x8c6d7a6d, 0x9177ff77, + 0x007fc000, 0xb8fa04a1, + 0x847a967a, 0x8c777a77, + 0xb8fa0421, 0x847a957a, + 0x8c777a77, 0xb8fa3021, + 0x847a8e7a, 0x8c777a77, + 0xb980f821, 0x00000000, + 0xb8eff822, 0xb980f822, + 0x00000000, 0xb8fa2b01, + 0x847a997a, 0x8c6d7a6d, + 0xb9802b01, 0x00000000, + 0xbefa007e, 0x8b7bff7f, + 0x01ffffff, 0xbefe00c1, + 0xbeff00c1, 0xee0a407a, + 0x000c0000, 0x00000000, + 0x7e000280, 0xbefe007a, + 0xbeff007b, 0xb8fb0742, + 0x847b997b, 0xb8fa3b05, + 0x807a817a, 0xbf0d997b, + 0xbfa20002, 0x847a897a, + 0xbfa00001, 0x847a8a7a, + 0x8b7bff7f, 0x01ffffff, + 0x807aff7a, 0x000001c0, + 0x807a7e7a, 0x827b807b, + 0xd7610000, 0x00010870, + 0xd7610000, 0x00010a71, + 0xd7610000, 0x00010c72, + 0xd7610000, 0x00010e73, + 0xd7610000, 0x00011074, + 0xd7610000, 0x00011275, + 0xd7610000, 0x00011476, + 0xd7610000, 0x00011677, + 0xd7610000, 0x00011a79, + 0xd7610000, 0x00011c7e, + 0xd7610000, 0x00011e7f, + 0xbefe00ff, 0x00003fff, + 0xbeff0080, 0xee0a407a, + 0x000c0000, 0x00000000, + 0xd760007a, 0x00011d00, + 0xd760007b, 0x00011f00, + 0xbefe007a, 0xbeff007b, + 0xbef4007e, 0x8b75ff7f, + 0x01ffffff, 0xbef1007d, + 0xb8f30742, 0x84739973, + 0xbefe00c1, 0x857d9973, + 0x8b7d817d, 0xbf06817d, + 0xbfa20002, 0xbeff0080, + 0xbfa00002, 0xbeff00c1, + 0xbfa0000a, 0xee0a4074, + 0x008c0000, 0x00008000, + 0xee0a4074, 0x010c0000, + 0x00010000, 0xee0a4074, + 0x018c0000, 0x00018000, + 0xbfa00009, 0xee0a4074, + 0x008c0000, 0x00010000, + 0xee0a4074, 0x010c0000, + 0x00020000, 0xee0a4074, + 0x018c0000, 0x00030000, + 0xb8f03b05, 0x80708170, + 0xbf0d9973, 0xbfa20002, + 0x84708970, 0xbfa00001, + 0x84708a70, 0x8070ff70, + 0x00000200, 0x7e000280, + 0x7e020280, 0x7e040280, + 0xbefd0080, 0xb8faf802, + 0xbf0c8b7a, 0xbfa20003, + 0xbe804fc2, 0xbf94fffe, + 0xbfa10001, 0xbe804ec4, + 0xbf94fffc, 0xb8faf804, + 0x8b7aff7a, 0x0001000c, + 0x9178ff78, 0x0001000c, + 0x8c787a78, 0xd7610002, + 0x0000fa71, 0x807d817d, + 0xd7610002, 0x0000fa6c, + 0x807d817d, 0x917aff6d, + 0x80000000, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xd7610002, 0x0000fa6e, + 0x807d817d, 0xbefa0080, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xd7610002, + 0x0000fa78, 0x807d817d, + 0xb8faf811, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xd7610002, 0x0000fa6f, + 0x807d817d, 0xb8f1f801, + 0x937aff6d, 0x00060019, + 0x847a8c7a, 0x8c717a71, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f814, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f815, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f812, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f813, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8faf802, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xbefa50c1, + 0xbfc70000, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xbefa4c88, 0xbfc70000, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xbefe00ff, + 0x0000ffff, 0xbeff0080, + 0x80767074, 0x82778075, + 0xee0a4076, 0x010c0000, + 0x00000000, 0xbefe00c1, + 0x7e040280, 0xbefa5081, + 0xbfc70000, 0xd7610002, + 0x0001007a, 0xbefa5082, + 0xbfc70000, 0xd7610002, + 0x0001027a, 0xbefa5083, + 0xbfc70000, 0xd7610002, + 0x0001047a, 0xbefa5084, + 0xbfc70000, 0xd7610002, + 0x0001067a, 0xbefa5085, + 0xbfc70000, 0xd7610002, + 0x0001087a, 0xbefa5086, + 0xbfc70000, 0xd7610002, + 0x00010a7a, 0xbefa5087, + 0xbfc70000, 0xd7610002, + 0x00010c7a, 0xbefa5088, + 0xbfc70000, 0xd7610002, + 0x00010e7a, 0xbefa5089, + 0xbfc70000, 0xd7610002, + 0x0001107a, 0xbefa508a, + 0xbfc70000, 0xd7610002, + 0x0001127a, 0xbefa508b, + 0xbfc70000, 0xd7610002, + 0x0001147a, 0xbefa508c, + 0xbfc70000, 0xd7610002, + 0x0001167a, 0xbefa508d, + 0xbfc70000, 0xd7610002, + 0x0001187a, 0xbefa508e, + 0xbfc70000, 0xd7610002, + 0x00011a7a, 0xbefa508f, + 0xbfc70000, 0xd7610002, + 0x00011c7a, 0xbefa5090, + 0xbfc70000, 0xd7610002, + 0x00011e7a, 0xee0a4076, + 0x010c0000, 0x00008000, + 0xb8f03b05, 0x80708170, + 0xbf0d9973, 0xbfa20002, + 0x84708970, 0xbfa00001, + 0x84708a70, 0xbef90080, + 0xbefd0080, 0xbf800000, + 0xbe804100, 0xbe824102, + 0xbe844104, 0xbe864106, + 0xbe884108, 0xbe8a410a, + 0xbe8c410c, 0xbe8e410e, + 0xd7610002, 0x0000f200, + 0x80798179, 0xd7610002, + 0x0000f201, 0x80798179, + 0xd7610002, 0x0000f202, + 0x80798179, 0xd7610002, + 0x0000f203, 0x80798179, + 0xd7610002, 0x0000f204, + 0x80798179, 0xd7610002, + 0x0000f205, 0x80798179, + 0xd7610002, 0x0000f206, + 0x80798179, 0xd7610002, + 0x0000f207, 0x80798179, + 0xd7610002, 0x0000f208, + 0x80798179, 0xd7610002, + 0x0000f209, 0x80798179, + 0xd7610002, 0x0000f20a, + 0x80798179, 0xd7610002, + 0x0000f20b, 0x80798179, + 0xd7610002, 0x0000f20c, + 0x80798179, 0xd7610002, + 0x0000f20d, 0x80798179, + 0xd7610002, 0x0000f20e, + 0x80798179, 0xd7610002, + 0x0000f20f, 0x80798179, + 0xbf06a079, 0xbfa10009, + 0x80767074, 0x82778075, + 0xee0a4076, 0x010c0000, + 0x00000000, 0x8070ff70, + 0x00000080, 0xbef90080, + 0x7e040280, 0x807d907d, + 0xbf0aff7d, 0x00000060, + 0xbfa2ffb9, 0xbe804100, + 0xbe824102, 0xbe844104, + 0xbe864106, 0xbe884108, + 0xbe8a410a, 0xd7610002, + 0x0000f200, 0x80798179, + 0xd7610002, 0x0000f201, + 0x80798179, 0xd7610002, + 0x0000f202, 0x80798179, + 0xd7610002, 0x0000f203, + 0x80798179, 0xd7610002, + 0x0000f204, 0x80798179, + 0xd7610002, 0x0000f205, + 0x80798179, 0xd7610002, + 0x0000f206, 0x80798179, + 0xd7610002, 0x0000f207, + 0x80798179, 0xd7610002, + 0x0000f208, 0x80798179, + 0xd7610002, 0x0000f209, + 0x80798179, 0xd7610002, + 0x0000f20a, 0x80798179, + 0xd7610002, 0x0000f20b, + 0x80798179, 0xbefe00ff, + 0x0000ffff, 0x80767074, + 0x82778075, 0xee0a4076, + 0x010c0000, 0x00000000, + 0xbefe00c1, 0x857d9973, + 0x8b7d817d, 0xbf06817d, + 0xbfa20002, 0xbeff0080, + 0xbfa00001, 0xbeff00c1, + 0xb8fb4306, 0x8b7bc17b, + 0xbfa10042, 0x8b7aff6d, + 0x80000000, 0xbfa1003f, + 0x847b8a7b, 0xb8f03b05, + 0x80708170, 0xbf0d9973, + 0xbfa20002, 0x84708970, + 0xbfa00001, 0x84708a70, + 0x8070ff70, 0x00000200, + 0x8070ff70, 0x00000200, + 0xd71f0000, 0x000100c1, + 0xd7200000, 0x000200c1, + 0x16000084, 0x857d9973, + 0x8b7d817d, 0xbf06817d, + 0xbefd0080, 0xbfa20015, + 0xbe8300ff, 0x00000080, + 0xbf800000, 0xbf800000, + 0xbf800000, 0xd8d80000, + 0x01000000, 0xbf8a0000, + 0x80767074, 0x82778075, + 0xee0a4076, 0x008c0000, + 0x00000000, 0x807d037d, + 0x80700370, 0xd5250000, + 0x0001ff00, 0x00000080, + 0xbf0a7b7d, 0xbfa2fff1, + 0xbfa00014, 0xbe8300ff, + 0x00000100, 0xbf800000, + 0xbf800000, 0xbf800000, + 0xd8d80000, 0x01000000, + 0xbf8a0000, 0x80767074, + 0x82778075, 0xee0a4076, + 0x008c0000, 0x00000000, + 0x807d037d, 0x80700370, + 0xd5250000, 0x0001ff00, + 0x00000100, 0xbf0a7b7d, + 0xbfa2fff1, 0xbefe00c1, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20004, + 0xbef000ff, 0x00000200, + 0xbeff0080, 0xbfa00003, + 0xbef000ff, 0x00000400, + 0xbeff00c1, 0xb8fb3b05, + 0x807b817b, 0x847b827b, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa2001b, + 0xbefd0084, 0xbf0a7b7d, + 0xbfa10032, 0x7e008700, + 0x7e028701, 0x7e048702, + 0x7e068703, 0x80767074, + 0x82778075, 0xee0a4076, + 0x000c0000, 0x00000000, + 0xee0a4076, 0x008c0000, + 0x00008000, 0xee0a4076, + 0x010c0000, 0x00010000, + 0xee0a4076, 0x018c0000, + 0x00018000, 0x807d847d, + 0x8070ff70, 0x00000200, + 0xbf0a7b7d, 0xbfa2ffe9, + 0xbfa0001a, 0xbefd0084, + 0xbf0a7b7d, 0xbfa10017, + 0x7e008700, 0x7e028701, + 0x7e048702, 0x7e068703, + 0x80767074, 0x82778075, + 0xee0a4076, 0x000c0000, + 0x00000000, 0xee0a4076, + 0x008c0000, 0x00010000, + 0xee0a4076, 0x010c0000, + 0x00020000, 0xee0a4076, + 0x018c0000, 0x00030000, + 0x807d847d, 0x8070ff70, + 0x00000400, 0xbf0a7b7d, + 0xbfa2ffe9, 0xbfa00180, + 0xbef4007e, 0x8b75ff7f, + 0x01ffffff, 0xbef1007f, + 0xb8f20742, 0x84729972, + 0x8b6eff7f, 0x04000000, + 0xbfa10044, 0xbefe00c1, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00001, + 0xbeff00c1, 0xb8ef4306, + 0x8b6fc16f, 0xbfa10039, + 0x846f8a6f, 0xb8f83b05, + 0x80788178, 0xbf0d9972, + 0xbfa20002, 0x84788978, + 0xbfa00001, 0x84788a78, + 0x8078ff78, 0x00000200, + 0x8078ff78, 0x00000200, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbefd0080, + 0xd71f0001, 0x000100c1, + 0xd7200001, 0x000202c1, + 0x30020282, 0xbfa20012, + 0x80767874, 0x82778075, + 0xee0a0076, 0x000c0000, + 0x00000000, 0xbf8a0000, + 0xd8340000, 0x00000001, + 0xd5250001, 0x0001ff01, + 0x00000080, 0x807dff7d, + 0x00000080, 0x8078ff78, + 0x00000080, 0xbf0a6f7d, + 0xbfa2ffef, 0xbfa00011, + 0x80767874, 0x82778075, + 0xee0a0076, 0x000c0000, + 0x00000000, 0xbf8a0000, + 0xd8340000, 0x00000001, + 0xd5250001, 0x0001ff01, + 0x00000100, 0x807dff7d, + 0x00000100, 0x8078ff78, + 0x00000100, 0xbf0a6f7d, + 0xbfa2ffef, 0xbef80080, + 0xbefe00c1, 0x857d9972, + 0x8b7d817d, 0xbf06817d, + 0xbfa20002, 0xbeff0080, + 0xbfa00001, 0xbeff00c1, + 0xb8ef3b05, 0x806f816f, + 0x846f826f, 0x857d9972, + 0x8b7d817d, 0xbf06817d, + 0xbfa2002c, 0xbeee0078, + 0x8078ff78, 0x00000200, + 0xbefd0084, 0x80767874, + 0x82778075, 0xee0a0076, + 0x000c0000, 0x00000000, + 0xee0a0076, 0x000c0001, + 0x00008000, 0xee0a0076, + 0x000c0002, 0x00010000, + 0xee0a0076, 0x000c0003, + 0x00018000, 0xbf8a0000, + 0x7e008500, 0x7e028501, + 0x7e048502, 0x7e068503, + 0x807d847d, 0x8078ff78, + 0x00000200, 0xbf0a6f7d, + 0xbfa2ffe8, 0x80766e74, + 0x82778075, 0xee0a0076, + 0x000c0000, 0x00000000, + 0xee0a0076, 0x000c0001, + 0x00008000, 0xee0a0076, + 0x000c0002, 0x00010000, + 0xee0a0076, 0x000c0003, + 0x00018000, 0xbf8a0000, + 0xbfa0002d, 0xbeee0078, + 0x8078ff78, 0x00000400, + 0xbefd0084, 0xbf0a6f7d, + 0xbfa10018, 0x80767874, + 0x82778075, 0xee0a0076, + 0x000c0000, 0x00000000, + 0xee0a0076, 0x000c0001, + 0x00010000, 0xee0a0076, + 0x000c0002, 0x00020000, + 0xee0a0076, 0x000c0003, + 0x00030000, 0xbf8a0000, + 0x7e008500, 0x7e028501, + 0x7e048502, 0x7e068503, + 0x807d847d, 0x8078ff78, + 0x00000400, 0xbf0a6f7d, + 0xbfa2ffe8, 0x80766e74, + 0x82778075, 0xee0a0076, + 0x000c0000, 0x00000000, + 0xee0a0076, 0x000c0001, + 0x00010000, 0xee0a0076, + 0x000c0002, 0x00020000, + 0xee0a0076, 0x000c0003, + 0x00030000, 0xbf8a0000, + 0xb8f83b05, 0x80788178, + 0xbf0d9972, 0xbfa20002, + 0x84788978, 0xbfa00001, + 0x84788a78, 0x8078ff78, + 0x00000200, 0x80f8ff78, + 0x00000060, 0x80767874, + 0x82778075, 0xbefd00ff, + 0x0000006c, 0xf460403b, + 0xf8000000, 0xbf8a0000, + 0x80fd847d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0x80f6a076, 0x82f78077, + 0xf460603b, 0xf8000000, + 0xbf8a0000, 0x80fd887d, + 0xbf800000, 0xbe804300, + 0xbe824302, 0xbe844304, + 0xbe864306, 0x80f6c076, + 0x82f78077, 0xf460803b, + 0xf8000000, 0xbf8a0000, + 0x80fd907d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0xbe844304, 0xbe864306, + 0xbe884308, 0xbe8a430a, + 0xbe8c430c, 0xbe8e430e, + 0xbf06807d, 0xbfa1ffef, + 0xb980f801, 0x00000000, + 0xb8f83b05, 0x80788178, + 0xbf0d9972, 0xbfa20002, + 0x84788978, 0xbfa00001, + 0x84788a78, 0x8078ff78, + 0x00000200, 0x80767874, + 0x82778075, 0xbeff0071, + 0xf4601bfb, 0xf8000000, + 0xf4601b3b, 0xf8000004, + 0xf4601b7b, 0xf8000008, + 0xf4601c3b, 0xf800000c, + 0xf4601c7b, 0xf8000010, + 0xf4601ebb, 0xf8000014, + 0xf4601efb, 0xf8000018, + 0xf4601e7b, 0xf800001c, + 0xf4601cfb, 0xf8000020, + 0xf4601bbb, 0xf8000024, + 0xbf8a0000, 0xb96ef814, + 0xf4601bbb, 0xf8000028, + 0xbf8a0000, 0xb96ef815, + 0xf4601bbb, 0xf800002c, + 0xbf8a0000, 0xb96ef812, + 0xf4601bbb, 0xf8000030, + 0xbf8a0000, 0xb96ef813, + 0x8b6eff7f, 0x04000000, + 0xbfa10022, 0xf4601bbb, + 0xf8000038, 0xbf8a0000, + 0xbf0d806e, 0xbfa1001d, + 0x856e906e, 0x8b6e6e6e, + 0xbfa10003, 0xbe804ec1, + 0x816ec16e, 0xbfa0fffb, + 0xbef800ff, 0x00000080, + 0xbefd0081, 0xf4601bbb, + 0xf0000000, 0xbfc70000, + 0x80788478, 0x937eff6e, + 0x00070004, 0x847e907e, + 0x8c7d7e7d, 0xbe80517d, + 0x917dff7d, 0x007f0000, + 0x856e906e, 0x8b6e6e6e, + 0xbfa10003, 0xbe804e7d, + 0x816ec16e, 0xbfa0fffb, + 0x807d817d, 0xbf08907d, + 0xbfa1ffec, 0xf4601bbb, + 0xf800003c, 0xbfc70000, + 0xbf0d806e, 0xbfa1000c, + 0xbf0d9a7f, 0xbfa10002, + 0xbf068180, 0xbe804fc4, + 0xbf94fffc, 0xbfa10006, + 0x856e906e, 0x8b6e6e6e, + 0xbfa10003, 0xbe804ec3, + 0x816ec16e, 0xbfa0fffb, + 0xbefd006f, 0xbefe0070, + 0xbeff0071, 0xb979f822, + 0xb97b2011, 0x857b867b, + 0xb97b0191, 0x857b827b, + 0xb97bba11, 0xb973f801, + 0xb8ee3b05, 0x806e816e, + 0xbf0d9972, 0xbfa20002, + 0x846e896e, 0xbfa00001, + 0x846e8a6e, 0x806eff6e, + 0x000001c0, 0x806e746e, + 0x826f8075, 0xf4605c37, + 0xf8000010, 0xf4605d37, + 0xf8000020, 0xf4601e77, + 0xf8000034, 0xbf8a0000, + 0x856e9677, 0xb96e04a1, + 0x856e9577, 0xb96e0421, + 0x856e8e77, 0xb96e3021, + 0x8b6dff6d, 0x01ffffff, + 0x8bfe7e7e, 0x8bea6a6a, + 0xb97af804, 0xb8eef802, + 0xbf0c8b6e, 0xbfa20003, + 0xbe804fc2, 0xbf94fffe, + 0xbfa10001, 0xbe804ec4, + 0xbf94fffc, 0x857a897a, + 0xb97a0244, 0xbe804a6c, + 0xb8eef802, 0xbf0c8b6e, + 0xbfa20003, 0xbe804fc2, + 0xbf94fffe, 0xbfa10001, + 0xbe804ec4, 0xbf94fffc, + 0xbfb10000, 0xbf9f0000, + 0xbf9f0000, 0xbf9f0000, + 0xbf9f0000, 0xbf9f0000, +}; diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm index 07999b4649de..a807e7557e93 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx12.asm @@ -28,11 +28,27 @@ */ #define CHIP_GFX12 37 +#define CHIP_GC_12_0_3 38 + +#define HAVE_XNACK (ASIC_FAMILY == CHIP_GC_12_0_3) +#define HAVE_57BIT_ADDRESS (ASIC_FAMILY == CHIP_GC_12_0_3) +#define HAVE_BANKED_VGPRS (ASIC_FAMILY == CHIP_GC_12_0_3) +#define NUM_NAMED_BARRIERS (ASIC_FAMILY == CHIP_GC_12_0_3 ? 0x10 : 0) +#define HAVE_CLUSTER_BARRIER (ASIC_FAMILY == CHIP_GC_12_0_3) #define SINGLE_STEP_MISSED_WORKAROUND 1 //workaround for lost TRAP_AFTER_INST exception when SAVECTX raised #define HAVE_VALU_SGPR_HAZARD (ASIC_FAMILY == CHIP_GFX12) +#define WAVE32_ONLY (ASIC_FAMILY == CHIP_GC_12_0_3) +#define SAVE_TTMPS_IN_SGPR_BLOCK (ASIC_FAMILY >= CHIP_GC_12_0_3) + +#if HAVE_XNACK && !WAVE32_ONLY +# error +#endif -var SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK = 0x4 +#define ADDRESS_HI32_NUM_BITS ((HAVE_57BIT_ADDRESS ? 57 : 48) - 32) +#define ADDRESS_HI32_MASK ((1 << ADDRESS_HI32_NUM_BITS) - 1) + +var SQ_WAVE_STATE_PRIV_ALL_BARRIER_COMPLETE_MASK = 0x4 | (NUM_NAMED_BARRIERS ? 0x8 : 0) | (HAVE_CLUSTER_BARRIER ? 0x10000 : 0) var SQ_WAVE_STATE_PRIV_SCC_SHIFT = 9 var SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK = 0xC00 var SQ_WAVE_STATE_PRIV_HALT_MASK = 0x4000 @@ -41,6 +57,7 @@ var SQ_WAVE_STATE_PRIV_POISON_ERR_SHIFT = 15 var SQ_WAVE_STATUS_WAVE64_SHIFT = 29 var SQ_WAVE_STATUS_WAVE64_SIZE = 1 var SQ_WAVE_STATUS_NO_VGPRS_SHIFT = 24 +var SQ_WAVE_STATUS_IN_WG_SHIFT = 11 var SQ_WAVE_STATE_PRIV_ALWAYS_CLEAR_MASK = SQ_WAVE_STATE_PRIV_SYS_PRIO_MASK|SQ_WAVE_STATE_PRIV_POISON_ERR_MASK var S_SAVE_PC_HI_TRAP_ID_MASK = 0xF0000000 @@ -48,9 +65,12 @@ var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9 var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE = 8 var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT = 12 -var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT = 24 -var SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE = 4 + +#if ASIC_FAMILY < CHIP_GC_12_0_3 var SQ_WAVE_LDS_ALLOC_GRANULARITY = 9 +#else +var SQ_WAVE_LDS_ALLOC_GRANULARITY = 10 +#endif var SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK = 0xF var SQ_WAVE_EXCP_FLAG_PRIV_MEM_VIOL_MASK = 0x10 @@ -83,13 +103,38 @@ var SQ_WAVE_SCHED_MODE_DEP_MODE_SHIFT = 0 var SQ_WAVE_SCHED_MODE_DEP_MODE_SIZE = 2 var BARRIER_STATE_SIGNAL_OFFSET = 16 +var BARRIER_STATE_MEMBER_OFFSET = 4 +var BARRIER_STATE_MEMBER_SIZE = 7 var BARRIER_STATE_VALID_OFFSET = 0 var TTMP11_SCHED_MODE_SHIFT = 26 var TTMP11_SCHED_MODE_SIZE = 2 var TTMP11_SCHED_MODE_MASK = 0xC000000 + +var NAMED_BARRIERS_SR_OFFSET_FROM_HWREG = 0x80 +var S_BARRIER_INIT_MEMBERCNT_MASK = 0x7F0000 +var S_BARRIER_INIT_MEMBERCNT_SHIFT = 0x10 + +var SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SHIFT = 18 +var SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SIZE = 1 +var SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SHIFT = 16 +var SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SIZE = 1 +var SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SHIFT = 0 +var SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SIZE = 7 + +#if HAVE_BANKED_VGPRS +var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT = 12 +var SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SIZE = 6 +#endif + var TTMP11_DEBUG_TRAP_ENABLED_SHIFT = 23 var TTMP11_DEBUG_TRAP_ENABLED_MASK = 0x800000 +var TTMP11_FIRST_REPLAY_SHIFT = 22 +var TTMP11_FIRST_REPLAY_MASK = 0x400000 +var TTMP11_REPLAY_W64H_SHIFT = 21 +var TTMP11_REPLAY_W64H_MASK = 0x200000 +var TTMP11_FXPTR_SHIFT = 14 +var TTMP11_FXPTR_MASK = 0x1FC000 // SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] // when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE @@ -101,6 +146,11 @@ var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT = 26 var S_SAVE_PC_HI_FIRST_WAVE_MASK = 0x80000000 var S_SAVE_PC_HI_FIRST_WAVE_SHIFT = 31 +#if HAVE_BANKED_VGPRS +var S_SAVE_PC_HI_DST_SRC0_SRC1_VGPR_MSB_SHIFT = 25 +var S_SAVE_PC_HI_DST_SRC0_SRC1_VGPR_MSB_SIZE = 6 +#endif + var s_sgpr_save_num = 108 var s_save_spi_init_lo = exec_lo @@ -111,12 +161,12 @@ var s_save_exec_lo = ttmp2 var s_save_exec_hi = ttmp3 var s_save_state_priv = ttmp12 var s_save_excp_flag_priv = ttmp15 -var s_save_xnack_mask = s_save_excp_flag_priv +var s_save_xnack_mask = s_save_exec_hi var s_wave_size = ttmp7 -var s_save_buf_rsrc0 = ttmp8 -var s_save_buf_rsrc1 = ttmp9 -var s_save_buf_rsrc2 = ttmp10 -var s_save_buf_rsrc3 = ttmp11 +var s_save_base_addr_lo = ttmp8 +var s_save_base_addr_hi = ttmp9 +var s_save_addr_lo = ttmp10 +var s_save_addr_hi = ttmp11 var s_save_mem_offset = ttmp4 var s_save_alloc_size = s_save_excp_flag_priv var s_save_tmp = ttmp14 @@ -124,9 +174,6 @@ var s_save_m0 = ttmp5 var s_save_ttmps_lo = s_save_tmp var s_save_ttmps_hi = s_save_excp_flag_priv -var S_RESTORE_BUF_RSRC_WORD1_STRIDE = S_SAVE_BUF_RSRC_WORD1_STRIDE -var S_RESTORE_BUF_RSRC_WORD3_MISC = S_SAVE_BUF_RSRC_WORD3_MISC - var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK = 0x04000000 var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT = 26 var S_WAVE_SIZE = 25 @@ -147,15 +194,21 @@ var s_restore_exec_hi = ttmp5 var s_restore_state_priv = ttmp14 var s_restore_excp_flag_priv = ttmp15 var s_restore_xnack_mask = ttmp13 -var s_restore_buf_rsrc0 = ttmp8 -var s_restore_buf_rsrc1 = ttmp9 -var s_restore_buf_rsrc2 = ttmp10 -var s_restore_buf_rsrc3 = ttmp11 +var s_restore_base_addr_lo = ttmp8 +var s_restore_base_addr_hi = ttmp9 +var s_restore_addr_lo = ttmp10 +var s_restore_addr_hi = ttmp11 var s_restore_size = ttmp6 var s_restore_ttmps_lo = s_restore_tmp var s_restore_ttmps_hi = s_restore_alloc_size var s_restore_spi_init_hi_save = s_restore_exec_hi +#if SAVE_TTMPS_IN_SGPR_BLOCK +var TTMP_SR_OFFSET_FROM_HWREG = -0x40 +#else +var TTMP_SR_OFFSET_FROM_HWREG = 0x40 +#endif + shader main asic(DEFAULT) type(CS) @@ -245,6 +298,10 @@ L_CHECK_TRAP_ID: s_cbranch_scc1 L_SAVE L_FETCH_2ND_TRAP: +#if HAVE_XNACK + save_and_clear_xnack_state_priv(ttmp14) +#endif + // Read second-level TBA/TMA from first-level TMA and jump if available. // ttmp[2:5] and ttmp12 can be used (others hold SPI-initialized debug data) // ttmp12 holds SQ_WAVE_STATUS @@ -252,9 +309,9 @@ L_FETCH_2ND_TRAP: s_wait_idle s_lshl_b64 [ttmp14, ttmp15], [ttmp14, ttmp15], 0x8 - s_bitcmp1_b32 ttmp15, 0xF + s_bitcmp1_b32 ttmp15, (ADDRESS_HI32_NUM_BITS - 1) s_cbranch_scc0 L_NO_SIGN_EXTEND_TMA - s_or_b32 ttmp15, ttmp15, 0xFFFF0000 + s_or_b32 ttmp15, ttmp15, ~ADDRESS_HI32_MASK L_NO_SIGN_EXTEND_TMA: #if ASIC_FAMILY == CHIP_GFX12 // Move SCHED_MODE[1:0] from ttmp11 to unused bits in ttmp1[27:26] (return PC_HI). @@ -303,13 +360,13 @@ L_TRAP_CASE: s_addc_u32 ttmp1, ttmp1, 0x0 L_EXIT_TRAP: - s_and_b32 ttmp1, ttmp1, 0xFFFF + s_and_b32 ttmp1, ttmp1, ADDRESS_HI32_MASK // Restore SQ_WAVE_STATUS. s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 - // STATE_PRIV.BARRIER_COMPLETE may have changed since we read it. + // STATE_PRIV.*BARRIER_COMPLETE may have changed since we read it. // Only restore fields which the trap handler changes. s_lshr_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_SCC_SHIFT @@ -329,8 +386,7 @@ L_SAVE: s_cbranch_scc0 L_HAVE_VGPRS s_endpgm L_HAVE_VGPRS: - - s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff //pc[47:32] + s_and_b32 s_save_pc_hi, s_save_pc_hi, ADDRESS_HI32_MASK s_mov_b32 s_save_tmp, 0 s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, SQ_WAVE_EXCP_FLAG_PRIV_SAVE_CONTEXT_SHIFT, 1), s_save_tmp //clear saveCtx bit @@ -347,11 +403,25 @@ L_HAVE_VGPRS: s_lshl_b32 s_save_tmp, s_save_tmp, (S_SAVE_PC_HI_FIRST_WAVE_SHIFT - S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT) s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp +#if HAVE_XNACK + save_and_clear_xnack_state_priv(s_save_tmp) + s_getreg_b32 s_save_xnack_mask, hwreg(HW_REG_WAVE_XNACK_MASK) + s_setreg_imm32_b32 hwreg(HW_REG_WAVE_XNACK_MASK), 0 +#endif + +#if HAVE_BANKED_VGPRS + // Save and clear shader's DST/SRC0/SRC1 VGPR bank selection so we can use v[0-255]. + s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_MODE, SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT, SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SIZE) + s_lshl_b32 s_save_tmp, s_save_tmp, S_SAVE_PC_HI_DST_SRC0_SRC1_VGPR_MSB_SHIFT + s_or_b32 s_save_pc_hi, s_save_pc_hi, s_save_tmp + s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT, SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SIZE), 0 +#endif + // Trap temporaries must be saved via VGPR but all VGPRs are in use. // There is no ttmp space to hold the resource constant for VGPR save. // Save v0 by itself since it requires only two SGPRs. s_mov_b32 s_save_ttmps_lo, exec_lo - s_and_b32 s_save_ttmps_hi, exec_hi, 0xFFFF + s_and_b32 s_save_ttmps_hi, exec_hi, ADDRESS_HI32_MASK s_mov_b32 exec_lo, 0xFFFFFFFF s_mov_b32 exec_hi, 0xFFFFFFFF global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] scope:SCOPE_SYS @@ -360,13 +430,13 @@ L_HAVE_VGPRS: s_mov_b32 exec_hi, s_save_ttmps_hi // Save trap temporaries 4-11, 13 initialized by SPI debug dispatch logic - // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40 + // ttmp SR memory offset: + // - gfx12: size(VGPR)+size(SGPR)+0x40 + // - gfx12.5: size(VGPR)+size(SGPR)-0x40 get_wave_size2(s_save_ttmps_hi) get_vgpr_size_bytes(s_save_ttmps_lo, s_save_ttmps_hi) - get_svgpr_size_bytes(s_save_ttmps_hi) - s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_ttmps_hi - s_and_b32 s_save_ttmps_hi, s_save_spi_init_hi, 0xFFFF - s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, get_sgpr_size_bytes() + s_and_b32 s_save_ttmps_hi, s_save_spi_init_hi, ADDRESS_HI32_MASK + s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, (get_sgpr_size_bytes() + TTMP_SR_OFFSET_FROM_HWREG) s_add_u32 s_save_ttmps_lo, s_save_ttmps_lo, s_save_spi_init_lo s_addc_u32 s_save_ttmps_hi, s_save_ttmps_hi, 0x0 @@ -381,27 +451,19 @@ L_HAVE_VGPRS: v_writelane_b32 v0, ttmp13, 0xD v_writelane_b32 v0, exec_lo, 0xE v_writelane_b32 v0, exec_hi, 0xF - valu_sgpr_hazard() s_mov_b32 exec_lo, 0x3FFF s_mov_b32 exec_hi, 0x0 - global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] offset:0x40 scope:SCOPE_SYS + global_store_dword_addtid v0, [s_save_ttmps_lo, s_save_ttmps_hi] scope:SCOPE_SYS v_readlane_b32 ttmp14, v0, 0xE v_readlane_b32 ttmp15, v0, 0xF s_mov_b32 exec_lo, ttmp14 s_mov_b32 exec_hi, ttmp15 - /* setup Resource Contants */ - s_mov_b32 s_save_buf_rsrc0, s_save_spi_init_lo //base_addr_lo - s_and_b32 s_save_buf_rsrc1, s_save_spi_init_hi, 0x0000FFFF //base_addr_hi - s_or_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE - s_mov_b32 s_save_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited - s_mov_b32 s_save_buf_rsrc3, S_SAVE_BUF_RSRC_WORD3_MISC - + s_mov_b32 s_save_base_addr_lo, s_save_spi_init_lo + s_and_b32 s_save_base_addr_hi, s_save_spi_init_hi, ADDRESS_HI32_MASK s_mov_b32 s_save_m0, m0 - /* global mem offset */ - s_mov_b32 s_save_mem_offset, 0x0 get_wave_size2(s_wave_size) /* save first 4 VGPRs, needed for SGPR save */ @@ -416,102 +478,125 @@ L_ENABLE_SAVE_4VGPR_EXEC_HI: s_mov_b32 exec_hi, 0xFFFFFFFF s_branch L_SAVE_4VGPR_WAVE64 L_SAVE_4VGPR_WAVE32: - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR Allocated in 4-GPR granularity - - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128 - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128*2 - buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128*3 + global_store_addtid_b32 v1, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:128 + global_store_addtid_b32 v2, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:128*2 + global_store_addtid_b32 v3, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:128*3 s_branch L_SAVE_HWREG L_SAVE_4VGPR_WAVE64: - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR Allocated in 4-GPR granularity - - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256 - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256*2 - buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256*3 + global_store_addtid_b32 v1, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:256 + global_store_addtid_b32 v2, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:256*2 + global_store_addtid_b32 v3, [s_save_base_addr_lo, s_save_base_addr_hi] scope:SCOPE_SYS offset:256*3 /* save HW registers */ L_SAVE_HWREG: - // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR) + // HWREG SR memory offset : size(VGPR)+size(SGPR) get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) - get_svgpr_size_bytes(s_save_tmp) - s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes() - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - v_mov_b32 v0, 0x0 //Offset[31:0] from buffer resource v_mov_b32 v1, 0x0 //Offset[63:32] from buffer resource v_mov_b32 v2, 0x0 //Set of SGPRs for TCP store + s_mov_b32 m0, 0x0 //Next lane of v2 to write to // Ensure no further changes to barrier or LDS state. - // STATE_PRIV.BARRIER_COMPLETE may change up to this point. - s_barrier_signal -2 - s_barrier_wait -2 + // STATE_PRIV.*BARRIER_COMPLETE may change up to this point. + wait_trap_barriers(s_save_tmp) - // Re-read final state of BARRIER_COMPLETE field for save. + // Re-read final state of *BARRIER_COMPLETE fields for save. s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATE_PRIV) - s_and_b32 s_save_tmp, s_save_tmp, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK - s_andn2_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_MASK + s_and_b32 s_save_tmp, s_save_tmp, SQ_WAVE_STATE_PRIV_ALL_BARRIER_COMPLETE_MASK + s_andn2_b32 s_save_state_priv, s_save_state_priv, SQ_WAVE_STATE_PRIV_ALL_BARRIER_COMPLETE_MASK s_or_b32 s_save_state_priv, s_save_state_priv, s_save_tmp + write_hwreg_to_v2(s_save_m0) + write_hwreg_to_v2(s_save_pc_lo) s_andn2_b32 s_save_tmp, s_save_pc_hi, S_SAVE_PC_HI_FIRST_WAVE_MASK - v_writelane_b32 v2, s_save_m0, 0x0 - v_writelane_b32 v2, s_save_pc_lo, 0x1 - v_writelane_b32 v2, s_save_tmp, 0x2 - v_writelane_b32 v2, s_save_exec_lo, 0x3 - v_writelane_b32 v2, s_save_exec_hi, 0x4 - v_writelane_b32 v2, s_save_state_priv, 0x5 - v_writelane_b32 v2, s_save_xnack_mask, 0x7 - valu_sgpr_hazard() + write_hwreg_to_v2(s_save_tmp) + write_hwreg_to_v2(s_save_exec_lo) +#if WAVE32_ONLY + s_mov_b32 s_save_tmp, 0 + write_hwreg_to_v2(s_save_tmp) +#else + write_hwreg_to_v2(s_save_exec_hi) +#endif + write_hwreg_to_v2(s_save_state_priv) s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV) - v_writelane_b32 v2, s_save_tmp, 0x6 + write_hwreg_to_v2(s_save_tmp) - s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_MODE) - v_writelane_b32 v2, s_save_tmp, 0x8 +#if HAVE_XNACK + write_hwreg_to_v2(s_save_xnack_mask) +#else + s_mov_b32 s_save_tmp, 0 + write_hwreg_to_v2(s_save_tmp) +#endif - s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_SCRATCH_BASE_LO) - v_writelane_b32 v2, s_save_tmp, 0x9 + s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_MODE) - s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_SCRATCH_BASE_HI) - v_writelane_b32 v2, s_save_tmp, 0xA +#if HAVE_BANKED_VGPRS + s_bfe_u32 s_save_tmp, s_save_pc_hi, (S_SAVE_PC_HI_DST_SRC0_SRC1_VGPR_MSB_SHIFT | (S_SAVE_PC_HI_DST_SRC0_SRC1_VGPR_MSB_SIZE << 0x10)) + s_lshl_b32 s_save_tmp, s_save_tmp, SQ_WAVE_MODE_DST_SRC0_SRC1_VGPR_MSB_SHIFT + s_or_b32 s_save_m0, s_save_m0, s_save_tmp +#endif - s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_EXCP_FLAG_USER) - v_writelane_b32 v2, s_save_tmp, 0xB + write_hwreg_to_v2(s_save_m0) - s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_TRAP_CTRL) - v_writelane_b32 v2, s_save_tmp, 0xC + s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_SCRATCH_BASE_LO) + write_hwreg_to_v2(s_save_m0) + + s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_SCRATCH_BASE_HI) + write_hwreg_to_v2(s_save_m0) + + s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_EXCP_FLAG_USER) + write_hwreg_to_v2(s_save_m0) + + s_getreg_b32 s_save_m0, hwreg(HW_REG_WAVE_TRAP_CTRL) + write_hwreg_to_v2(s_save_m0) s_getreg_b32 s_save_tmp, hwreg(HW_REG_WAVE_STATUS) - v_writelane_b32 v2, s_save_tmp, 0xD + write_hwreg_to_v2(s_save_tmp) s_get_barrier_state s_save_tmp, -1 s_wait_kmcnt (0) - v_writelane_b32 v2, s_save_tmp, 0xE - valu_sgpr_hazard() + write_hwreg_to_v2(s_save_tmp) + +#if HAVE_CLUSTER_BARRIER + s_sendmsg_rtn_b32 s_save_tmp, sendmsg(MSG_RTN_GET_CLUSTER_BARRIER_STATE) + s_wait_kmcnt 0 + write_hwreg_to_v2(s_save_tmp) +#endif // Write HWREGs with 16 VGPR lanes. TTMPs occupy space after this. s_mov_b32 exec_lo, 0xFFFF s_mov_b32 exec_hi, 0x0 - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS // Write SGPRs with 32 VGPR lanes. This works in wave32 and wave64 mode. s_mov_b32 exec_lo, 0xFFFFFFFF +#if NUM_NAMED_BARRIERS + v_mov_b32 v2, 0 + + for var bar_idx = 0; bar_idx < NUM_NAMED_BARRIERS; bar_idx ++ + s_get_barrier_state s_save_tmp, (bar_idx + 1) + s_wait_kmcnt 0 + v_writelane_b32 v2, s_save_tmp, bar_idx + end + + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:NAMED_BARRIERS_SR_OFFSET_FROM_HWREG +#endif + /* save SGPRs */ // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save... - // SGPR SR memory offset : size(VGPR)+size(SVGPR) + // SGPR SR memory offset : size(VGPR) get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) - get_svgpr_size_bytes(s_save_tmp) - s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes s_mov_b32 ttmp13, 0x0 //next VGPR lane to copy SGPR into @@ -528,14 +613,14 @@ L_SAVE_SGPR_LOOP: s_movrels_b64 s12, s12 //s12 = s[12+m0], s13 = s[13+m0] s_movrels_b64 s14, s14 //s14 = s[14+m0], s15 = s[15+m0] - s_cmp_eq_u32 ttmp13, 0x0 - s_cbranch_scc0 L_WRITE_V2_SECOND_HALF - write_16sgpr_to_v2(s0, 0x0) - s_branch L_SAVE_SGPR_SKIP_TCP_STORE -L_WRITE_V2_SECOND_HALF: - write_16sgpr_to_v2(s0, 0x10) + write_16sgpr_to_v2(s0) - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS + s_cmp_eq_u32 ttmp13, 0x20 //have 32 VGPR lanes filled? + s_cbranch_scc0 L_SAVE_SGPR_SKIP_TCP_STORE + + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS s_add_u32 s_save_mem_offset, s_save_mem_offset, 0x80 s_mov_b32 ttmp13, 0x0 v_mov_b32 v2, 0x0 @@ -554,7 +639,14 @@ L_SAVE_SGPR_SKIP_TCP_STORE: s_movrels_b64 s10, s10 //s10 = s[10+m0], s11 = s[11+m0] write_12sgpr_to_v2(s0) - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS +#if SAVE_TTMPS_IN_SGPR_BLOCK + // Last 16 dwords of the SGPR block already contain the TTMPS. Make + // sure to not override them. + s_mov_b32 exec_lo, 0xFFFF +#endif + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS /* save LDS */ @@ -580,18 +672,13 @@ L_SAVE_LDS_NORMAL: // first wave do LDS save; s_lshl_b32 s_save_alloc_size, s_save_alloc_size, SQ_WAVE_LDS_ALLOC_GRANULARITY - s_mov_b32 s_save_buf_rsrc2, s_save_alloc_size //NUM_RECORDS in bytes - // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG) + // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG) // get_vgpr_size_bytes(s_save_mem_offset, s_wave_size) - get_svgpr_size_bytes(s_save_tmp) - s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp s_add_u32 s_save_mem_offset, s_save_mem_offset, get_sgpr_size_bytes() s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes() - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - //load 0~63*4(byte address) to vgpr v0 v_mbcnt_lo_u32_b32 v0, -1, 0 v_mbcnt_hi_u32_b32 v0, -1, v0 @@ -611,7 +698,9 @@ L_SAVE_LDS_W32: L_SAVE_LDS_LOOP_W32: ds_read_b32 v1, v0 s_wait_idle - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v1, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS s_add_u32 m0, m0, s3 //every buffer_store_lds does 128 bytes s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 @@ -629,7 +718,9 @@ L_SAVE_LDS_W64: L_SAVE_LDS_LOOP_W64: ds_read_b32 v1, v0 s_wait_idle - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v1, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS s_add_u32 m0, m0, s3 //every buffer_store_lds does 256 bytes s_add_u32 s_save_mem_offset, s_save_mem_offset, s3 @@ -662,8 +753,6 @@ L_SAVE_VGPR_NORMAL: s_cmp_eq_u32 m0, 1 s_cbranch_scc1 L_SAVE_VGPR_WAVE64 - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR Allocated in 4-GPR granularity // VGPR store using dw burst @@ -677,10 +766,12 @@ L_SAVE_VGPR_W32_LOOP: v_movrels_b32 v2, v2 //v2 = v[2+m0] v_movrels_b32 v3, v3 //v3 = v[3+m0] - buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128 - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128*2 - buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:128*3 + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v0, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS + global_store_addtid_b32 v1, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:128 + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:128*2 + global_store_addtid_b32 v3, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:128*3 s_add_u32 m0, m0, 4 //next vgpr index s_add_u32 s_save_mem_offset, s_save_mem_offset, 128*4 //every buffer_store_dword does 128 bytes @@ -690,12 +781,10 @@ L_SAVE_VGPR_W32_LOOP: s_branch L_SAVE_VGPR_END L_SAVE_VGPR_WAVE64: - s_mov_b32 s_save_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR store using dw burst s_mov_b32 m0, 0x4 //VGPR initial index value =4 s_cmp_lt_u32 m0, s_save_alloc_size - s_cbranch_scc0 L_SAVE_SHARED_VGPR + s_cbranch_scc0 L_SAVE_VGPR_END L_SAVE_VGPR_W64_LOOP: v_movrels_b32 v0, v0 //v0 = v[0+m0] @@ -703,45 +792,24 @@ L_SAVE_VGPR_W64_LOOP: v_movrels_b32 v2, v2 //v2 = v[2+m0] v_movrels_b32 v3, v3 //v3 = v[3+m0] - buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS - buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256 - buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256*2 - buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS offset:256*3 + s_add_u32 s_save_addr_lo, s_save_base_addr_lo, s_save_mem_offset + s_addc_u32 s_save_addr_hi, s_save_base_addr_hi, 0x0 + global_store_addtid_b32 v0, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS + global_store_addtid_b32 v1, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:256 + global_store_addtid_b32 v2, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:256*2 + global_store_addtid_b32 v3, [s_save_addr_lo, s_save_addr_hi] scope:SCOPE_SYS offset:256*3 s_add_u32 m0, m0, 4 //next vgpr index s_add_u32 s_save_mem_offset, s_save_mem_offset, 256*4 //every buffer_store_dword does 256 bytes s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0 s_cbranch_scc1 L_SAVE_VGPR_W64_LOOP //VGPR save is complete? -L_SAVE_SHARED_VGPR: - s_getreg_b32 s_save_alloc_size, hwreg(HW_REG_WAVE_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) - s_and_b32 s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero? - s_cbranch_scc0 L_SAVE_VGPR_END //no shared_vgpr used? jump to L_SAVE_LDS - s_lshl_b32 s_save_alloc_size, s_save_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value) - //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count. - //save shared_vgpr will start from the index of m0 - s_add_u32 s_save_alloc_size, s_save_alloc_size, m0 - s_mov_b32 exec_lo, 0xFFFFFFFF - s_mov_b32 exec_hi, 0x00000000 - -L_SAVE_SHARED_VGPR_WAVE64_LOOP: - v_movrels_b32 v0, v0 //v0 = v[0+m0] - buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset scope:SCOPE_SYS - s_add_u32 m0, m0, 1 //next vgpr index - s_add_u32 s_save_mem_offset, s_save_mem_offset, 128 - s_cmp_lt_u32 m0, s_save_alloc_size //scc = (m0 < s_save_alloc_size) ? 1 : 0 - s_cbranch_scc1 L_SAVE_SHARED_VGPR_WAVE64_LOOP //SHARED_VGPR save is complete? - L_SAVE_VGPR_END: s_branch L_END_PGM L_RESTORE: - /* Setup Resource Contants */ - s_mov_b32 s_restore_buf_rsrc0, s_restore_spi_init_lo //base_addr_lo - s_and_b32 s_restore_buf_rsrc1, s_restore_spi_init_hi, 0x0000FFFF //base_addr_hi - s_or_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE - s_mov_b32 s_restore_buf_rsrc2, 0 //NUM_RECORDS initial value = 0 (in bytes) - s_mov_b32 s_restore_buf_rsrc3, S_RESTORE_BUF_RSRC_WORD3_MISC + s_mov_b32 s_restore_base_addr_lo, s_restore_spi_init_lo + s_and_b32 s_restore_base_addr_hi, s_restore_spi_init_hi, ADDRESS_HI32_MASK // Save s_restore_spi_init_hi for later use. s_mov_b32 s_restore_spi_init_hi_save, s_restore_spi_init_hi @@ -768,28 +836,31 @@ L_RESTORE_LDS_NORMAL: s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //lds_size is zero? s_cbranch_scc0 L_RESTORE_VGPR //no lds used? jump to L_RESTORE_VGPR s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, SQ_WAVE_LDS_ALLOC_GRANULARITY - s_mov_b32 s_restore_buf_rsrc2, s_restore_alloc_size //NUM_RECORDS in bytes - // LDS at offset: size(VGPR)+size(SVGPR)+SIZE(SGPR)+SIZE(HWREG) + // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG) // get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) - get_svgpr_size_bytes(s_restore_tmp) - s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes() - s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - s_lshr_b32 m0, s_restore_size, S_WAVE_SIZE s_and_b32 m0, m0, 1 s_cmp_eq_u32 m0, 1 s_mov_b32 m0, 0x0 + + v_mbcnt_lo_u32_b32 v1, -1, 0 + v_mbcnt_hi_u32_b32 v1, -1, v1 + v_lshlrev_b32 v1, 2, v1 // 0, 4, 8, ... 124 (W32) or 252 (W64) + s_cbranch_scc1 L_RESTORE_LDS_LOOP_W64 L_RESTORE_LDS_LOOP_W32: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS s_wait_idle - ds_store_addtid_b32 v0 + ds_store_b32 v1, v0 + v_add_nc_u32 v1, v1, 128 s_add_u32 m0, m0, 128 // 128 DW s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 //mem offset increased by 128DW s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0 @@ -797,9 +868,12 @@ L_RESTORE_LDS_LOOP_W32: s_branch L_RESTORE_VGPR L_RESTORE_LDS_LOOP_W64: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS s_wait_idle - ds_store_addtid_b32 v0 + ds_store_b32 v1, v0 + v_add_nc_u32 v1, v1, 256 s_add_u32 m0, m0, 256 // 256 DW s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256 //mem offset increased by 256DW s_cmp_lt_u32 m0, s_restore_alloc_size //scc=(m0 < s_restore_alloc_size) ? 1 : 0 @@ -828,20 +902,18 @@ L_RESTORE_VGPR_NORMAL: s_cmp_eq_u32 m0, 1 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64 - s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR load using dw burst s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v1, v0 will be the last s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128*4 s_mov_b32 m0, 4 //VGPR initial index value = 4 - s_cmp_lt_u32 m0, s_restore_alloc_size - s_cbranch_scc0 L_RESTORE_SGPR L_RESTORE_VGPR_WAVE32_LOOP: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS - buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:128 - buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:128*2 - buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:128*3 + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS + global_load_addtid_b32 v1, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128 + global_load_addtid_b32 v2, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128*2 + global_load_addtid_b32 v3, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128*3 s_wait_idle v_movreld_b32 v0, v0 //v[0+m0] = v0 v_movreld_b32 v1, v1 @@ -853,29 +925,31 @@ L_RESTORE_VGPR_WAVE32_LOOP: s_cbranch_scc1 L_RESTORE_VGPR_WAVE32_LOOP //VGPR restore (except v0) is complete? /* VGPR restore on v0 */ - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS - buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:128 - buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:128*2 - buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:128*3 + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset_save + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS + global_load_addtid_b32 v1, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128 + global_load_addtid_b32 v2, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128*2 + global_load_addtid_b32 v3, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:128*3 s_wait_idle s_branch L_RESTORE_SGPR L_RESTORE_VGPR_WAVE64: - s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes - // VGPR load using dw burst s_mov_b32 s_restore_mem_offset_save, s_restore_mem_offset // restore start with v4, v0 will be the last s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 256*4 s_mov_b32 m0, 4 //VGPR initial index value = 4 s_cmp_lt_u32 m0, s_restore_alloc_size - s_cbranch_scc0 L_RESTORE_SHARED_VGPR + s_cbranch_scc0 L_RESTORE_V0 L_RESTORE_VGPR_WAVE64_LOOP: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS - buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:256 - buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:256*2 - buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS offset:256*3 + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS + global_load_addtid_b32 v1, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256 + global_load_addtid_b32 v2, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256*2 + global_load_addtid_b32 v3, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256*3 s_wait_idle v_movreld_b32 v0, v0 //v[0+m0] = v0 v_movreld_b32 v1, v1 @@ -886,50 +960,29 @@ L_RESTORE_VGPR_WAVE64_LOOP: s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0 s_cbranch_scc1 L_RESTORE_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete? -L_RESTORE_SHARED_VGPR: - s_getreg_b32 s_restore_alloc_size, hwreg(HW_REG_WAVE_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) //shared_vgpr_size - s_and_b32 s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF //shared_vgpr_size is zero? - s_cbranch_scc0 L_RESTORE_V0 //no shared_vgpr used? - s_lshl_b32 s_restore_alloc_size, s_restore_alloc_size, 3 //Number of SHARED_VGPRs = shared_vgpr_size * 8 (non-zero value) - //m0 now has the value of normal vgpr count, just add the m0 with shared_vgpr count to get the total count. - //restore shared_vgpr will start from the index of m0 - s_add_u32 s_restore_alloc_size, s_restore_alloc_size, m0 - s_mov_b32 exec_lo, 0xFFFFFFFF - s_mov_b32 exec_hi, 0x00000000 -L_RESTORE_SHARED_VGPR_WAVE64_LOOP: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset scope:SCOPE_SYS - s_wait_idle - v_movreld_b32 v0, v0 //v[0+m0] = v0 - s_add_u32 m0, m0, 1 //next vgpr index - s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 128 - s_cmp_lt_u32 m0, s_restore_alloc_size //scc = (m0 < s_restore_alloc_size) ? 1 : 0 - s_cbranch_scc1 L_RESTORE_SHARED_VGPR_WAVE64_LOOP //VGPR restore (except v0) is complete? - - s_mov_b32 exec_hi, 0xFFFFFFFF //restore back exec_hi before restoring V0!! - /* VGPR restore on v0 */ L_RESTORE_V0: - buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS - buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:256 - buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:256*2 - buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save scope:SCOPE_SYS offset:256*3 + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset_save + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 + global_load_addtid_b32 v0, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS + global_load_addtid_b32 v1, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256 + global_load_addtid_b32 v2, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256*2 + global_load_addtid_b32 v3, [s_restore_addr_lo, s_restore_addr_hi] scope:SCOPE_SYS offset:256*3 s_wait_idle /* restore SGPRs */ //will be 2+8+16*6 - // SGPR SR memory offset : size(VGPR)+size(SVGPR) + // SGPR SR memory offset : size(VGPR) L_RESTORE_SGPR: get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) - get_svgpr_size_bytes(s_restore_tmp) - s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() - s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 20*4 //s108~s127 is not saved - - s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes + s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 24*4 // s[104:107] + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 s_mov_b32 m0, s_sgpr_save_num - read_4sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b128 s0, [s_restore_addr_lo, s_restore_addr_hi], 0x0 scope:SCOPE_SYS s_wait_idle s_sub_u32 m0, m0, 4 // Restore from S[0] to S[104] @@ -938,7 +991,9 @@ L_RESTORE_SGPR: s_movreld_b64 s0, s0 //s[0+m0] = s0 s_movreld_b64 s2, s2 - read_8sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) + s_sub_co_u32 s_restore_addr_lo, s_restore_addr_lo, 8*4 // s[96:103] + s_sub_co_ci_u32 s_restore_addr_hi, s_restore_addr_hi, 0 + s_load_b256 s0, [s_restore_addr_lo, s_restore_addr_hi], 0x0 scope:SCOPE_SYS s_wait_idle s_sub_u32 m0, m0, 8 // Restore from S[0] to S[96] @@ -950,7 +1005,9 @@ L_RESTORE_SGPR: s_movreld_b64 s6, s6 L_RESTORE_SGPR_LOOP: - read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset) + s_sub_co_u32 s_restore_addr_lo, s_restore_addr_lo, 16*4 // s[0,16,32,48,64,80] + s_sub_co_ci_u32 s_restore_addr_hi, s_restore_addr_hi, 0 + s_load_b512 s0, [s_restore_addr_lo, s_restore_addr_hi], 0x0 scope:SCOPE_SYS s_wait_idle s_sub_u32 m0, m0, 16 // Restore from S[n] to S[0] @@ -974,76 +1031,117 @@ L_RESTORE_SGPR: /* restore HW registers */ L_RESTORE_HWREG: - // HWREG SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR) + // HWREG SR memory offset : size(VGPR)+size(SGPR) get_vgpr_size_bytes(s_restore_mem_offset, s_restore_size) - get_svgpr_size_bytes(s_restore_tmp) - s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp s_add_u32 s_restore_mem_offset, s_restore_mem_offset, get_sgpr_size_bytes() - - s_mov_b32 s_restore_buf_rsrc2, 0x1000000 //NUM_RECORDS in bytes + s_add_u32 s_restore_addr_lo, s_restore_base_addr_lo, s_restore_mem_offset + s_addc_u32 s_restore_addr_hi, s_restore_base_addr_hi, 0x0 // Restore s_restore_spi_init_hi before the saved value gets clobbered. s_mov_b32 s_restore_spi_init_hi, s_restore_spi_init_hi_save - read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_state_priv, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_excp_flag_priv, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_xnack_mask, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset) - read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b32 s_restore_m0, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS + s_load_b32 s_restore_pc_lo, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x4 + s_load_b32 s_restore_pc_hi, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x8 + s_load_b32 s_restore_exec_lo, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0xC + s_load_b32 s_restore_exec_hi, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x10 + s_load_b32 s_restore_state_priv, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x14 + s_load_b32 s_restore_excp_flag_priv, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x18 + s_load_b32 s_restore_xnack_mask, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x1C + s_load_b32 s_restore_mode, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x20 + s_load_b32 s_restore_flat_scratch, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x24 s_wait_idle s_setreg_b32 hwreg(HW_REG_WAVE_SCRATCH_BASE_LO), s_restore_flat_scratch - read_hwreg_from_mem(s_restore_flat_scratch, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b32 s_restore_flat_scratch, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x28 s_wait_idle s_setreg_b32 hwreg(HW_REG_WAVE_SCRATCH_BASE_HI), s_restore_flat_scratch - read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b32 s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x2C s_wait_idle s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_USER), s_restore_tmp - read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b32 s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x30 s_wait_idle s_setreg_b32 hwreg(HW_REG_WAVE_TRAP_CTRL), s_restore_tmp - // Only the first wave needs to restore the workgroup barrier. + // Only the first wave needs to restore group barriers. s_and_b32 s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK - s_cbranch_scc0 L_SKIP_BARRIER_RESTORE + s_cbranch_scc0 L_SKIP_GROUP_BARRIER_RESTORE // Skip over WAVE_STATUS, since there is no state to restore from it - s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 4 - read_hwreg_from_mem(s_restore_tmp, s_restore_buf_rsrc0, s_restore_mem_offset) + s_load_b32 s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x38 s_wait_idle + // Skip group barriers if wave is not part of a group. s_bitcmp1_b32 s_restore_tmp, BARRIER_STATE_VALID_OFFSET - s_cbranch_scc0 L_SKIP_BARRIER_RESTORE + s_cbranch_scc0 L_SKIP_GROUP_BARRIER_RESTORE - // extract the saved signal count from s_restore_tmp - s_lshr_b32 s_restore_tmp, s_restore_tmp, BARRIER_STATE_SIGNAL_OFFSET + // Restore workgroup barrier signal count. + restore_barrier_signal_count(-1) - // We need to call s_barrier_signal repeatedly to restore the signal - // count of the work group barrier. The member count is already - // initialized with the number of waves in the work group. -L_BARRIER_RESTORE_LOOP: - s_and_b32 s_restore_tmp, s_restore_tmp, s_restore_tmp - s_cbranch_scc0 L_SKIP_BARRIER_RESTORE - s_barrier_signal -1 - s_add_i32 s_restore_tmp, s_restore_tmp, -1 - s_branch L_BARRIER_RESTORE_LOOP +#if NUM_NAMED_BARRIERS + s_mov_b32 s_restore_mem_offset, NAMED_BARRIERS_SR_OFFSET_FROM_HWREG + s_mov_b32 m0, 1 + +L_RESTORE_NAMED_BARRIER_LOOP: + s_load_b32 s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], s_restore_mem_offset scope:SCOPE_SYS + s_wait_kmcnt 0 + s_add_u32 s_restore_mem_offset, s_restore_mem_offset, 0x4 + + // Restore named barrier member count. + s_bfe_u32 exec_lo, s_restore_tmp, (BARRIER_STATE_MEMBER_OFFSET | (BARRIER_STATE_MEMBER_SIZE << 16)) + s_lshl_b32 exec_lo, exec_lo, S_BARRIER_INIT_MEMBERCNT_SHIFT + s_or_b32 m0, m0, exec_lo + s_barrier_init m0 + s_andn2_b32 m0, m0, S_BARRIER_INIT_MEMBERCNT_MASK -L_SKIP_BARRIER_RESTORE: + // Restore named barrier signal count. + restore_barrier_signal_count(m0) + + s_add_u32 m0, m0, 1 + s_cmp_gt_u32 m0, NUM_NAMED_BARRIERS + s_cbranch_scc0 L_RESTORE_NAMED_BARRIER_LOOP +#endif + +L_SKIP_GROUP_BARRIER_RESTORE: +#if HAVE_CLUSTER_BARRIER + s_load_b32 s_restore_tmp, [s_restore_addr_lo, s_restore_addr_hi], null scope:SCOPE_SYS offset:0x3C + s_wait_kmcnt 0 + + // Skip cluster barrier restore if wave is not part of a cluster. + s_bitcmp1_b32 s_restore_tmp, BARRIER_STATE_VALID_OFFSET + s_cbranch_scc0 L_SKIP_CLUSTER_BARRIER_RESTORE + + // Only the first wave in the group signals the trap cluster barrier. + s_bitcmp1_b32 s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT + s_cbranch_scc0 L_SKIP_TRAP_CLUSTER_BARRIER_SIGNAL + + // Clear SCC: s_barrier_signal_isfirst -4 writes SCC=>1 but not SCC=>0. + s_cmp_eq_u32 0, 1 + s_barrier_signal_isfirst -4 +L_SKIP_TRAP_CLUSTER_BARRIER_SIGNAL: + s_barrier_wait -4 + + // Only the first wave in the cluster restores the barrier. + s_cbranch_scc0 L_SKIP_CLUSTER_BARRIER_RESTORE + + // Restore cluster barrier signal count. + restore_barrier_signal_count(-3) +L_SKIP_CLUSTER_BARRIER_RESTORE: +#endif s_mov_b32 m0, s_restore_m0 s_mov_b32 exec_lo, s_restore_exec_lo s_mov_b32 exec_hi, s_restore_exec_hi +#if HAVE_XNACK + s_setreg_b32 hwreg(HW_REG_WAVE_XNACK_MASK), s_restore_xnack_mask +#endif + // EXCP_FLAG_PRIV.SAVE_CONTEXT and HOST_TRAP may have changed. // Only restore the other fields to avoid clobbering them. s_setreg_b32 hwreg(HW_REG_WAVE_EXCP_FLAG_PRIV, 0, SQ_WAVE_EXCP_FLAG_PRIV_RESTORE_PART_1_SIZE), s_restore_excp_flag_priv @@ -1055,20 +1153,23 @@ L_SKIP_BARRIER_RESTORE: s_setreg_b32 hwreg(HW_REG_WAVE_MODE), s_restore_mode // Restore trap temporaries 4-11, 13 initialized by SPI debug dispatch logic - // ttmp SR memory offset : size(VGPR)+size(SVGPR)+size(SGPR)+0x40 + // ttmp SR memory offset : + // - gfx12: size(VGPR)+size(SGPR)+0x40 + // - gfx12.5: size(VGPR)+size(SGPR)-0x40 get_vgpr_size_bytes(s_restore_ttmps_lo, s_restore_size) - get_svgpr_size_bytes(s_restore_ttmps_hi) - s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_ttmps_hi - s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, get_sgpr_size_bytes() - s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_buf_rsrc0 - s_addc_u32 s_restore_ttmps_hi, s_restore_buf_rsrc1, 0x0 - s_and_b32 s_restore_ttmps_hi, s_restore_ttmps_hi, 0xFFFF - s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x50 scope:SCOPE_SYS - s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x60 scope:SCOPE_SYS - s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x74 scope:SCOPE_SYS + s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, (get_sgpr_size_bytes() + TTMP_SR_OFFSET_FROM_HWREG) + s_add_u32 s_restore_ttmps_lo, s_restore_ttmps_lo, s_restore_base_addr_lo + s_addc_u32 s_restore_ttmps_hi, s_restore_base_addr_hi, 0x0 + s_load_dwordx4 [ttmp4, ttmp5, ttmp6, ttmp7], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x10 scope:SCOPE_SYS + s_load_dwordx4 [ttmp8, ttmp9, ttmp10, ttmp11], [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x20 scope:SCOPE_SYS + s_load_dword ttmp13, [s_restore_ttmps_lo, s_restore_ttmps_hi], 0x34 scope:SCOPE_SYS s_wait_idle - s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS +#if HAVE_XNACK + restore_xnack_state_priv(s_restore_tmp) +#endif + + s_and_b32 s_restore_pc_hi, s_restore_pc_hi, ADDRESS_HI32_MASK //Do it here in order not to affect STATUS s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 @@ -1077,56 +1178,47 @@ L_SKIP_BARRIER_RESTORE: s_setreg_b32 hwreg(HW_REG_WAVE_STATE_PRIV), s_restore_state_priv // SCC is included, which is changed by previous salu - // Make barrier and LDS state visible to all waves in the group. - // STATE_PRIV.BARRIER_COMPLETE may change after this point. - s_barrier_signal -2 - s_barrier_wait -2 + // Make barrier and LDS state visible to all waves in the group/cluster. + // STATE_PRIV.*BARRIER_COMPLETE may change after this point. + wait_trap_barriers(s_restore_tmp) + +#if HAVE_CLUSTER_BARRIER + // SCC is changed by wait_trap_barriers, restore it separately. + s_lshr_b32 s_restore_state_priv, s_restore_state_priv, SQ_WAVE_STATE_PRIV_SCC_SHIFT + s_setreg_b32 hwreg(HW_REG_WAVE_STATE_PRIV, SQ_WAVE_STATE_PRIV_SCC_SHIFT, 1), s_restore_state_priv +#endif s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution L_END_PGM: - // Make sure that no wave of the workgroup can exit the trap handler - // before the workgroup barrier state is saved. - s_barrier_signal -2 - s_barrier_wait -2 + // Make sure that no wave of the group/cluster can exit the trap handler + // before the group/cluster barrier state is saved. + wait_trap_barriers(s_restore_tmp) + s_endpgm_saved end -function write_16sgpr_to_v2(s, lane_offset) +function write_hwreg_to_v2(s) + // Copy into VGPR for later TCP store. + v_writelane_b32 v2, s, m0 + s_add_u32 m0, m0, 0x1 +end + + +function write_16sgpr_to_v2(s) // Copy into VGPR for later TCP store. for var sgpr_idx = 0; sgpr_idx < 16; sgpr_idx ++ - v_writelane_b32 v2, s[sgpr_idx], sgpr_idx + lane_offset + v_writelane_b32 v2, s[sgpr_idx], ttmp13 + s_add_u32 ttmp13, ttmp13, 0x1 end - valu_sgpr_hazard() - s_add_u32 ttmp13, ttmp13, 0x10 end function write_12sgpr_to_v2(s) // Copy into VGPR for later TCP store. for var sgpr_idx = 0; sgpr_idx < 12; sgpr_idx ++ - v_writelane_b32 v2, s[sgpr_idx], sgpr_idx + v_writelane_b32 v2, s[sgpr_idx], ttmp13 + s_add_u32 ttmp13, ttmp13, 0x1 end - valu_sgpr_hazard() -end - -function read_hwreg_from_mem(s, s_rsrc, s_mem_offset) - s_buffer_load_dword s, s_rsrc, s_mem_offset scope:SCOPE_SYS - s_add_u32 s_mem_offset, s_mem_offset, 4 -end - -function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset) - s_sub_u32 s_mem_offset, s_mem_offset, 4*16 - s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset scope:SCOPE_SYS -end - -function read_8sgpr_from_mem(s, s_rsrc, s_mem_offset) - s_sub_u32 s_mem_offset, s_mem_offset, 4*8 - s_buffer_load_dwordx8 s, s_rsrc, s_mem_offset scope:SCOPE_SYS -end - -function read_4sgpr_from_mem(s, s_rsrc, s_mem_offset) - s_sub_u32 s_mem_offset, s_mem_offset, 4*4 - s_buffer_load_dwordx4 s, s_rsrc, s_mem_offset scope:SCOPE_SYS end function get_vgpr_size_bytes(s_vgpr_size_byte, s_size) @@ -1141,17 +1233,16 @@ L_ENABLE_SHIFT_W64: L_SHIFT_DONE: end -function get_svgpr_size_bytes(s_svgpr_size_byte) - s_getreg_b32 s_svgpr_size_byte, hwreg(HW_REG_WAVE_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_VGPR_SHARED_SIZE_SIZE) - s_lshl_b32 s_svgpr_size_byte, s_svgpr_size_byte, (3+7) -end - function get_sgpr_size_bytes return 512 end function get_hwreg_size_bytes +#if ASIC_FAMILY >= CHIP_GC_12_0_3 + return 512 +#else return 128 +#endif end function get_wave_size2(s_reg) @@ -1159,15 +1250,80 @@ function get_wave_size2(s_reg) s_lshl_b32 s_reg, s_reg, S_WAVE_SIZE end -function valu_sgpr_hazard -#if HAVE_VALU_SGPR_HAZARD - for var rep = 0; rep < 8; rep ++ - ds_nop - end +#if HAVE_XNACK +function save_and_clear_xnack_state_priv(s_tmp) + // Preserve and clear XNACK state before issuing further translations. + // Save XNACK_STATE_PRIV.{FIRST_REPLAY, REPLAY_W64H, FXPTR} into ttmp11[22:14]. + s_andn2_b32 ttmp11, ttmp11, (TTMP11_FIRST_REPLAY_MASK | TTMP11_REPLAY_W64H_MASK | TTMP11_FXPTR_MASK) + + s_getreg_b32 s_tmp, hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SIZE) + s_lshl_b32 s_tmp, s_tmp, TTMP11_FIRST_REPLAY_SHIFT + s_or_b32 ttmp11, ttmp11, s_tmp + + s_getreg_b32 s_tmp, hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SIZE) + s_lshl_b32 s_tmp, s_tmp, TTMP11_REPLAY_W64H_SHIFT + s_or_b32 ttmp11, ttmp11, s_tmp + + s_getreg_b32 s_tmp, hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SIZE) + s_lshl_b32 s_tmp, s_tmp, TTMP11_FXPTR_SHIFT + s_or_b32 ttmp11, ttmp11, s_tmp + + s_setreg_imm32_b32 hwreg(HW_REG_WAVE_XNACK_STATE_PRIV), 0 +end + +function restore_xnack_state_priv(s_tmp) + s_lshr_b32 s_tmp, ttmp11, TTMP11_FIRST_REPLAY_SHIFT + s_setreg_b32 hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_FIRST_REPLAY_SIZE), s_tmp + + s_lshr_b32 s_tmp, ttmp11, TTMP11_REPLAY_W64H_SHIFT + s_setreg_b32 hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_REPLAY_W64H_SIZE), s_tmp + + s_lshr_b32 s_tmp, ttmp11, TTMP11_FXPTR_SHIFT + s_setreg_b32 hwreg(HW_REG_WAVE_XNACK_STATE_PRIV, SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SHIFT, SQ_WAVE_XNACK_STATE_PRIV_FXPTR_SIZE), s_tmp +end +#endif + +function wait_trap_barriers(s_tmp) +#if HAVE_CLUSTER_BARRIER + // If not in a WG then wave cannot use s_barrier_signal_isfirst. + s_getreg_b32 s_tmp, hwreg(HW_REG_WAVE_STATUS) + s_bitcmp0_b32 s_tmp, SQ_WAVE_STATUS_IN_WG_SHIFT + s_cbranch_scc1 L_TRAP_CLUSTER_BARRIER_SIGNAL + + s_barrier_signal_isfirst -2 + s_barrier_wait -2 + + // Only the first wave in the group signals the trap cluster barrier. + s_cbranch_scc0 L_SKIP_TRAP_CLUSTER_BARRIER_SIGNAL + +L_TRAP_CLUSTER_BARRIER_SIGNAL: + s_barrier_signal -4 + +L_SKIP_TRAP_CLUSTER_BARRIER_SIGNAL: + s_barrier_wait -4 +#else + s_barrier_signal -2 + s_barrier_wait -2 #endif end + function restore_sched_mode(s_tmp) s_bfe_u32 s_tmp, ttmp11, (TTMP11_SCHED_MODE_SHIFT | (TTMP11_SCHED_MODE_SIZE << 0x10)) s_setreg_b32 hwreg(HW_REG_WAVE_SCHED_MODE), s_tmp + +function restore_barrier_signal_count(barrier_id) + // extract the saved signal count from s_restore_tmp + s_lshr_b32 s_restore_tmp, s_restore_tmp, BARRIER_STATE_SIGNAL_OFFSET + + // We need to call s_barrier_signal repeatedly to restore the signal count + // of the group/cluster barrier. The member count is already initialized. +L_BARRIER_RESTORE_LOOP: + s_and_b32 s_restore_tmp, s_restore_tmp, s_restore_tmp + s_cbranch_scc0 L_BARRIER_RESTORE_DONE + s_barrier_signal barrier_id + s_add_i32 s_restore_tmp, s_restore_tmp, -1 + s_branch L_BARRIER_RESTORE_LOOP + +L_BARRIER_RESTORE_DONE: end diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 1c430b307bde..db37c2949d19 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -545,11 +545,16 @@ static void kfd_cwsr_init(struct kfd_dev *kfd) BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE); kfd->cwsr_isa = cwsr_trap_gfx11_hex; kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx11_hex); - } else { + } else if (KFD_GC_VERSION(kfd) < IP_VERSION(12, 1, 0)) { BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) > KFD_CWSR_TMA_OFFSET); kfd->cwsr_isa = cwsr_trap_gfx12_hex; kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_hex); + } else { + BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_1_0_hex) + > KFD_CWSR_TMA_OFFSET); + kfd->cwsr_isa = cwsr_trap_gfx12_1_0_hex; + kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_1_0_hex); } kfd->cwsr_enabled = true; |